diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index bfb175ad..6998eafe 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -17,6 +17,7 @@ on:
pull_request:
branches:
- fast-dev
+ - fast-dev-gke
- master
tags:
- ci
@@ -61,13 +62,3 @@ jobs:
id: documentation-links-fabric
run: |
python3 tools/check_links.py .
-
- # markdown-link-check:
- # runs-on: ubuntu-latest
- # steps:
- # - uses: actions/checkout@master
- # - uses: gaurav-nelson/github-action-markdown-link-check@v1
- # with:
- # use-quiet-mode: "yes"
- # use-verbose-mode: "yes"
- # config-file: ".github/workflows/markdown-link-check.json"
diff --git a/.github/workflows/markdown-link-check.json b/.github/workflows/markdown-link-check.json
deleted file mode 100644
index 52b3f3d0..00000000
--- a/.github/workflows/markdown-link-check.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "aliveStatusCodes": [429, 200],
- "retryOn429": false,
- "ignorePatterns": [
- {
- "pattern": "^https://medium.com"
- }
- ]
-}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 1eeeb722..5375865f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -19,6 +19,7 @@ on:
pull_request:
branches:
- fast-dev
+ - fast-dev-gke
- master
tags:
- ci
@@ -44,7 +45,7 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.9"
-
+
- name: Run tests on documentation examples
run: |
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
@@ -71,7 +72,7 @@ jobs:
with:
terraform_version: 1.1.4
terraform_wrapper: false
-
+
- name: Run tests environments
run: |
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
@@ -98,9 +99,9 @@ jobs:
with:
terraform_version: 1.1.4
terraform_wrapper: false
-
+
- name: Run tests modules
- run: |
+ run: |
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
pip install -r tests/requirements.txt
pytest -vv tests/modules
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 796c934f..0cdb189c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file.
- support service dependencies for crypto key bindings in project module
- refactor project module in multiple files
- add support for per-file option overrides to tfdoc
+- the `net-vpc` and `project` modules now use the beta provider for shared VPC-related resources
## [12.0.0] - 2022-01-11
diff --git a/README.md b/README.md
index c1fb07bc..782fd5bc 100644
--- a/README.md
+++ b/README.md
@@ -12,8 +12,6 @@ This repository provides **end-to-end examples** and a **suite of Terraform modu
The whole repository is meant to be cloned as a single unit, and then forked into separate owned repositories to seed production usage, or used as-is and periodically updated as a complete toolkit for prototyping. You can read more on this approach in our [manifesto](./MANIFESTO.md).
-Both the examples and modules require some measure of Terraform skills to be used effectively. If you are looking for a feature-rich black box to manage project or product creation with minimal specific skills, you might be better served by the [Cloud Foundation Toolkit](https://registry.terraform.io/modules/terraform-google-modules) suite of modules.
-
## Organization blueprint (Fabric FAST)
Setting up a production-ready GCP organization is often a time-consuming process. Fabric [FAST](fast/) aims to speed up this process via two complementary goals. On the one hand, FAST provides a design of a GCP organization that includes the typical elements required by enterprise customers. Secondly, we provide a reference implementation of the FAST design using Terraform.
diff --git a/examples/factories/project-factory/main.tf b/examples/factories/project-factory/main.tf
index a49fb993..c4928371 100644
--- a/examples/factories/project-factory/main.tf
+++ b/examples/factories/project-factory/main.tf
@@ -73,8 +73,13 @@ locals {
}
labels = merge(coalesce(var.labels, {}), coalesce(var.defaults.labels, {}))
network_user_service_accounts = concat(
- contains(local.services, "compute.googleapis.com") ? ["serviceAccount:${local.service_accounts_robots.compute}"] : [],
- contains(local.services, "container.googleapis.com") ? ["serviceAccount:${local.service_accounts_robots.container-engine}"] : [],
+ contains(local.services, "compute.googleapis.com") ? [
+ "serviceAccount:${local.service_accounts_robots.compute}"
+ ] : [],
+ contains(local.services, "container.googleapis.com") ? [
+ "serviceAccount:${local.service_accounts_robots.container-engine}",
+ "serviceAccount:${local.service_accounts.cloud_services}"
+ ] : [],
[])
services = distinct(concat(var.services, local._services))
service_accounts_robots = {
diff --git a/fast/stages/00-bootstrap/IAM.md b/fast/stages/00-bootstrap/IAM.md
new file mode 100644
index 00000000..1daaeee0
--- /dev/null
+++ b/fast/stages/00-bootstrap/IAM.md
@@ -0,0 +1,36 @@
+# IAM bindings reference
+
+Legend: + additive, • conditional.
+
+## Organization [org_id #0]
+
+| members | roles |
+|---|---|
+|
domain|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser)
[roles/resourcemanager.organizationViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationViewer) |
+|gcp-network-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) +
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) +|
+|gcp-organization-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.admin)
[roles/compute.osAdminLogin](https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin)
[roles/compute.osLoginExternalUser](https://cloud.google.com/iam/docs/understanding-roles#compute.osLoginExternalUser)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +|
+|gcp-security-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/iam.securityReviewer](https://cloud.google.com/iam/docs/understanding-roles#iam.securityReviewer)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/securitycenter.admin](https://cloud.google.com/iam/docs/understanding-roles#securitycenter.admin)
[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) +
[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +|
+|gcp-support
group|[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/logging.viewer](https://cloud.google.com/iam/docs/understanding-roles#logging.viewer)
[roles/monitoring.viewer](https://cloud.google.com/iam/docs/understanding-roles#monitoring.viewer) |
+|prod-bootstrap-0
serviceAccount|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) +|
+|prod-resman-0
serviceAccount|organizations/[org_id #0]/roles/organizationIamAdmin •
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +|
+
+## Project prod-audit-logs-0
+
+| members | roles |
+|---|---|
+|prod-bootstrap-0
serviceAccount|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
+
+## Project prod-billing-export-0
+
+| members | roles |
+|---|---|
+|prod-bootstrap-0
serviceAccount|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
+
+## Project prod-iac-core-0
+
+| members | roles |
+|---|---|
+|gcp-devops
group|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin)
[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) |
+|gcp-organization-admins
group|[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) |
+|prod-bootstrap-0
serviceAccount|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
+|prod-resman-0
serviceAccount|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md
index 06ee9986..3842aa07 100644
--- a/fast/stages/00-bootstrap/README.md
+++ b/fast/stages/00-bootstrap/README.md
@@ -1,6 +1,6 @@
# Organization bootstrap
-The primary purpose of this stage is to enable critical organization-level functionality that depends on broad administrative permissions, and prepare the prerequisites needed to enable automation in this and future stages.
+The primary purpose of this stage is to enable critical organization-level functionalities that depend on broad administrative permissions, and prepare the prerequisites needed to enable automation in this and future stages.
It is intentionally simple, to minimize usage of administrative-level permissions and enable simple auditing and troubleshooting, and only deals with three sets of resources:
@@ -28,7 +28,7 @@ We have standardized the initial set of groups on those outlined in the [GCP Ent
### Organization-level IAM
-The service account used in the [Resource Management stage](../01-resman) needs to be able to grant specific roles at the organizational level (`roles/billing.user`, `roles/compute.xpnAdmin`, etc.), to enable specific functionality for subsequent stages that deal with network or security resources, or billing-related activities.
+The service account used in the [Resource Management stage](../01-resman) needs to be able to grant specific permissions at the organizational level, to enable specific functionality for subsequent stages that deal with network or security resources, or billing-related activities.
In order to be able to assign those roles without having the full authority of the Organization Admin role, this stage defines a custom role that only allows setting IAM policies on the organization, and grants it via a [delegated role grant](https://cloud.google.com/iam/docs/setting-limits-on-granting-roles) that only allows it to be used to grant a limited subset of roles.
@@ -36,6 +36,8 @@ In this way, the Resource Management service account can effectively act as an O
One consequence of the above setup, is the need to configure IAM bindings as non-authoritative for the roles included in the IAM condition, since those same roles are effectively under the control of two stages: this one and Resource Management. Using authoritative bindings for these roles (instead of non-authoritative ones) would generate potential conflicts, where each stage could try to overwrite and negate the bindings applied by the other at each `apply` cycle.
+A full reference of IAM roles managed by this stage [is available here](./IAM.md).
+
### Automation project and resources
One other design choice worth mentioning here is using a single automation project for all foundational stages. We trade off some complexity on the API side (single source for usage quota, multiple service activation) for increased flexibility and simpler operations, while still effectively providing the same degree of separation via resource-level IAM.
@@ -95,7 +97,7 @@ To quickly self-grant the above roles, run the following code snippet as the ini
```bash
export BOOTSTRAP_ORG_ID=123456
export BOOTSTRAP_USER=$(gcloud config list --format 'value(core.account)')
-export BOOTSTRAP_ROLES=(roles/billing.admin roles/logging.admin roles/iam.organizationRoleAdmin roles/resourcemanager.projectCreator)
+export BOOTSTRAP_ROLES="roles/billing.admin roles/logging.admin roles/iam.organizationRoleAdmin roles/resourcemanager.projectCreator"
for role in $BOOTSTRAP_ROLES; do
gcloud organizations add-iam-policy-binding $BOOTSTRAP_ORG_ID \
--member user:$BOOTSTRAP_USER --role $role
@@ -144,10 +146,10 @@ Before the first run, the following IAM groups must exist to allow IAM bindings
#### Configure variables
-Then make sure you have configured the correct values for the following variables by editing providing a `terraform.tfvars` file:
+Then make sure you have configured the correct values for the following variables by providing a `terraform.tfvars` file:
- `billing_account`
- an object containing the id of your billing account, derived from the Cloud Console UI or by running `gcloud beta billing accounts list`, and the id of the organization owning it, or `null` to use the billing account in isolation
+ an object containing `id` as the id of your billing account, derived from the Cloud Console UI or by running `gcloud beta billing accounts list`, and `organization_id` as the id of the organization owning it, or `null` to use the billing account in isolation
- `groups`
the name mappings for your groups, if you're following the default convention you can leave this to the provided default
- `organization.id`, `organization.domain`, `organization.customer_id`
@@ -155,6 +157,25 @@ Then make sure you have configured the correct values for the following variable
- `prefix`
the fixed prefix used in your naming convention
+You can also adapt the example that follows to your needs:
+
+```hcl
+# fetch the required id by running `gcloud beta billing accounts list`
+billing_account={
+ id="012345-67890A-BCDEF0"
+ organization_id="01234567890"
+}
+# get the required info by running `gcloud organizations list`
+organization={
+ id="01234567890"
+ domain="fast.example.com"
+ customer_id="Cxxxxxxx"
+}
+# create your own 4-letters prefix
+prefix="fast"
+outputs_location = "../../fast-config"
+```
+
### Output files and cross-stage variables
At any time during the life of this stage, you can configure it to automatically generate provider configurations and variable files for the following, to simplify exchanging inputs and outputs between stages and avoid having to edit files manually.
@@ -178,8 +199,6 @@ Below is the outline of the output files generated by this stage:
│ ├── terraform-bootstrap.auto.tfvars.json
├── 02-networking
│ ├── terraform-bootstrap.auto.tfvars.json
-├── 02-networking-nva
-│ ├── terraform-bootstrap.auto.tfvars.json
├── 02-security
│ ├── terraform-bootstrap.auto.tfvars.json
├── 03-gke-multitenant-dev
@@ -214,7 +233,7 @@ terraform output -json providers | jq -r '.["00-bootstrap"]' \
> providers.tf
# migrate state to GCS bucket configured in providers file
terraform init -migrate-state
-# run terraform apply to remo user iam binding
+# run terraform apply to remove the bootstrap_user iam binding
terraform apply
```
diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf
index f55cfdc5..0c6951ee 100644
--- a/fast/stages/00-bootstrap/automation.tf
+++ b/fast/stages/00-bootstrap/automation.tf
@@ -34,13 +34,13 @@ module "automation-project" {
}
# machine (service accounts) IAM bindings
iam = {
- "roles/owner" = [module.automation-tf-bootstrap-sa.iam_email]
+ "roles/owner" = [
+ module.automation-tf-bootstrap-sa.iam_email
+ ]
"roles/iam.serviceAccountAdmin" = [
- module.automation-tf-bootstrap-sa.iam_email,
module.automation-tf-resman-sa.iam_email
]
"roles/storage.admin" = [
- module.automation-tf-bootstrap-sa.iam_email,
module.automation-tf-resman-sa.iam_email
]
}
diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/00-bootstrap/outputs.tf
index 8912fb87..d7924781 100644
--- a/fast/stages/00-bootstrap/outputs.tf
+++ b/fast/stages/00-bootstrap/outputs.tf
@@ -70,13 +70,13 @@ locals {
resource "local_file" "providers" {
for_each = var.outputs_location == null ? {} : local.providers
- filename = "${var.outputs_location}/${each.key}/providers.tf"
+ filename = "${pathexpand(var.outputs_location)}/${each.key}/providers.tf"
content = each.value
}
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : local.tfvars
- filename = "${var.outputs_location}/${each.key}/terraform-bootstrap.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-bootstrap.auto.tfvars.json"
content = each.value
}
diff --git a/fast/stages/01-resman/IAM.md b/fast/stages/01-resman/IAM.md
new file mode 100644
index 00000000..f915bb20
--- /dev/null
+++ b/fast/stages/01-resman/IAM.md
@@ -0,0 +1,44 @@
+# IAM bindings reference
+
+Legend: + additive, • conditional.
+
+## Organization [org_id #0]
+
+| members | roles |
+|---|---|
+|dev-resman-pf-0
serviceAccount|[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +|
+|prod-resman-networking-0
serviceAccount|[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +
[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) +
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) +|
+|prod-resman-pf-0
serviceAccount|[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +|
+|prod-resman-security-0
serviceAccount|[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) +
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +|
+
+## Folder networking
+
+| members | roles |
+|---|---|
+|gcp-network-admins
group|[roles/editor](https://cloud.google.com/iam/docs/understanding-roles#editor) |
+|prod-resman-networking-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) |
+
+## Folder sandbox
+
+| members | roles |
+|---|---|
+|dev-resman-sandbox-0
serviceAccount|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) |
+
+## Folder security
+
+| members | roles |
+|---|---|
+|gcp-security-admins
group|[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
+|prod-resman-security-0
serviceAccount|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) |
+
+## Folder dev
+
+| members | roles |
+|---|---|
+|dev-resman-pf-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) |
+
+## Folder prod
+
+| members | roles |
+|---|---|
+|prod-resman-pf-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) |
diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md
index 46a8a383..e39cf3d2 100644
--- a/fast/stages/01-resman/README.md
+++ b/fast/stages/01-resman/README.md
@@ -65,7 +65,7 @@ terraform output -json providers | jq -r '.["01-resman"]' \
> ../01-resman/providers.tf
```
-If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as deacribed [here](../00-bootstrap/#output-files-and-cross-stage-variables).
+If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../00-bootstrap/#output-files-and-cross-stage-variables).
### Variable configuration
@@ -136,6 +136,8 @@ For policies where additional data is needed, a root-level `organization_policy_
IAM roles can be easily edited in the relevant `branch-xxx.tf` file, following the best practice outlined in the [bootstrap stage](../00-bootstrap#customizations) documentation of separating user-level and service-account level IAM policies in modules' `iam_groups`, `iam`, and `iam_additive` variables.
+A full reference of IAM roles managed by this stage [is available here](./IAM.md).
+
### Additional folders
Due to its simplicity, this stage lends itself easily to customizations: adding a new top-level branch (e.g. for shared GKE clusters) is as easy as cloning one of the `branch-xxx.tf` files, and changing names.
@@ -175,12 +177,12 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
| name | description | sensitive | consumers |
|---|---|:---:|---|
-| [networking](outputs.tf#L84) | Data for the networking stage. | | 02-networking |
-| [project_factories](outputs.tf#L94) | Data for the project factories stage. | | xx-teams |
-| [providers](outputs.tf#L111) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · xx-sandbox · xx-teams |
-| [sandbox](outputs.tf#L118) | Data for the sandbox stage. | | xx-sandbox |
-| [security](outputs.tf#L128) | Data for the networking stage. | | 02-security |
-| [teams](outputs.tf#L138) | Data for the teams stage. | | |
-| [tfvars](outputs.tf#L151) | Terraform variable files for the following stages. | ✓ | |
+| [networking](outputs.tf#L83) | Data for the networking stage. | | 02-networking |
+| [project_factories](outputs.tf#L93) | Data for the project factories stage. | | xx-teams |
+| [providers](outputs.tf#L110) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · xx-sandbox · xx-teams |
+| [sandbox](outputs.tf#L117) | Data for the sandbox stage. | | xx-sandbox |
+| [security](outputs.tf#L127) | Data for the networking stage. | | 02-security |
+| [teams](outputs.tf#L137) | Data for the teams stage. | | |
+| [tfvars](outputs.tf#L150) | Terraform variable files for the following stages. | ✓ | |
diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/01-resman/branch-networking.tf
index 3b291197..49203051 100644
--- a/fast/stages/01-resman/branch-networking.tf
+++ b/fast/stages/01-resman/branch-networking.tf
@@ -36,6 +36,7 @@ module "branch-network-folder" {
"roles/owner" = [module.branch-network-sa.iam_email]
"roles/resourcemanager.folderAdmin" = [module.branch-network-sa.iam_email]
"roles/resourcemanager.projectCreator" = [module.branch-network-sa.iam_email]
+ "roles/compute.xpnAdmin" = [module.branch-network-sa.iam_email]
}
}
@@ -57,3 +58,25 @@ module "branch-network-gcs" {
"roles/storage.objectAdmin" = [module.branch-network-sa.iam_email]
}
}
+
+module "branch-network-prod-folder" {
+ source = "../../../modules/folder"
+ parent = module.branch-network-folder.id
+ name = "prod"
+ iam = {
+ "roles/compute.xpnAdmin" = [
+ module.branch-teams-prod-projectfactory-sa.iam_email
+ ]
+ }
+}
+
+module "branch-network-dev-folder" {
+ source = "../../../modules/folder"
+ parent = module.branch-network-folder.id
+ name = "dev"
+ iam = {
+ "roles/compute.xpnAdmin" = [
+ module.branch-teams-dev-projectfactory-sa.iam_email
+ ]
+ }
+}
diff --git a/fast/stages/01-resman/branch-teams.tf b/fast/stages/01-resman/branch-teams.tf
index 7967fc9b..408a34ce 100644
--- a/fast/stages/01-resman/branch-teams.tf
+++ b/fast/stages/01-resman/branch-teams.tf
@@ -94,6 +94,9 @@ module "branch-teams-team-dev-folder" {
"roles/resourcemanager.projectCreator" = [
module.branch-teams-dev-projectfactory-sa.iam_email
]
+ "roles/compute.xpnAdmin" = [
+ module.branch-teams-dev-projectfactory-sa.iam_email
+ ]
}
}
@@ -141,6 +144,9 @@ module "branch-teams-team-prod-folder" {
"roles/resourcemanager.projectCreator" = [
module.branch-teams-prod-projectfactory-sa.iam_email
]
+ "roles/compute.xpnAdmin" = [
+ module.branch-teams-prod-projectfactory-sa.iam_email
+ ]
}
}
diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf
index 62aed289..23056119 100644
--- a/fast/stages/01-resman/outputs.tf
+++ b/fast/stages/01-resman/outputs.tf
@@ -25,11 +25,6 @@ locals {
name = "networking"
sa = module.branch-network-sa.email
})
- "02-networking-nva" = templatefile("${path.module}/../../assets/templates/providers.tpl", {
- bucket = module.branch-network-gcs.name
- name = "networking-nva"
- sa = module.branch-network-sa.email
- })
"02-security" = templatefile("${path.module}/../../assets/templates/providers.tpl", {
bucket = module.branch-security-gcs.name
name = "security"
@@ -53,7 +48,11 @@ locals {
}
tfvars = {
"02-networking" = jsonencode({
- folder_id = module.branch-network-folder.id
+ folder_ids = {
+ networking = module.branch-network-folder.id
+ networking-dev = module.branch-network-dev-folder.id
+ networking-prod = module.branch-network-prod-folder.id
+ }
project_factory_sa = local._project_factory_sas
})
"02-security" = jsonencode({
@@ -69,13 +68,13 @@ locals {
resource "local_file" "providers" {
for_each = var.outputs_location == null ? {} : local.providers
- filename = "${var.outputs_location}/${each.key}/providers.tf"
+ filename = "${pathexpand(var.outputs_location)}/${each.key}/providers.tf"
content = each.value
}
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : local.tfvars
- filename = "${var.outputs_location}/${each.key}/terraform-resman.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-resman.auto.tfvars.json"
content = each.value
}
diff --git a/fast/stages/02-networking-nva/README.md b/fast/stages/02-networking-nva/README.md
index 2f692df5..1ffc4d0c 100644
--- a/fast/stages/02-networking-nva/README.md
+++ b/fast/stages/02-networking-nva/README.md
@@ -187,12 +187,13 @@ If you have set a valid value for `outputs_location` in the bootstrap and in the
ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json
ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json
```
+If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../00-bootstrap/#output-files-and-cross-stage-variables).
Please, refer to the [variables](#variables) table below for a map of the variable origins, and use the sections below to understand how to adapt this stage to your networking configuration.
### VPCs
-VPCs are defined in separate files, one for `untrusted landing`, one for `trusted landing`, one for `prod` and one for `dev`.
+VPCs are defined in separate files, one for `landing` (trusted and untrusted), one for `prod` and one for `dev`.
These files contain different resources:
@@ -321,19 +322,19 @@ Don't forget to add a peering zone in the landing project and point it to the ne
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | 00-bootstrap |
-| [organization](variables.tf#L99) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L115) | Prefix used for resources that need unique names. | string | ✓ | | 00-bootstrap |
+| [folder_ids](variables.tf#L59) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | map(string) | ✓ | | 01-resman |
+| [organization](variables.tf#L91) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
+| [prefix](variables.tf#L107) | Prefix used for resources that need unique names. | string | ✓ | | 00-bootstrap |
| [custom_adv](variables.tf#L23) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
| [data_dir](variables.tf#L45) | Relative path for the folder storing configuration data for network resources. | string | | "data" | |
| [dns](variables.tf#L51) | Onprem DNS resolvers | map(list(string)) | | {…} | |
-| [folder_id](variables.tf#L59) | Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | string | | null | 01-resman |
-| [l7ilb_subnets](variables.tf#L73) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
-| [onprem_cidr](variables.tf#L91) | Onprem addresses in name => range format. | map(string) | | {…} | |
-| [outputs_location](variables.tf#L109) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
-| [project_factory_sa](variables.tf#L121) | IAM emails for project factory service accounts | map(string) | | {} | 01-resman |
-| [psa_ranges](variables.tf#L128) | IP ranges used for Private Service Access (e.g. CloudSQL). | map(map(string)) | | {…} | |
-| [router_configs](variables.tf#L143) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | |
-| [vpn_onprem_configs](variables.tf#L166) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
+| [l7ilb_subnets](variables.tf#L65) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
+| [onprem_cidr](variables.tf#L83) | Onprem addresses in name => range format. | map(string) | | {…} | |
+| [outputs_location](variables.tf#L101) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
+| [project_factory_sa](variables.tf#L113) | IAM emails for project factory service accounts | map(string) | | {} | 01-resman |
+| [psa_ranges](variables.tf#L120) | IP ranges used for Private Service Access (e.g. CloudSQL). | map(map(string)) | | {…} | |
+| [router_configs](variables.tf#L135) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | |
+| [vpn_onprem_configs](variables.tf#L158) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
## Outputs
diff --git a/fast/stages/02-networking-nva/main.tf b/fast/stages/02-networking-nva/main.tf
index db03c69a..932191dc 100644
--- a/fast/stages/02-networking-nva/main.tf
+++ b/fast/stages/02-networking-nva/main.tf
@@ -29,8 +29,8 @@ module "folder" {
source = "../../../modules/folder"
parent = "organizations/${var.organization.id}"
name = "Networking"
- folder_create = var.folder_id == null
- id = var.folder_id
+ folder_create = var.folder_ids.networking == null
+ id = var.folder_ids.networking
firewall_policy_factory = {
cidr_file = "${var.data_dir}/cidrs.yaml"
policy_name = null
diff --git a/fast/stages/02-networking-nva/outputs.tf b/fast/stages/02-networking-nva/outputs.tf
index 39c5d2ef..cf39b1df 100644
--- a/fast/stages/02-networking-nva/outputs.tf
+++ b/fast/stages/02-networking-nva/outputs.tf
@@ -33,7 +33,7 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : local.tfvars
- filename = "${var.outputs_location}/${each.key}/terraform-networking.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-networking.auto.tfvars.json"
content = each.value
}
diff --git a/fast/stages/02-networking-nva/variables.tf b/fast/stages/02-networking-nva/variables.tf
index 6756e5b6..355eccf4 100644
--- a/fast/stages/02-networking-nva/variables.tf
+++ b/fast/stages/02-networking-nva/variables.tf
@@ -56,18 +56,10 @@ variable "dns" {
}
}
-variable "folder_id" {
+variable "folder_ids" {
# tfdoc:variable:source 01-resman
- description = "Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
- type = string
- default = null
- validation {
- condition = (
- var.folder_id == null ||
- can(regex("folders/[0-9]{8,}", var.folder_id))
- )
- error_message = "Invalid folder_id. Should be in 'folders/nnnnnnnnnnn' format."
- }
+ description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
+ type = map(string)
}
variable "l7ilb_subnets" {
diff --git a/fast/stages/02-networking-nva/vpc-landing.tf b/fast/stages/02-networking-nva/vpc-landing.tf
index 9f6d0a92..ed983da0 100644
--- a/fast/stages/02-networking-nva/vpc-landing.tf
+++ b/fast/stages/02-networking-nva/vpc-landing.tf
@@ -20,7 +20,7 @@ module "landing-project" {
source = "../../../modules/project"
billing_account = var.billing_account_id
name = "prod-net-landing-0"
- parent = var.folder_id
+ parent = var.folder_ids.networking-prod
prefix = var.prefix
service_config = {
disable_on_destroy = false
diff --git a/fast/stages/02-networking-nva/vpc-spoke-dev.tf b/fast/stages/02-networking-nva/vpc-spoke-dev.tf
index 392b5750..628b6490 100644
--- a/fast/stages/02-networking-nva/vpc-spoke-dev.tf
+++ b/fast/stages/02-networking-nva/vpc-spoke-dev.tf
@@ -20,7 +20,7 @@ module "dev-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account_id
name = "dev-net-spoke-0"
- parent = var.folder_id
+ parent = var.folder_ids.networking-dev
prefix = var.prefix
service_config = {
disable_on_destroy = false
diff --git a/fast/stages/02-networking-nva/vpc-spoke-prod.tf b/fast/stages/02-networking-nva/vpc-spoke-prod.tf
index 320175dc..f0555263 100644
--- a/fast/stages/02-networking-nva/vpc-spoke-prod.tf
+++ b/fast/stages/02-networking-nva/vpc-spoke-prod.tf
@@ -20,7 +20,7 @@ module "prod-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account_id
name = "prod-net-spoke-0"
- parent = var.folder_id
+ parent = var.folder_ids.networking-prod
prefix = var.prefix
service_config = {
disable_on_destroy = false
diff --git a/fast/stages/02-networking-vpn/IAM.md b/fast/stages/02-networking-vpn/IAM.md
new file mode 100644
index 00000000..f5c69067
--- /dev/null
+++ b/fast/stages/02-networking-vpn/IAM.md
@@ -0,0 +1,16 @@
+# IAM bindings reference
+
+Legend: + additive, • conditional.
+
+## Project dev-net-spoke-0
+
+| members | roles |
+|---|---|
+|dev-resman-pf-0
serviceAccount|[roles/resourcemanager.projectIamAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) •
[roles/dns.admin](https://cloud.google.com/iam/docs/understanding-roles#dns.admin) |
+|prod-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin |
+
+## Project prod-net-spoke-0
+
+| members | roles |
+|---|---|
+|prod-resman-pf-0
serviceAccount|[roles/resourcemanager.projectIamAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) •
organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/dns.admin](https://cloud.google.com/iam/docs/understanding-roles#dns.admin) |
diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md
index a485d1c7..e6cb4fe4 100644
--- a/fast/stages/02-networking-vpn/README.md
+++ b/fast/stages/02-networking-vpn/README.md
@@ -309,20 +309,20 @@ DNS configurations are centralised in the `dns.tf` file. Spokes delegate DNS res
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | 00-bootstrap |
-| [organization](variables.tf#L93) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L109) | Prefix used for resources that need unique names. | string | ✓ | | 00-bootstrap |
+| [folder_ids](variables.tf#L61) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | map(string) | ✓ | | 01-resman |
+| [organization](variables.tf#L85) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
+| [prefix](variables.tf#L101) | Prefix used for resources that need unique names. | string | ✓ | | 00-bootstrap |
| [custom_adv](variables.tf#L23) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
| [custom_roles](variables.tf#L40) | Custom roles defined at the org level, in key => id format. | map(string) | | {} | 00-bootstrap |
| [data_dir](variables.tf#L47) | Relative path for the folder storing configuration data for network resources. | string | | "data" | |
| [dns](variables.tf#L53) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
-| [folder_id](variables.tf#L61) | Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | string | | null | 01-resman |
-| [l7ilb_subnets](variables.tf#L75) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
-| [outputs_location](variables.tf#L103) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
-| [project_factory_sa](variables.tf#L115) | IAM emails for project factory service accounts. | map(string) | | {} | 01-resman |
-| [psa_ranges](variables.tf#L122) | IP ranges used for Private Service Access (e.g. CloudSQL). | map(map(string)) | | {…} | |
-| [router_configs](variables.tf#L137) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | |
-| [vpn_onprem_configs](variables.tf#L161) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
-| [vpn_spoke_configs](variables.tf#L217) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | |
+| [l7ilb_subnets](variables.tf#L67) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
+| [outputs_location](variables.tf#L95) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
+| [project_factory_sa](variables.tf#L107) | IAM emails for project factory service accounts. | map(string) | | {} | 01-resman |
+| [psa_ranges](variables.tf#L114) | IP ranges used for Private Service Access (e.g. CloudSQL). | map(map(string)) | | {…} | |
+| [router_configs](variables.tf#L129) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | |
+| [vpn_onprem_configs](variables.tf#L153) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
+| [vpn_spoke_configs](variables.tf#L209) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | |
## Outputs
diff --git a/fast/stages/02-networking-vpn/main.tf b/fast/stages/02-networking-vpn/main.tf
index 4a3f4748..fcca8867 100644
--- a/fast/stages/02-networking-vpn/main.tf
+++ b/fast/stages/02-networking-vpn/main.tf
@@ -53,8 +53,8 @@ module "folder" {
source = "../../../modules/folder"
parent = "organizations/${var.organization.id}"
name = "Networking"
- folder_create = var.folder_id == null
- id = var.folder_id
+ folder_create = var.folder_ids.networking == null
+ id = var.folder_ids.networking
firewall_policy_factory = {
cidr_file = "${var.data_dir}/cidrs.yaml"
policy_name = null
diff --git a/fast/stages/02-networking-vpn/outputs.tf b/fast/stages/02-networking-vpn/outputs.tf
index 4efe9bc6..15b4c49c 100644
--- a/fast/stages/02-networking-vpn/outputs.tf
+++ b/fast/stages/02-networking-vpn/outputs.tf
@@ -32,7 +32,7 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : local.tfvars
- filename = "${var.outputs_location}/${each.key}/terraform-networking.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-networking.auto.tfvars.json"
content = each.value
}
diff --git a/fast/stages/02-networking-vpn/variables.tf b/fast/stages/02-networking-vpn/variables.tf
index 4c134e2f..3eb141e7 100644
--- a/fast/stages/02-networking-vpn/variables.tf
+++ b/fast/stages/02-networking-vpn/variables.tf
@@ -58,18 +58,10 @@ variable "dns" {
}
}
-variable "folder_id" {
+variable "folder_ids" {
# tfdoc:variable:source 01-resman
- description = "Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
- type = string
- default = null
- validation {
- condition = (
- var.folder_id == null ||
- can(regex("folders/[0-9]{8,}", var.folder_id))
- )
- error_message = "Invalid folder_id. Should be in 'folders/nnnnnnnnnnn' format."
- }
+ description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
+ type = map(string)
}
variable "l7ilb_subnets" {
diff --git a/fast/stages/02-networking-vpn/vpc-landing.tf b/fast/stages/02-networking-vpn/vpc-landing.tf
index 5b6673f7..e2b6c45a 100644
--- a/fast/stages/02-networking-vpn/vpc-landing.tf
+++ b/fast/stages/02-networking-vpn/vpc-landing.tf
@@ -20,7 +20,7 @@ module "landing-project" {
source = "../../../modules/project"
billing_account = var.billing_account_id
name = "prod-net-landing-0"
- parent = var.folder_id
+ parent = var.folder_ids.networking-prod
prefix = var.prefix
service_config = {
disable_on_destroy = false
diff --git a/fast/stages/02-networking-vpn/vpc-spoke-dev.tf b/fast/stages/02-networking-vpn/vpc-spoke-dev.tf
index 90d11f16..9b3c0f9e 100644
--- a/fast/stages/02-networking-vpn/vpc-spoke-dev.tf
+++ b/fast/stages/02-networking-vpn/vpc-spoke-dev.tf
@@ -20,7 +20,7 @@ module "dev-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account_id
name = "dev-net-spoke-0"
- parent = var.folder_id
+ parent = var.folder_ids.networking-dev
prefix = var.prefix
service_config = {
disable_on_destroy = false
diff --git a/fast/stages/02-networking-vpn/vpc-spoke-prod.tf b/fast/stages/02-networking-vpn/vpc-spoke-prod.tf
index 0132d8fd..7f42ab2c 100644
--- a/fast/stages/02-networking-vpn/vpc-spoke-prod.tf
+++ b/fast/stages/02-networking-vpn/vpc-spoke-prod.tf
@@ -20,7 +20,7 @@ module "prod-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account_id
name = "prod-net-spoke-0"
- parent = var.folder_id
+ parent = var.folder_ids.networking-prod
prefix = var.prefix
service_config = {
disable_on_destroy = false
diff --git a/fast/stages/02-security/IAM.md b/fast/stages/02-security/IAM.md
new file mode 100644
index 00000000..51bdc462
--- /dev/null
+++ b/fast/stages/02-security/IAM.md
@@ -0,0 +1,15 @@
+# IAM bindings reference
+
+Legend: + additive, • conditional.
+
+## Project dev-sec-core-0
+
+| members | roles |
+|---|---|
+|dev-resman-pf-0
serviceAccount|[roles/cloudkms.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.admin) +•
[roles/cloudkms.viewer](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.viewer) |
+
+## Project prod-sec-core-0
+
+| members | roles |
+|---|---|
+|prod-resman-pf-0
serviceAccount|[roles/cloudkms.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.admin) +•
[roles/cloudkms.viewer](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.viewer) |
diff --git a/fast/stages/02-security/outputs.tf b/fast/stages/02-security/outputs.tf
index 8f296d86..15c75a2a 100644
--- a/fast/stages/02-security/outputs.tf
+++ b/fast/stages/02-security/outputs.tf
@@ -18,7 +18,7 @@
resource "local_file" "dev_sec_kms" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
- filename = "${var.outputs_location}/yamls/02-security-kms-dev-keys.yaml"
+ filename = "${pathexpand(var.outputs_location)}/yamls/02-security-kms-dev-keys.yaml"
content = yamlencode({
for k, m in module.dev-sec-kms : k => m.key_ids
})
@@ -26,7 +26,7 @@ resource "local_file" "dev_sec_kms" {
resource "local_file" "prod_sec_kms" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
- filename = "${var.outputs_location}/yamls/02-security-kms-prod-keys.yaml"
+ filename = "${pathexpand(var.outputs_location)}/yamls/02-security-kms-prod-keys.yaml"
content = yamlencode({
for k, m in module.prod-sec-kms : k => m.key_ids
})
diff --git a/fast/stages/03-project-factory/prod/README.md b/fast/stages/03-project-factory/prod/README.md
index 328571a2..2b31523b 100644
--- a/fast/stages/03-project-factory/prod/README.md
+++ b/fast/stages/03-project-factory/prod/README.md
@@ -108,11 +108,11 @@ terraform apply
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L19) | Billing account id. | string | ✓ | | 00-bootstrap |
-| [shared_vpc_self_link](variables.tf#L44) | Self link for the shared VPC. | string | ✓ | | 02-networking |
-| [vpc_host_project](variables.tf#L50) | Host project for the shared VPC. | string | ✓ | | 02-networking |
| [data_dir](variables.tf#L25) | Relative path for the folder storing configuration data. | string | | "data/projects" | |
| [defaults_file](variables.tf#L38) | Relative path for the file storing the project factory configuration. | string | | "data/defaults.yaml" | |
| [environment_dns_zone](variables.tf#L31) | DNS zone suffix for environment. | string | | null | 02-networking |
+| [shared_vpc_self_link](variables.tf#L44) | Self link for the shared VPC. | string | | null | 02-networking |
+| [vpc_host_project](variables.tf#L51) | Host project for the shared VPC. | string | | null | 02-networking |
## Outputs
diff --git a/fast/stages/03-project-factory/prod/variables.tf b/fast/stages/03-project-factory/prod/variables.tf
index 8bb9f035..721a0ca9 100644
--- a/fast/stages/03-project-factory/prod/variables.tf
+++ b/fast/stages/03-project-factory/prod/variables.tf
@@ -45,10 +45,12 @@ variable "shared_vpc_self_link" {
# tfdoc:variable:source 02-networking
description = "Self link for the shared VPC."
type = string
+ default = null
}
variable "vpc_host_project" {
# tfdoc:variable:source 02-networking
description = "Host project for the shared VPC."
type = string
+ default = null
}
diff --git a/modules/iot-core/README.md b/modules/iot-core/README.md
new file mode 100644
index 00000000..d28006db
--- /dev/null
+++ b/modules/iot-core/README.md
@@ -0,0 +1,145 @@
+# Google Cloud IoT Core Module
+
+This module sets up Cloud IoT Core Registry, registers IoT Devices and configures Pub/Sub topics required in Cloud IoT Core.
+
+To use this module, ensure the following APIs are enabled:
+* pubsub.googleapis.com
+* cloudiot.googleapis.com
+
+## Simple Example
+
+Basic example showing how to create an IoT Platform (IoT Core), connected to a set of given Pub/Sub topics and provision IoT devices.
+
+Devices certificates must exist before calling this module. You can generate these certificates using the following command
+
+```
+openssl req -x509 -newkey rsa:2048 -keyout rsa_private.pem -nodes -out rsa_cert.pem -subj "/CN=unused"
+```
+
+And then provision public certificate path, together with the rest of device configuration in a devices yaml file following the following format
+```yaml
+device_id: # id of your IoT Device
+ is_blocked: # false to allow device connection with IoT Registry
+ is_gateway: # true to indicate the device connecting acts as a gateway for other IoT Devices
+ log_level: # device logs level
+ certificate_file: # public certificate path, generated as explained in the previous step
+ certificate_format: # Certificates format values are RSA_PEM, RSA_X509_PEM, ES256_PEM, and ES256_X509_PEM
+```
+
+Example Device config yaml configuration
+```yaml
+device_1:
+ is_blocked: false
+ is_gateway: false
+ log_level: INFO
+ certificate_file: device_certs/rsa_cert5.pem
+ certificate_format: RSA_X509_PEM
+device_2:
+ is_blocked: true
+ is_gateway: false
+ log_level: INFO
+ certificate_file: device_certs/rsa_cert5.pem
+ certificate_format: RSA_X509_PEM
+```
+
+```hcl
+module "iot-platform" {
+ source = "./modules/iot-core"
+ project_id = "my_project_id"
+ region = "europe-west1"
+ telemetry_pubsub_topic_id = "telemetry_topic_id"
+ status_pubsub_topic_id = "status_topic_id"
+ protocols = {
+ http = false,
+ mqtt = true
+ }
+ devices_config_directory = "./devices_config_folder"
+}
+# tftest:skip
+
+```
+
+Now, we can test sending telemetry messages from devices to our IoT Platform, for example using the MQTT demo client at https://github.com/googleapis/nodejs-iot/tree/main/samples/mqtt_example
+
+## Example with specific PubSub topics for custom MQTT topics
+
+If you need to match specific MQTT topics (eg, /temperature) into specific PubSub topics, you can use extra_telemetry_pubsub_topic_ids for that, as in the following example:
+
+```hcl
+module "iot-platform" {
+ source = "./modules/iot-core"
+ project_id = "my_project_id"
+ region = "europe-west1"
+ telemetry_pubsub_topic_id = "telemetry_topic_id"
+ status_pubsub_topic_id = "status_topic_id"
+ extra_telemetry_pubsub_topic_ids = {
+ "temperature" = "temp_topic_id",
+ "humidity" = "hum_topic_id"
+ }
+ protocols = {
+ http = false,
+ mqtt = true
+ }
+ devices_config_directory = "./devices_config_folder"
+}
+# tftest:skip
+
+```
+
+## Example integrated with Data Foundation Platform
+In this example, we will show how to extend the **[Data Foundations Platform](../../data-solutions/data-platform-foundations/)** to include IoT Platform as a new source of data.
+
+
+
+1. First, we will setup Environment following instructions in **[Environment Setup](../../data-solutions/data-platform-foundations/01-environment/)** to setup projects and SAs required. Get output variable project_ids.landing as will be used later
+
+1. Second, execute instructions in **[Environment Setup](../../data-solutions/data-platform-foundations/02-resources/)** to provision PubSub, DataFlow, BQ,... Get variable landing-pubsub as will be used later to create IoT Registry
+
+1. Now it is time to provision IoT Platform. Modify landing-project-id and landing_pubsub_topic_id with output variables obtained before. Create device certificates as shown in the Simple Example and register them in devices.yaml file together with deviceids.
+
+```hcl
+module "iot-platform" {
+ source = "./modules/iot-core"
+ project_id = "landing-project-id"
+ region = "europe-west1"
+ telemetry_pubsub_topic_id = "landing_pubsub_topic_id"
+ status_pubsub_topic_id = "status_pubsub_topic_id"
+ protocols = {
+ http = false,
+ mqtt = true
+ }
+ devices_config_directory = "./devices_config_folder"
+}
+# tftest:skip
+```
+1. After that, we can setup the pipeline "PubSub to BigQuery" shown at **[Pipeline Setup](../../data-solutions/data-platform-foundations/03-pipeline/pubsub_to_bigquery.md)**
+
+1. Finally, instead of testing the pipeline by sending messages to PubSub, we can now test sending telemetry messages from simulated IoT devices to our IoT Platform, for example using the MQTT demo client at https://github.com/googleapis/nodejs-iot/tree/main/samples/mqtt_example . We shall edit the client script cloudiot_mqtt_example_nodejs.js to send messages following the pipeline message format, so they are processed by DataFlow job and inserted in the BigQuery table.
+```
+const payload = '{"name": "device4", "surname": "NA", "timestamp":"'+Math.floor(Date.now()/1000)+'"}';
+```
+
+Or even better, create a new BigQuery table with our IoT sensors data columns and modify the DataFlow job to push data to it.
+
+
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---:|:---:|:---:|
+| [devices_config_directory](variables.tf#L17) | Path to folder where devices configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`. | string | ✓ | |
+| [project_id](variables.tf#L34) | Project were resources will be deployed | string | ✓ | |
+| [region](variables.tf#L48) | Region were resources will be deployed | string | ✓ | |
+| [status_pubsub_topic_id](variables.tf#L59) | pub sub topic for status messages (GCP-->Device) | string | ✓ | |
+| [telemetry_pubsub_topic_id](variables.tf#L64) | pub sub topic for telemetry messages (Device-->GCP) | string | ✓ | |
+| [extra_telemetry_pubsub_topic_ids](variables.tf#L22) | additional pubsub topics linked to adhoc MQTT topics (Device-->GCP) in the format MQTT_TOPIC: PUBSUB_TOPIC_ID | map(string) | | {} |
+| [log_level](variables.tf#L28) | IoT Registry Log level | string | | "INFO" |
+| [protocols](variables.tf#L39) | IoT protocols (HTTP / MQTT) activation | object({…}) | | { http = true, mqtt = true } |
+| [registry_name](variables.tf#L53) | Name for the IoT Core Registry | string | | "cloudiot-registry" |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [iot_registry](outputs.tf#L17) | Cloud IoT Core Registry | |
+
+
diff --git a/modules/iot-core/diagram.png b/modules/iot-core/diagram.png
new file mode 100644
index 00000000..7a339321
Binary files /dev/null and b/modules/iot-core/diagram.png differ
diff --git a/modules/iot-core/diagram_iot.png b/modules/iot-core/diagram_iot.png
new file mode 100644
index 00000000..4c82259f
Binary files /dev/null and b/modules/iot-core/diagram_iot.png differ
diff --git a/modules/iot-core/main.tf b/modules/iot-core/main.tf
new file mode 100644
index 00000000..6a8c3db1
--- /dev/null
+++ b/modules/iot-core/main.tf
@@ -0,0 +1,95 @@
+
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+ devices_config_files = [
+ for config_file in fileset("${path.root}/${var.devices_config_directory}", "**/*.yaml") :
+ "${path.root}/${var.devices_config_directory}/${config_file}"
+ ]
+
+ device_config = merge(
+ [
+ for config_file in local.devices_config_files :
+ try(yamldecode(file(config_file)), {})
+ ]...
+ )
+}
+
+#---------------------------------------------------------
+# Create IoT Core Registry
+#---------------------------------------------------------
+
+resource "google_cloudiot_registry" "registry" {
+
+ name = var.registry_name
+ project = var.project_id
+ region = var.region
+
+ dynamic "event_notification_configs" {
+ for_each = var.extra_telemetry_pubsub_topic_ids
+ content {
+ pubsub_topic_name = event_notification_configs.value
+ subfolder_matches = event_notification_configs.key
+ }
+ }
+
+ event_notification_configs {
+ pubsub_topic_name = var.telemetry_pubsub_topic_id
+ subfolder_matches = ""
+ }
+
+ state_notification_config = {
+ pubsub_topic_name = var.status_pubsub_topic_id
+ }
+
+ mqtt_config = {
+ mqtt_enabled_state = var.protocols.mqtt ? "MQTT_ENABLED" : "MQTT_DISABLED"
+ }
+
+ http_config = {
+ http_enabled_state = var.protocols.http ? "HTTP_ENABLED" : "HTTP_DISABLED"
+ }
+
+ log_level = var.log_level
+
+}
+
+#---------------------------------------------------------
+# Create IoT Core Device
+# certificate created using: openssl req -x509 -newkey rsa:2048 -keyout rsa_private.pem -nodes -out rsa_cert.pem -subj "/CN=unused"
+#---------------------------------------------------------
+
+resource "google_cloudiot_device" "device" {
+ for_each = local.device_config
+ name = each.key
+ registry = google_cloudiot_registry.registry.id
+
+ credentials {
+ public_key {
+ format = try(each.value.certificate_format, null)
+ key = try(file(each.value.certificate_file), null)
+ }
+ }
+
+ blocked = try(each.value.is_blocked, null)
+
+ log_level = try(each.value.log_level, null)
+
+ gateway_config {
+ gateway_type = try(each.value.is_gateway, null) ? "GATEWAY" : "NON_GATEWAY"
+ }
+}
\ No newline at end of file
diff --git a/modules/iot-core/outputs.tf b/modules/iot-core/outputs.tf
new file mode 100644
index 00000000..fcce598a
--- /dev/null
+++ b/modules/iot-core/outputs.tf
@@ -0,0 +1,20 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "iot_registry" {
+ description = "Cloud IoT Core Registry"
+ value = google_cloudiot_registry.registry
+}
\ No newline at end of file
diff --git a/modules/iot-core/variables.tf b/modules/iot-core/variables.tf
new file mode 100644
index 00000000..f1924982
--- /dev/null
+++ b/modules/iot-core/variables.tf
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "devices_config_directory" {
+ description = "Path to folder where devices configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`."
+ type = string
+}
+
+variable "extra_telemetry_pubsub_topic_ids" {
+ description = "additional pubsub topics linked to adhoc MQTT topics (Device-->GCP) in the format MQTT_TOPIC: PUBSUB_TOPIC_ID"
+ type = map(string)
+ default = {}
+}
+
+variable "log_level" {
+ description = "IoT Registry Log level"
+ type = string
+ default = "INFO"
+}
+
+variable "project_id" {
+ description = "Project were resources will be deployed"
+ type = string
+}
+
+variable "protocols" {
+ description = "IoT protocols (HTTP / MQTT) activation"
+ type = object({
+ http = bool,
+ mqtt = bool
+ })
+ default = { http = true, mqtt = true }
+}
+
+variable "region" {
+ description = "Region were resources will be deployed"
+ type = string
+}
+
+variable "registry_name" {
+ description = "Name for the IoT Core Registry"
+ type = string
+ default = "cloudiot-registry"
+}
+
+variable "status_pubsub_topic_id" {
+ description = "pub sub topic for status messages (GCP-->Device)"
+ type = string
+}
+
+variable "telemetry_pubsub_topic_id" {
+ description = "pub sub topic for telemetry messages (Device-->GCP)"
+ type = string
+}
\ No newline at end of file
diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf
index 730e17f2..676c52f9 100644
--- a/modules/net-vpc/main.tf
+++ b/modules/net-vpc/main.tf
@@ -153,12 +153,14 @@ resource "google_compute_network_peering" "remote" {
}
resource "google_compute_shared_vpc_host_project" "shared_vpc_host" {
+ provider = google-beta
count = var.shared_vpc_host ? 1 : 0
project = var.project_id
depends_on = [local.network]
}
resource "google_compute_shared_vpc_service_project" "service_projects" {
+ provider = google-beta
for_each = (
var.shared_vpc_host && var.shared_vpc_service_projects != null
? toset(var.shared_vpc_service_projects)
diff --git a/modules/project/shared-vpc.tf b/modules/project/shared-vpc.tf
index 6e720b45..ecce0df5 100644
--- a/modules/project/shared-vpc.tf
+++ b/modules/project/shared-vpc.tf
@@ -17,11 +17,13 @@
# tfdoc:file:description Shared VPC project-level configuration.
resource "google_compute_shared_vpc_host_project" "shared_vpc_host" {
- count = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0
- project = local.project.project_id
+ provider = google-beta
+ count = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0
+ project = local.project.project_id
}
resource "google_compute_shared_vpc_service_project" "service_projects" {
+ provider = google-beta
for_each = (
try(var.shared_vpc_host_config.enabled, false)
? toset(coalesce(var.shared_vpc_host_config.service_projects, []))
@@ -33,6 +35,7 @@ resource "google_compute_shared_vpc_service_project" "service_projects" {
}
resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
+ provider = google-beta
count = try(var.shared_vpc_service_config.attach, false) ? 1 : 0
host_project = var.shared_vpc_service_config.host_project
service_project = local.project.project_id
diff --git a/tools/state_iam.py b/tools/state_iam.py
new file mode 100755
index 00000000..83360179
--- /dev/null
+++ b/tools/state_iam.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'Parse and output IAM bindings from Terraform state file.'
+
+import collections
+import json
+import itertools
+import re
+import sys
+
+import click
+
+
+FIELDS = (
+ 'authoritative', 'resource_type', 'resource_id', 'role', 'member_type',
+ 'member_id', 'conditions'
+)
+ORG_IDS = {}
+RESOURCE_SORT = {'organization': 0, 'folder': 1, 'project': 2}
+RESOURCE_TYPE_RE = re.compile(r'^google_([^_]+)_iam_([^_]+)$')
+Binding = collections.namedtuple('Binding', ' '.join(FIELDS))
+
+
+def _org_id(resource_id):
+ if resource_id not in ORG_IDS:
+ ORG_IDS[resource_id] = f'[org_id #{len(ORG_IDS)}]'
+ return ORG_IDS[resource_id]
+
+
+def get_bindings(resources, prefix=None, folders=None):
+ 'Parse resources and return bindings.'
+ org_ids = {}
+ for r in resources:
+ m = RESOURCE_TYPE_RE.match(r['type'])
+ if not m:
+ continue
+ resource_type = m.group(1)
+ authoritative = m.group(2) == 'binding'
+ for i in r.get('instances'):
+ attrs = i['attributes']
+ conditions = ' '.join(c['title'] for c in attrs.get('condition', []))
+ if resource_type == 'organization':
+ resource_id = _org_id(attrs['org_id'])
+ else:
+ resource_id = attrs[resource_type]
+ if prefix and resource_id.startswith(prefix):
+ resource_id = resource_id[len(prefix) + 1:]
+ role = attrs['role']
+ if role.startswith('organizations/'):
+ org_id = role.split('/')[1]
+ role = role.replace(org_id, _org_id(org_id))
+ members = attrs['members'] if authoritative else [attrs['member']]
+ if resource_type == 'folder' and folders:
+ resource_id = folders.get(resource_id, resource_id)
+ for member in members:
+ member_type, _, member_id = member.partition(':')
+ if member_type == 'user':
+ continue
+ member_id = member_id.rpartition('@')[0]
+ if prefix and member_id.startswith(prefix):
+ member_id = member_id[len(prefix) + 1:]
+ yield Binding(authoritative, resource_type, resource_id, role,
+ member_type, member_id, conditions)
+
+
+def get_folders(resources):
+ 'Parse resources and return folder id, name tuples.'
+ for r in resources:
+ if r['type'] != 'google_folder':
+ continue
+ for i in r['instances']:
+ yield i['attributes']['id'], i['attributes']['display_name']
+
+
+def output_csv(bindings):
+ 'Output bindings in CSV format.'
+ print(','.join(FIELDS))
+ for b in bindings:
+ print(','.join(str(getattr(b, f)) for f in FIELDS))
+
+
+def output_principals(bindings):
+ 'Output bindings in Markdown format by principals.'
+ resource_grouper = itertools.groupby(
+ bindings, key=lambda b: (b.resource_type, b.resource_id))
+ print('# IAM bindings reference')
+ print('\nLegend: + additive, • conditional.')
+ for resource, resource_groups in resource_grouper:
+ print(f'\n## {resource[0].title()} {resource[1].lower()}\n')
+ principal_grouper = itertools.groupby(
+ resource_groups, key=lambda b: (b.member_type, b.member_id))
+ print('| members | roles |')
+ print('|---|---|')
+ for principal, principal_groups in principal_grouper:
+ roles = []
+ for b in principal_groups:
+ additive = '+' if not b.authoritative else ''
+ conditions = '•' if b.conditions else ''
+ if b.role.startswith('organizations/'):
+ roles.append(f'{b.role} {additive}{conditions}')
+ else:
+ url = (
+ 'https://cloud.google.com/iam/docs/understanding-roles#'
+ f'{b.role.replace("roles/", "")}'
+ )
+ roles.append(f'[{b.role}]({url}) {additive}{conditions}')
+ print((
+ f'|{principal[1]}
{principal[0]}|'
+ f'{"
".join(roles)}|'
+ ))
+
+
+@click.command()
+@click.argument('state-file', type=click.File('r'), default=sys.stdin)
+@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']), default='raw')
+@click.option('--prefix', default=None)
+def main(state_file, format, prefix=None):
+ 'Output IAM bindings parsed from Terraform state file or standard input.'
+ with state_file:
+ data = json.load(state_file)
+ resources = data.get('resources', [])
+ folders = dict(get_folders(resources))
+ bindings = get_bindings(resources, prefix=prefix, folders=folders)
+ bindings = sorted(bindings, key=lambda b: (
+ RESOURCE_SORT.get(b.resource_type, 99), b.resource_id,
+ b.member_type, b.member_id))
+ if format == 'raw':
+ for b in bindings:
+ print(b)
+ else:
+ func = globals().get(f'output_{format}')
+ if not func:
+ raise SystemExit('Unknown format.')
+ func(bindings)
+
+
+if __name__ == '__main__':
+ main()