Merge branch 'master' into fast-dev-dp
This commit is contained in:
lcaggio
GitHub
commit
ee0d0774b4
|
|
@ -17,6 +17,7 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- fast-dev
|
||||
- fast-dev-gke
|
||||
- master
|
||||
tags:
|
||||
- ci
|
||||
|
|
@ -61,13 +62,3 @@ jobs:
|
|||
id: documentation-links-fabric
|
||||
run: |
|
||||
python3 tools/check_links.py .
|
||||
|
||||
# markdown-link-check:
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@master
|
||||
# - uses: gaurav-nelson/github-action-markdown-link-check@v1
|
||||
# with:
|
||||
# use-quiet-mode: "yes"
|
||||
# use-verbose-mode: "yes"
|
||||
# config-file: ".github/workflows/markdown-link-check.json"
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
{
|
||||
"aliveStatusCodes": [429, 200],
|
||||
"retryOn429": false,
|
||||
"ignorePatterns": [
|
||||
{
|
||||
"pattern": "^https://medium.com"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- fast-dev
|
||||
- fast-dev-gke
|
||||
- master
|
||||
tags:
|
||||
- ci
|
||||
|
|
@ -44,7 +45,7 @@ jobs:
|
|||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.9"
|
||||
|
||||
|
||||
- name: Run tests on documentation examples
|
||||
run: |
|
||||
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
|
||||
|
|
@ -71,7 +72,7 @@ jobs:
|
|||
with:
|
||||
terraform_version: 1.1.4
|
||||
terraform_wrapper: false
|
||||
|
||||
|
||||
- name: Run tests environments
|
||||
run: |
|
||||
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
|
||||
|
|
@ -98,9 +99,9 @@ jobs:
|
|||
with:
|
||||
terraform_version: 1.1.4
|
||||
terraform_wrapper: false
|
||||
|
||||
|
||||
- name: Run tests modules
|
||||
run: |
|
||||
run: |
|
||||
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
|
||||
pip install -r tests/requirements.txt
|
||||
pytest -vv tests/modules
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file.
|
|||
- support service dependencies for crypto key bindings in project module
|
||||
- refactor project module in multiple files
|
||||
- add support for per-file option overrides to tfdoc
|
||||
- the `net-vpc` and `project` modules now use the beta provider for shared VPC-related resources
|
||||
|
||||
## [12.0.0] - 2022-01-11
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,6 @@ This repository provides **end-to-end examples** and a **suite of Terraform modu
|
|||
|
||||
The whole repository is meant to be cloned as a single unit, and then forked into separate owned repositories to seed production usage, or used as-is and periodically updated as a complete toolkit for prototyping. You can read more on this approach in our [manifesto](./MANIFESTO.md).
|
||||
|
||||
Both the examples and modules require some measure of Terraform skills to be used effectively. If you are looking for a feature-rich black box to manage project or product creation with minimal specific skills, you might be better served by the [Cloud Foundation Toolkit](https://registry.terraform.io/modules/terraform-google-modules) suite of modules.
|
||||
|
||||
## Organization blueprint (Fabric FAST)
|
||||
|
||||
Setting up a production-ready GCP organization is often a time-consuming process. Fabric [FAST](fast/) aims to speed up this process via two complementary goals. On the one hand, FAST provides a design of a GCP organization that includes the typical elements required by enterprise customers. Secondly, we provide a reference implementation of the FAST design using Terraform.
|
||||
|
|
|
|||
|
|
@ -73,8 +73,13 @@ locals {
|
|||
}
|
||||
labels = merge(coalesce(var.labels, {}), coalesce(var.defaults.labels, {}))
|
||||
network_user_service_accounts = concat(
|
||||
contains(local.services, "compute.googleapis.com") ? ["serviceAccount:${local.service_accounts_robots.compute}"] : [],
|
||||
contains(local.services, "container.googleapis.com") ? ["serviceAccount:${local.service_accounts_robots.container-engine}"] : [],
|
||||
contains(local.services, "compute.googleapis.com") ? [
|
||||
"serviceAccount:${local.service_accounts_robots.compute}"
|
||||
] : [],
|
||||
contains(local.services, "container.googleapis.com") ? [
|
||||
"serviceAccount:${local.service_accounts_robots.container-engine}",
|
||||
"serviceAccount:${local.service_accounts.cloud_services}"
|
||||
] : [],
|
||||
[])
|
||||
services = distinct(concat(var.services, local._services))
|
||||
service_accounts_robots = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
# IAM bindings reference
|
||||
|
||||
Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||
|
||||
## Organization <i>[org_id #0]</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b></b><br><small><i>domain</i></small>|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser) <br>[roles/resourcemanager.organizationViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationViewer) |
|
||||
|<b>gcp-network-admins</b><br><small><i>group</i></small>|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner) <br>[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor) <br>[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) <code>+</code><br>[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) <code>+</code>|
|
||||
|<b>gcp-organization-admins</b><br><small><i>group</i></small>|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner) <br>[roles/cloudsupport.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.admin) <br>[roles/compute.osAdminLogin](https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin) <br>[roles/compute.osLoginExternalUser](https://cloud.google.com/iam/docs/understanding-roles#compute.osLoginExternalUser) <br>[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) <br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) <br>[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) <code>+</code>|
|
||||
|<b>gcp-security-admins</b><br><small><i>group</i></small>|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner) <br>[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor) <br>[roles/iam.securityReviewer](https://cloud.google.com/iam/docs/understanding-roles#iam.securityReviewer) <br>[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/securitycenter.admin](https://cloud.google.com/iam/docs/understanding-roles#securitycenter.admin) <br>[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) <code>+</code><br>[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) <code>+</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
|
||||
|<b>gcp-support</b><br><small><i>group</i></small>|[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor) <br>[roles/logging.viewer](https://cloud.google.com/iam/docs/understanding-roles#logging.viewer) <br>[roles/monitoring.viewer](https://cloud.google.com/iam/docs/understanding-roles#monitoring.viewer) |
|
||||
|<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) <br>[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) <code>+</code><br>[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) <code>+</code>|
|
||||
|<b>prod-resman-0</b><br><small><i>serviceAccount</i></small>|organizations/[org_id #0]/roles/organizationIamAdmin <code>•</code><br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) <code>+</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
|
||||
|
||||
## Project <i>prod-audit-logs-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
|
||||
|
||||
## Project <i>prod-billing-export-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
|
||||
|
||||
## Project <i>prod-iac-core-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-devops</b><br><small><i>group</i></small>|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin) <br>[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) |
|
||||
|<b>gcp-organization-admins</b><br><small><i>group</i></small>|[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) |
|
||||
|<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
|
||||
|<b>prod-resman-0</b><br><small><i>serviceAccount</i></small>|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# Organization bootstrap
|
||||
|
||||
The primary purpose of this stage is to enable critical organization-level functionality that depends on broad administrative permissions, and prepare the prerequisites needed to enable automation in this and future stages.
|
||||
The primary purpose of this stage is to enable critical organization-level functionalities that depend on broad administrative permissions, and prepare the prerequisites needed to enable automation in this and future stages.
|
||||
|
||||
It is intentionally simple, to minimize usage of administrative-level permissions and enable simple auditing and troubleshooting, and only deals with three sets of resources:
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ We have standardized the initial set of groups on those outlined in the [GCP Ent
|
|||
|
||||
### Organization-level IAM
|
||||
|
||||
The service account used in the [Resource Management stage](../01-resman) needs to be able to grant specific roles at the organizational level (`roles/billing.user`, `roles/compute.xpnAdmin`, etc.), to enable specific functionality for subsequent stages that deal with network or security resources, or billing-related activities.
|
||||
The service account used in the [Resource Management stage](../01-resman) needs to be able to grant specific permissions at the organizational level, to enable specific functionality for subsequent stages that deal with network or security resources, or billing-related activities.
|
||||
|
||||
In order to be able to assign those roles without having the full authority of the Organization Admin role, this stage defines a custom role that only allows setting IAM policies on the organization, and grants it via a [delegated role grant](https://cloud.google.com/iam/docs/setting-limits-on-granting-roles) that only allows it to be used to grant a limited subset of roles.
|
||||
|
||||
|
|
@ -36,6 +36,8 @@ In this way, the Resource Management service account can effectively act as an O
|
|||
|
||||
One consequence of the above setup, is the need to configure IAM bindings as non-authoritative for the roles included in the IAM condition, since those same roles are effectively under the control of two stages: this one and Resource Management. Using authoritative bindings for these roles (instead of non-authoritative ones) would generate potential conflicts, where each stage could try to overwrite and negate the bindings applied by the other at each `apply` cycle.
|
||||
|
||||
A full reference of IAM roles managed by this stage [is available here](./IAM.md).
|
||||
|
||||
### Automation project and resources
|
||||
|
||||
One other design choice worth mentioning here is using a single automation project for all foundational stages. We trade off some complexity on the API side (single source for usage quota, multiple service activation) for increased flexibility and simpler operations, while still effectively providing the same degree of separation via resource-level IAM.
|
||||
|
|
@ -95,7 +97,7 @@ To quickly self-grant the above roles, run the following code snippet as the ini
|
|||
```bash
|
||||
export BOOTSTRAP_ORG_ID=123456
|
||||
export BOOTSTRAP_USER=$(gcloud config list --format 'value(core.account)')
|
||||
export BOOTSTRAP_ROLES=(roles/billing.admin roles/logging.admin roles/iam.organizationRoleAdmin roles/resourcemanager.projectCreator)
|
||||
export BOOTSTRAP_ROLES="roles/billing.admin roles/logging.admin roles/iam.organizationRoleAdmin roles/resourcemanager.projectCreator"
|
||||
for role in $BOOTSTRAP_ROLES; do
|
||||
gcloud organizations add-iam-policy-binding $BOOTSTRAP_ORG_ID \
|
||||
--member user:$BOOTSTRAP_USER --role $role
|
||||
|
|
@ -144,10 +146,10 @@ Before the first run, the following IAM groups must exist to allow IAM bindings
|
|||
|
||||
#### Configure variables
|
||||
|
||||
Then make sure you have configured the correct values for the following variables by editing providing a `terraform.tfvars` file:
|
||||
Then make sure you have configured the correct values for the following variables by providing a `terraform.tfvars` file:
|
||||
|
||||
- `billing_account`
|
||||
an object containing the id of your billing account, derived from the Cloud Console UI or by running `gcloud beta billing accounts list`, and the id of the organization owning it, or `null` to use the billing account in isolation
|
||||
an object containing `id` as the id of your billing account, derived from the Cloud Console UI or by running `gcloud beta billing accounts list`, and `organization_id` as the id of the organization owning it, or `null` to use the billing account in isolation
|
||||
- `groups`
|
||||
the name mappings for your groups, if you're following the default convention you can leave this to the provided default
|
||||
- `organization.id`, `organization.domain`, `organization.customer_id`
|
||||
|
|
@ -155,6 +157,25 @@ Then make sure you have configured the correct values for the following variable
|
|||
- `prefix`
|
||||
the fixed prefix used in your naming convention
|
||||
|
||||
You can also adapt the example that follows to your needs:
|
||||
|
||||
```hcl
|
||||
# fetch the required id by running `gcloud beta billing accounts list`
|
||||
billing_account={
|
||||
id="012345-67890A-BCDEF0"
|
||||
organization_id="01234567890"
|
||||
}
|
||||
# get the required info by running `gcloud organizations list`
|
||||
organization={
|
||||
id="01234567890"
|
||||
domain="fast.example.com"
|
||||
customer_id="Cxxxxxxx"
|
||||
}
|
||||
# create your own 4-letters prefix
|
||||
prefix="fast"
|
||||
outputs_location = "../../fast-config"
|
||||
```
|
||||
|
||||
### Output files and cross-stage variables
|
||||
|
||||
At any time during the life of this stage, you can configure it to automatically generate provider configurations and variable files for the following, to simplify exchanging inputs and outputs between stages and avoid having to edit files manually.
|
||||
|
|
@ -178,8 +199,6 @@ Below is the outline of the output files generated by this stage:
|
|||
│ ├── terraform-bootstrap.auto.tfvars.json
|
||||
├── 02-networking
|
||||
│ ├── terraform-bootstrap.auto.tfvars.json
|
||||
├── 02-networking-nva
|
||||
│ ├── terraform-bootstrap.auto.tfvars.json
|
||||
├── 02-security
|
||||
│ ├── terraform-bootstrap.auto.tfvars.json
|
||||
├── 03-gke-multitenant-dev
|
||||
|
|
@ -214,7 +233,7 @@ terraform output -json providers | jq -r '.["00-bootstrap"]' \
|
|||
> providers.tf
|
||||
# migrate state to GCS bucket configured in providers file
|
||||
terraform init -migrate-state
|
||||
# run terraform apply to remo user iam binding
|
||||
# run terraform apply to remove the bootstrap_user iam binding
|
||||
terraform apply
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -34,13 +34,13 @@ module "automation-project" {
|
|||
}
|
||||
# machine (service accounts) IAM bindings
|
||||
iam = {
|
||||
"roles/owner" = [module.automation-tf-bootstrap-sa.iam_email]
|
||||
"roles/owner" = [
|
||||
module.automation-tf-bootstrap-sa.iam_email
|
||||
]
|
||||
"roles/iam.serviceAccountAdmin" = [
|
||||
module.automation-tf-bootstrap-sa.iam_email,
|
||||
module.automation-tf-resman-sa.iam_email
|
||||
]
|
||||
"roles/storage.admin" = [
|
||||
module.automation-tf-bootstrap-sa.iam_email,
|
||||
module.automation-tf-resman-sa.iam_email
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,13 +70,13 @@ locals {
|
|||
|
||||
resource "local_file" "providers" {
|
||||
for_each = var.outputs_location == null ? {} : local.providers
|
||||
filename = "${var.outputs_location}/${each.key}/providers.tf"
|
||||
filename = "${pathexpand(var.outputs_location)}/${each.key}/providers.tf"
|
||||
content = each.value
|
||||
}
|
||||
|
||||
resource "local_file" "tfvars" {
|
||||
for_each = var.outputs_location == null ? {} : local.tfvars
|
||||
filename = "${var.outputs_location}/${each.key}/terraform-bootstrap.auto.tfvars.json"
|
||||
filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-bootstrap.auto.tfvars.json"
|
||||
content = each.value
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
# IAM bindings reference
|
||||
|
||||
Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||
|
||||
## Organization <i>[org_id #0]</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>dev-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) <code>+</code><br>[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) <code>+</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
|
||||
|<b>prod-resman-networking-0</b><br><small><i>serviceAccount</i></small>|[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) <code>+</code><br>[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) <code>+</code><br>[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) <code>+</code>|
|
||||
|<b>prod-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) <code>+</code><br>[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) <code>+</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
|
||||
|<b>prod-resman-security-0</b><br><small><i>serviceAccount</i></small>|[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) <code>+</code><br>[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) <code>+</code>|
|
||||
|
||||
## Folder <i>networking</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-network-admins</b><br><small><i>group</i></small>|[roles/editor](https://cloud.google.com/iam/docs/understanding-roles#editor) |
|
||||
|<b>prod-resman-networking-0</b><br><small><i>serviceAccount</i></small>|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) <br>[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) <br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) |
|
||||
|
||||
## Folder <i>sandbox</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>dev-resman-sandbox-0</b><br><small><i>serviceAccount</i></small>|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) <br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) |
|
||||
|
||||
## Folder <i>security</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-security-admins</b><br><small><i>group</i></small>|[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
|
||||
|<b>prod-resman-security-0</b><br><small><i>serviceAccount</i></small>|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) <br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) |
|
||||
|
||||
## Folder <i>dev</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>dev-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) |
|
||||
|
||||
## Folder <i>prod</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>prod-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) |
|
||||
|
|
@ -65,7 +65,7 @@ terraform output -json providers | jq -r '.["01-resman"]' \
|
|||
> ../01-resman/providers.tf
|
||||
```
|
||||
|
||||
If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as deacribed [here](../00-bootstrap/#output-files-and-cross-stage-variables).
|
||||
If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../00-bootstrap/#output-files-and-cross-stage-variables).
|
||||
|
||||
### Variable configuration
|
||||
|
||||
|
|
@ -136,6 +136,8 @@ For policies where additional data is needed, a root-level `organization_policy_
|
|||
|
||||
IAM roles can be easily edited in the relevant `branch-xxx.tf` file, following the best practice outlined in the [bootstrap stage](../00-bootstrap#customizations) documentation of separating user-level and service-account level IAM policies in modules' `iam_groups`, `iam`, and `iam_additive` variables.
|
||||
|
||||
A full reference of IAM roles managed by this stage [is available here](./IAM.md).
|
||||
|
||||
### Additional folders
|
||||
|
||||
Due to its simplicity, this stage lends itself easily to customizations: adding a new top-level branch (e.g. for shared GKE clusters) is as easy as cloning one of the `branch-xxx.tf` files, and changing names.
|
||||
|
|
@ -175,12 +177,12 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
|
|||
|
||||
| name | description | sensitive | consumers |
|
||||
|---|---|:---:|---|
|
||||
| [networking](outputs.tf#L84) | Data for the networking stage. | | <code>02-networking</code> |
|
||||
| [project_factories](outputs.tf#L94) | Data for the project factories stage. | | <code>xx-teams</code> |
|
||||
| [providers](outputs.tf#L111) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
|
||||
| [sandbox](outputs.tf#L118) | Data for the sandbox stage. | | <code>xx-sandbox</code> |
|
||||
| [security](outputs.tf#L128) | Data for the networking stage. | | <code>02-security</code> |
|
||||
| [teams](outputs.tf#L138) | Data for the teams stage. | | |
|
||||
| [tfvars](outputs.tf#L151) | Terraform variable files for the following stages. | ✓ | |
|
||||
| [networking](outputs.tf#L83) | Data for the networking stage. | | <code>02-networking</code> |
|
||||
| [project_factories](outputs.tf#L93) | Data for the project factories stage. | | <code>xx-teams</code> |
|
||||
| [providers](outputs.tf#L110) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
|
||||
| [sandbox](outputs.tf#L117) | Data for the sandbox stage. | | <code>xx-sandbox</code> |
|
||||
| [security](outputs.tf#L127) | Data for the networking stage. | | <code>02-security</code> |
|
||||
| [teams](outputs.tf#L137) | Data for the teams stage. | | |
|
||||
| [tfvars](outputs.tf#L150) | Terraform variable files for the following stages. | ✓ | |
|
||||
|
||||
<!-- END TFDOC -->
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ module "branch-network-folder" {
|
|||
"roles/owner" = [module.branch-network-sa.iam_email]
|
||||
"roles/resourcemanager.folderAdmin" = [module.branch-network-sa.iam_email]
|
||||
"roles/resourcemanager.projectCreator" = [module.branch-network-sa.iam_email]
|
||||
"roles/compute.xpnAdmin" = [module.branch-network-sa.iam_email]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -57,3 +58,25 @@ module "branch-network-gcs" {
|
|||
"roles/storage.objectAdmin" = [module.branch-network-sa.iam_email]
|
||||
}
|
||||
}
|
||||
|
||||
module "branch-network-prod-folder" {
|
||||
source = "../../../modules/folder"
|
||||
parent = module.branch-network-folder.id
|
||||
name = "prod"
|
||||
iam = {
|
||||
"roles/compute.xpnAdmin" = [
|
||||
module.branch-teams-prod-projectfactory-sa.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "branch-network-dev-folder" {
|
||||
source = "../../../modules/folder"
|
||||
parent = module.branch-network-folder.id
|
||||
name = "dev"
|
||||
iam = {
|
||||
"roles/compute.xpnAdmin" = [
|
||||
module.branch-teams-dev-projectfactory-sa.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,6 +94,9 @@ module "branch-teams-team-dev-folder" {
|
|||
"roles/resourcemanager.projectCreator" = [
|
||||
module.branch-teams-dev-projectfactory-sa.iam_email
|
||||
]
|
||||
"roles/compute.xpnAdmin" = [
|
||||
module.branch-teams-dev-projectfactory-sa.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -141,6 +144,9 @@ module "branch-teams-team-prod-folder" {
|
|||
"roles/resourcemanager.projectCreator" = [
|
||||
module.branch-teams-prod-projectfactory-sa.iam_email
|
||||
]
|
||||
"roles/compute.xpnAdmin" = [
|
||||
module.branch-teams-prod-projectfactory-sa.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,11 +25,6 @@ locals {
|
|||
name = "networking"
|
||||
sa = module.branch-network-sa.email
|
||||
})
|
||||
"02-networking-nva" = templatefile("${path.module}/../../assets/templates/providers.tpl", {
|
||||
bucket = module.branch-network-gcs.name
|
||||
name = "networking-nva"
|
||||
sa = module.branch-network-sa.email
|
||||
})
|
||||
"02-security" = templatefile("${path.module}/../../assets/templates/providers.tpl", {
|
||||
bucket = module.branch-security-gcs.name
|
||||
name = "security"
|
||||
|
|
@ -53,7 +48,11 @@ locals {
|
|||
}
|
||||
tfvars = {
|
||||
"02-networking" = jsonencode({
|
||||
folder_id = module.branch-network-folder.id
|
||||
folder_ids = {
|
||||
networking = module.branch-network-folder.id
|
||||
networking-dev = module.branch-network-dev-folder.id
|
||||
networking-prod = module.branch-network-prod-folder.id
|
||||
}
|
||||
project_factory_sa = local._project_factory_sas
|
||||
})
|
||||
"02-security" = jsonencode({
|
||||
|
|
@ -69,13 +68,13 @@ locals {
|
|||
|
||||
resource "local_file" "providers" {
|
||||
for_each = var.outputs_location == null ? {} : local.providers
|
||||
filename = "${var.outputs_location}/${each.key}/providers.tf"
|
||||
filename = "${pathexpand(var.outputs_location)}/${each.key}/providers.tf"
|
||||
content = each.value
|
||||
}
|
||||
|
||||
resource "local_file" "tfvars" {
|
||||
for_each = var.outputs_location == null ? {} : local.tfvars
|
||||
filename = "${var.outputs_location}/${each.key}/terraform-resman.auto.tfvars.json"
|
||||
filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-resman.auto.tfvars.json"
|
||||
content = each.value
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -187,12 +187,13 @@ If you have set a valid value for `outputs_location` in the bootstrap and in the
|
|||
ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json
|
||||
ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json
|
||||
```
|
||||
If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../00-bootstrap/#output-files-and-cross-stage-variables).
|
||||
|
||||
Please, refer to the [variables](#variables) table below for a map of the variable origins, and use the sections below to understand how to adapt this stage to your networking configuration.
|
||||
|
||||
### VPCs
|
||||
|
||||
VPCs are defined in separate files, one for `untrusted landing`, one for `trusted landing`, one for `prod` and one for `dev`.
|
||||
VPCs are defined in separate files, one for `landing` (trusted and untrusted), one for `prod` and one for `dev`.
|
||||
|
||||
These files contain different resources:
|
||||
|
||||
|
|
@ -321,19 +322,19 @@ Don't forget to add a peering zone in the landing project and point it to the ne
|
|||
| name | description | type | required | default | producer |
|
||||
|---|---|:---:|:---:|:---:|:---:|
|
||||
| [billing_account_id](variables.tf#L17) | Billing account id. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [organization](variables.tf#L99) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [prefix](variables.tf#L115) | Prefix used for resources that need unique names. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L59) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code>map(string)</code> | ✓ | | <code>01-resman</code> |
|
||||
| [organization](variables.tf#L91) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [prefix](variables.tf#L107) | Prefix used for resources that need unique names. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [custom_adv](variables.tf#L23) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev_ew1 = "10.128.128.0/19" gcp_dev_ew4 = "10.128.160.0/19" gcp_landing_trusted_ew1 = "10.128.64.0/19" gcp_landing_trusted_ew4 = "10.128.96.0/19" gcp_landing_untrusted_ew1 = "10.128.0.0/19" gcp_landing_untrusted_ew4 = "10.128.32.0/19" gcp_prod_ew1 = "10.128.192.0/19" gcp_prod_ew4 = "10.128.224.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||
| [data_dir](variables.tf#L45) | Relative path for the folder storing configuration data for network resources. | <code>string</code> | | <code>"data"</code> | |
|
||||
| [dns](variables.tf#L51) | Onprem DNS resolvers | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
||||
| [folder_id](variables.tf#L59) | Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code>string</code> | | <code>null</code> | <code>01-resman</code> |
|
||||
| [l7ilb_subnets](variables.tf#L73) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" } ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" } ] }">{…}</code> | |
|
||||
| [onprem_cidr](variables.tf#L91) | Onprem addresses in name => range format. | <code>map(string)</code> | | <code title="{ main = "10.0.0.0/24" }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L109) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [project_factory_sa](variables.tf#L121) | IAM emails for project factory service accounts | <code>map(string)</code> | | <code>{}</code> | <code>01-resman</code> |
|
||||
| [psa_ranges](variables.tf#L128) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code>map(map(string))</code> | | <code title="{ prod = { cloudsql-mysql = "10.128.94.0/24" cloudsql-sqlserver = "10.128.95.0/24" } dev = { cloudsql-mysql = "10.128.62.0/24" cloudsql-sqlserver = "10.128.63.0/24" } }">{…}</code> | |
|
||||
| [router_configs](variables.tf#L143) | Configurations for CRs and onprem routers. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-trusted-ew1 = { asn = "65534" adv = null } landing-trusted-ew4 = { asn = "65534" adv = null } }">{…}</code> | |
|
||||
| [vpn_onprem_configs](variables.tf#L166) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(object({ id = number ip_address = string })) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-trusted-ew1 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } landing-trusted-ew4 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [l7ilb_subnets](variables.tf#L65) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" } ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" } ] }">{…}</code> | |
|
||||
| [onprem_cidr](variables.tf#L83) | Onprem addresses in name => range format. | <code>map(string)</code> | | <code title="{ main = "10.0.0.0/24" }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L101) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [project_factory_sa](variables.tf#L113) | IAM emails for project factory service accounts | <code>map(string)</code> | | <code>{}</code> | <code>01-resman</code> |
|
||||
| [psa_ranges](variables.tf#L120) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code>map(map(string))</code> | | <code title="{ prod = { cloudsql-mysql = "10.128.94.0/24" cloudsql-sqlserver = "10.128.95.0/24" } dev = { cloudsql-mysql = "10.128.62.0/24" cloudsql-sqlserver = "10.128.63.0/24" } }">{…}</code> | |
|
||||
| [router_configs](variables.tf#L135) | Configurations for CRs and onprem routers. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-trusted-ew1 = { asn = "65534" adv = null } landing-trusted-ew4 = { asn = "65534" adv = null } }">{…}</code> | |
|
||||
| [vpn_onprem_configs](variables.tf#L158) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(object({ id = number ip_address = string })) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-trusted-ew1 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } landing-trusted-ew4 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ module "folder" {
|
|||
source = "../../../modules/folder"
|
||||
parent = "organizations/${var.organization.id}"
|
||||
name = "Networking"
|
||||
folder_create = var.folder_id == null
|
||||
id = var.folder_id
|
||||
folder_create = var.folder_ids.networking == null
|
||||
id = var.folder_ids.networking
|
||||
firewall_policy_factory = {
|
||||
cidr_file = "${var.data_dir}/cidrs.yaml"
|
||||
policy_name = null
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ locals {
|
|||
|
||||
resource "local_file" "tfvars" {
|
||||
for_each = var.outputs_location == null ? {} : local.tfvars
|
||||
filename = "${var.outputs_location}/${each.key}/terraform-networking.auto.tfvars.json"
|
||||
filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-networking.auto.tfvars.json"
|
||||
content = each.value
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -56,18 +56,10 @@ variable "dns" {
|
|||
}
|
||||
}
|
||||
|
||||
variable "folder_id" {
|
||||
variable "folder_ids" {
|
||||
# tfdoc:variable:source 01-resman
|
||||
description = "Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
|
||||
type = string
|
||||
default = null
|
||||
validation {
|
||||
condition = (
|
||||
var.folder_id == null ||
|
||||
can(regex("folders/[0-9]{8,}", var.folder_id))
|
||||
)
|
||||
error_message = "Invalid folder_id. Should be in 'folders/nnnnnnnnnnn' format."
|
||||
}
|
||||
description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
|
||||
type = map(string)
|
||||
}
|
||||
|
||||
variable "l7ilb_subnets" {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ module "landing-project" {
|
|||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account_id
|
||||
name = "prod-net-landing-0"
|
||||
parent = var.folder_id
|
||||
parent = var.folder_ids.networking-prod
|
||||
prefix = var.prefix
|
||||
service_config = {
|
||||
disable_on_destroy = false
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ module "dev-spoke-project" {
|
|||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account_id
|
||||
name = "dev-net-spoke-0"
|
||||
parent = var.folder_id
|
||||
parent = var.folder_ids.networking-dev
|
||||
prefix = var.prefix
|
||||
service_config = {
|
||||
disable_on_destroy = false
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ module "prod-spoke-project" {
|
|||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account_id
|
||||
name = "prod-net-spoke-0"
|
||||
parent = var.folder_id
|
||||
parent = var.folder_ids.networking-prod
|
||||
prefix = var.prefix
|
||||
service_config = {
|
||||
disable_on_destroy = false
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
# IAM bindings reference
|
||||
|
||||
Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||
|
||||
## Project <i>dev-net-spoke-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>dev-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/resourcemanager.projectIamAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) <code>•</code><br>[roles/dns.admin](https://cloud.google.com/iam/docs/understanding-roles#dns.admin) |
|
||||
|<b>prod-resman-pf-0</b><br><small><i>serviceAccount</i></small>|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin |
|
||||
|
||||
## Project <i>prod-net-spoke-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>prod-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/resourcemanager.projectIamAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) <code>•</code><br>organizations/[org_id #0]/roles/serviceProjectNetworkAdmin <br>[roles/dns.admin](https://cloud.google.com/iam/docs/understanding-roles#dns.admin) |
|
||||
|
|
@ -309,20 +309,20 @@ DNS configurations are centralised in the `dns.tf` file. Spokes delegate DNS res
|
|||
| name | description | type | required | default | producer |
|
||||
|---|---|:---:|:---:|:---:|:---:|
|
||||
| [billing_account_id](variables.tf#L17) | Billing account id. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [organization](variables.tf#L93) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [prefix](variables.tf#L109) | Prefix used for resources that need unique names. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L61) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code>map(string)</code> | ✓ | | <code>01-resman</code> |
|
||||
| [organization](variables.tf#L85) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [prefix](variables.tf#L101) | Prefix used for resources that need unique names. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [custom_adv](variables.tf#L23) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev = "10.128.32.0/19" gcp_landing = "10.128.0.0/19" gcp_prod = "10.128.64.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||
| [custom_roles](variables.tf#L40) | Custom roles defined at the org level, in key => id format. | <code>map(string)</code> | | <code>{}</code> | <code>00-bootstrap</code> |
|
||||
| [data_dir](variables.tf#L47) | Relative path for the folder storing configuration data for network resources. | <code>string</code> | | <code>"data"</code> | |
|
||||
| [dns](variables.tf#L53) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
||||
| [folder_id](variables.tf#L61) | Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code>string</code> | | <code>null</code> | <code>01-resman</code> |
|
||||
| [l7ilb_subnets](variables.tf#L75) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" } ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" } ] }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L103) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [project_factory_sa](variables.tf#L115) | IAM emails for project factory service accounts. | <code>map(string)</code> | | <code>{}</code> | <code>01-resman</code> |
|
||||
| [psa_ranges](variables.tf#L122) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code>map(map(string))</code> | | <code title="{ prod = { cloudsql-mysql = "10.128.94.0/24" cloudsql-sqlserver = "10.128.95.0/24" } dev = { cloudsql-mysql = "10.128.62.0/24" cloudsql-sqlserver = "10.128.63.0/24" } }">{…}</code> | |
|
||||
| [router_configs](variables.tf#L137) | Configurations for CRs and onprem routers. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ onprem-ew1 = { asn = "65534" adv = null } landing-ew1 = { asn = "64512", adv = null } landing-ew4 = { asn = "64512", adv = null } spoke-dev-ew1 = { asn = "64513", adv = null } spoke-dev-ew4 = { asn = "64513", adv = null } spoke-prod-ew1 = { asn = "64514", adv = null } spoke-prod-ew4 = { asn = "64514", adv = null } }">{…}</code> | |
|
||||
| [vpn_onprem_configs](variables.tf#L161) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(object({ id = number ip_address = string })) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [vpn_spoke_configs](variables.tf#L217) | VPN gateway configuration for spokes. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) session_range = string }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { adv = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } session_range = null } landing-ew4 = { adv = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } session_range = null } dev-ew1 = { adv = { default = false custom = ["gcp_dev"] } session_range = "169.254.0.0/27" } prod-ew1 = { adv = { default = false custom = ["gcp_prod"] } session_range = "169.254.0.64/27" } prod-ew4 = { adv = { default = false custom = ["gcp_prod"] } session_range = "169.254.0.96/27" } }">{…}</code> | |
|
||||
| [l7ilb_subnets](variables.tf#L67) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" } ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" } ] }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L95) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [project_factory_sa](variables.tf#L107) | IAM emails for project factory service accounts. | <code>map(string)</code> | | <code>{}</code> | <code>01-resman</code> |
|
||||
| [psa_ranges](variables.tf#L114) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code>map(map(string))</code> | | <code title="{ prod = { cloudsql-mysql = "10.128.94.0/24" cloudsql-sqlserver = "10.128.95.0/24" } dev = { cloudsql-mysql = "10.128.62.0/24" cloudsql-sqlserver = "10.128.63.0/24" } }">{…}</code> | |
|
||||
| [router_configs](variables.tf#L129) | Configurations for CRs and onprem routers. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ onprem-ew1 = { asn = "65534" adv = null } landing-ew1 = { asn = "64512", adv = null } landing-ew4 = { asn = "64512", adv = null } spoke-dev-ew1 = { asn = "64513", adv = null } spoke-dev-ew4 = { asn = "64513", adv = null } spoke-prod-ew1 = { asn = "64514", adv = null } spoke-prod-ew4 = { asn = "64514", adv = null } }">{…}</code> | |
|
||||
| [vpn_onprem_configs](variables.tf#L153) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(object({ id = number ip_address = string })) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [vpn_spoke_configs](variables.tf#L209) | VPN gateway configuration for spokes. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) session_range = string }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { adv = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } session_range = null } landing-ew4 = { adv = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } session_range = null } dev-ew1 = { adv = { default = false custom = ["gcp_dev"] } session_range = "169.254.0.0/27" } prod-ew1 = { adv = { default = false custom = ["gcp_prod"] } session_range = "169.254.0.64/27" } prod-ew4 = { adv = { default = false custom = ["gcp_prod"] } session_range = "169.254.0.96/27" } }">{…}</code> | |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -53,8 +53,8 @@ module "folder" {
|
|||
source = "../../../modules/folder"
|
||||
parent = "organizations/${var.organization.id}"
|
||||
name = "Networking"
|
||||
folder_create = var.folder_id == null
|
||||
id = var.folder_id
|
||||
folder_create = var.folder_ids.networking == null
|
||||
id = var.folder_ids.networking
|
||||
firewall_policy_factory = {
|
||||
cidr_file = "${var.data_dir}/cidrs.yaml"
|
||||
policy_name = null
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ locals {
|
|||
|
||||
resource "local_file" "tfvars" {
|
||||
for_each = var.outputs_location == null ? {} : local.tfvars
|
||||
filename = "${var.outputs_location}/${each.key}/terraform-networking.auto.tfvars.json"
|
||||
filename = "${pathexpand(var.outputs_location)}/${each.key}/terraform-networking.auto.tfvars.json"
|
||||
content = each.value
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,18 +58,10 @@ variable "dns" {
|
|||
}
|
||||
}
|
||||
|
||||
variable "folder_id" {
|
||||
variable "folder_ids" {
|
||||
# tfdoc:variable:source 01-resman
|
||||
description = "Folder to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
|
||||
type = string
|
||||
default = null
|
||||
validation {
|
||||
condition = (
|
||||
var.folder_id == null ||
|
||||
can(regex("folders/[0-9]{8,}", var.folder_id))
|
||||
)
|
||||
error_message = "Invalid folder_id. Should be in 'folders/nnnnnnnnnnn' format."
|
||||
}
|
||||
description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
|
||||
type = map(string)
|
||||
}
|
||||
|
||||
variable "l7ilb_subnets" {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ module "landing-project" {
|
|||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account_id
|
||||
name = "prod-net-landing-0"
|
||||
parent = var.folder_id
|
||||
parent = var.folder_ids.networking-prod
|
||||
prefix = var.prefix
|
||||
service_config = {
|
||||
disable_on_destroy = false
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ module "dev-spoke-project" {
|
|||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account_id
|
||||
name = "dev-net-spoke-0"
|
||||
parent = var.folder_id
|
||||
parent = var.folder_ids.networking-dev
|
||||
prefix = var.prefix
|
||||
service_config = {
|
||||
disable_on_destroy = false
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ module "prod-spoke-project" {
|
|||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account_id
|
||||
name = "prod-net-spoke-0"
|
||||
parent = var.folder_id
|
||||
parent = var.folder_ids.networking-prod
|
||||
prefix = var.prefix
|
||||
service_config = {
|
||||
disable_on_destroy = false
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
# IAM bindings reference
|
||||
|
||||
Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||
|
||||
## Project <i>dev-sec-core-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>dev-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/cloudkms.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.admin) <code>+</code><code>•</code><br>[roles/cloudkms.viewer](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.viewer) |
|
||||
|
||||
## Project <i>prod-sec-core-0</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>prod-resman-pf-0</b><br><small><i>serviceAccount</i></small>|[roles/cloudkms.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.admin) <code>+</code><code>•</code><br>[roles/cloudkms.viewer](https://cloud.google.com/iam/docs/understanding-roles#cloudkms.viewer) |
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
resource "local_file" "dev_sec_kms" {
|
||||
for_each = var.outputs_location == null ? {} : { 1 = 1 }
|
||||
filename = "${var.outputs_location}/yamls/02-security-kms-dev-keys.yaml"
|
||||
filename = "${pathexpand(var.outputs_location)}/yamls/02-security-kms-dev-keys.yaml"
|
||||
content = yamlencode({
|
||||
for k, m in module.dev-sec-kms : k => m.key_ids
|
||||
})
|
||||
|
|
@ -26,7 +26,7 @@ resource "local_file" "dev_sec_kms" {
|
|||
|
||||
resource "local_file" "prod_sec_kms" {
|
||||
for_each = var.outputs_location == null ? {} : { 1 = 1 }
|
||||
filename = "${var.outputs_location}/yamls/02-security-kms-prod-keys.yaml"
|
||||
filename = "${pathexpand(var.outputs_location)}/yamls/02-security-kms-prod-keys.yaml"
|
||||
content = yamlencode({
|
||||
for k, m in module.prod-sec-kms : k => m.key_ids
|
||||
})
|
||||
|
|
|
|||
|
|
@ -108,11 +108,11 @@ terraform apply
|
|||
| name | description | type | required | default | producer |
|
||||
|---|---|:---:|:---:|:---:|:---:|
|
||||
| [billing_account_id](variables.tf#L19) | Billing account id. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||
| [shared_vpc_self_link](variables.tf#L44) | Self link for the shared VPC. | <code>string</code> | ✓ | | <code>02-networking</code> |
|
||||
| [vpc_host_project](variables.tf#L50) | Host project for the shared VPC. | <code>string</code> | ✓ | | <code>02-networking</code> |
|
||||
| [data_dir](variables.tf#L25) | Relative path for the folder storing configuration data. | <code>string</code> | | <code>"data/projects"</code> | |
|
||||
| [defaults_file](variables.tf#L38) | Relative path for the file storing the project factory configuration. | <code>string</code> | | <code>"data/defaults.yaml"</code> | |
|
||||
| [environment_dns_zone](variables.tf#L31) | DNS zone suffix for environment. | <code>string</code> | | <code>null</code> | <code>02-networking</code> |
|
||||
| [shared_vpc_self_link](variables.tf#L44) | Self link for the shared VPC. | <code>string</code> | | <code>null</code> | <code>02-networking</code> |
|
||||
| [vpc_host_project](variables.tf#L51) | Host project for the shared VPC. | <code>string</code> | | <code>null</code> | <code>02-networking</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -45,10 +45,12 @@ variable "shared_vpc_self_link" {
|
|||
# tfdoc:variable:source 02-networking
|
||||
description = "Self link for the shared VPC."
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "vpc_host_project" {
|
||||
# tfdoc:variable:source 02-networking
|
||||
description = "Host project for the shared VPC."
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,145 @@
|
|||
# Google Cloud IoT Core Module
|
||||
|
||||
This module sets up Cloud IoT Core Registry, registers IoT Devices and configures Pub/Sub topics required in Cloud IoT Core.
|
||||
|
||||
To use this module, ensure the following APIs are enabled:
|
||||
* pubsub.googleapis.com
|
||||
* cloudiot.googleapis.com
|
||||
|
||||
## Simple Example
|
||||
|
||||
Basic example showing how to create an IoT Platform (IoT Core), connected to a set of given Pub/Sub topics and provision IoT devices.
|
||||
|
||||
Devices certificates must exist before calling this module. You can generate these certificates using the following command
|
||||
|
||||
```
|
||||
openssl req -x509 -newkey rsa:2048 -keyout rsa_private.pem -nodes -out rsa_cert.pem -subj "/CN=unused"
|
||||
```
|
||||
|
||||
And then provision public certificate path, together with the rest of device configuration in a devices yaml file following the following format
|
||||
```yaml
|
||||
device_id: # id of your IoT Device
|
||||
is_blocked: # false to allow device connection with IoT Registry
|
||||
is_gateway: # true to indicate the device connecting acts as a gateway for other IoT Devices
|
||||
log_level: # device logs level
|
||||
certificate_file: # public certificate path, generated as explained in the previous step
|
||||
certificate_format: # Certificates format values are RSA_PEM, RSA_X509_PEM, ES256_PEM, and ES256_X509_PEM
|
||||
```
|
||||
|
||||
Example Device config yaml configuration
|
||||
```yaml
|
||||
device_1:
|
||||
is_blocked: false
|
||||
is_gateway: false
|
||||
log_level: INFO
|
||||
certificate_file: device_certs/rsa_cert5.pem
|
||||
certificate_format: RSA_X509_PEM
|
||||
device_2:
|
||||
is_blocked: true
|
||||
is_gateway: false
|
||||
log_level: INFO
|
||||
certificate_file: device_certs/rsa_cert5.pem
|
||||
certificate_format: RSA_X509_PEM
|
||||
```
|
||||
|
||||
```hcl
|
||||
module "iot-platform" {
|
||||
source = "./modules/iot-core"
|
||||
project_id = "my_project_id"
|
||||
region = "europe-west1"
|
||||
telemetry_pubsub_topic_id = "telemetry_topic_id"
|
||||
status_pubsub_topic_id = "status_topic_id"
|
||||
protocols = {
|
||||
http = false,
|
||||
mqtt = true
|
||||
}
|
||||
devices_config_directory = "./devices_config_folder"
|
||||
}
|
||||
# tftest:skip
|
||||
|
||||
```
|
||||
|
||||
Now, we can test sending telemetry messages from devices to our IoT Platform, for example using the MQTT demo client at https://github.com/googleapis/nodejs-iot/tree/main/samples/mqtt_example
|
||||
|
||||
## Example with specific PubSub topics for custom MQTT topics
|
||||
|
||||
If you need to match specific MQTT topics (eg, /temperature) into specific PubSub topics, you can use extra_telemetry_pubsub_topic_ids for that, as in the following example:
|
||||
|
||||
```hcl
|
||||
module "iot-platform" {
|
||||
source = "./modules/iot-core"
|
||||
project_id = "my_project_id"
|
||||
region = "europe-west1"
|
||||
telemetry_pubsub_topic_id = "telemetry_topic_id"
|
||||
status_pubsub_topic_id = "status_topic_id"
|
||||
extra_telemetry_pubsub_topic_ids = {
|
||||
"temperature" = "temp_topic_id",
|
||||
"humidity" = "hum_topic_id"
|
||||
}
|
||||
protocols = {
|
||||
http = false,
|
||||
mqtt = true
|
||||
}
|
||||
devices_config_directory = "./devices_config_folder"
|
||||
}
|
||||
# tftest:skip
|
||||
|
||||
```
|
||||
|
||||
## Example integrated with Data Foundation Platform
|
||||
In this example, we will show how to extend the **[Data Foundations Platform](../../data-solutions/data-platform-foundations/)** to include IoT Platform as a new source of data.
|
||||
|
||||
|
||||
|
||||
1. First, we will setup Environment following instructions in **[Environment Setup](../../data-solutions/data-platform-foundations/01-environment/)** to setup projects and SAs required. Get output variable project_ids.landing as will be used later
|
||||
|
||||
1. Second, execute instructions in **[Environment Setup](../../data-solutions/data-platform-foundations/02-resources/)** to provision PubSub, DataFlow, BQ,... Get variable landing-pubsub as will be used later to create IoT Registry
|
||||
|
||||
1. Now it is time to provision IoT Platform. Modify landing-project-id and landing_pubsub_topic_id with output variables obtained before. Create device certificates as shown in the Simple Example and register them in devices.yaml file together with deviceids.
|
||||
|
||||
```hcl
|
||||
module "iot-platform" {
|
||||
source = "./modules/iot-core"
|
||||
project_id = "landing-project-id"
|
||||
region = "europe-west1"
|
||||
telemetry_pubsub_topic_id = "landing_pubsub_topic_id"
|
||||
status_pubsub_topic_id = "status_pubsub_topic_id"
|
||||
protocols = {
|
||||
http = false,
|
||||
mqtt = true
|
||||
}
|
||||
devices_config_directory = "./devices_config_folder"
|
||||
}
|
||||
# tftest:skip
|
||||
```
|
||||
1. After that, we can setup the pipeline "PubSub to BigQuery" shown at **[Pipeline Setup](../../data-solutions/data-platform-foundations/03-pipeline/pubsub_to_bigquery.md)**
|
||||
|
||||
1. Finally, instead of testing the pipeline by sending messages to PubSub, we can now test sending telemetry messages from simulated IoT devices to our IoT Platform, for example using the MQTT demo client at https://github.com/googleapis/nodejs-iot/tree/main/samples/mqtt_example . We shall edit the client script cloudiot_mqtt_example_nodejs.js to send messages following the pipeline message format, so they are processed by DataFlow job and inserted in the BigQuery table.
|
||||
```
|
||||
const payload = '{"name": "device4", "surname": "NA", "timestamp":"'+Math.floor(Date.now()/1000)+'"}';
|
||||
```
|
||||
|
||||
Or even better, create a new BigQuery table with our IoT sensors data columns and modify the DataFlow job to push data to it.
|
||||
<!-- BEGIN TFDOC -->
|
||||
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [devices_config_directory](variables.tf#L17) | Path to folder where devices configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L34) | Project were resources will be deployed | <code>string</code> | ✓ | |
|
||||
| [region](variables.tf#L48) | Region were resources will be deployed | <code>string</code> | ✓ | |
|
||||
| [status_pubsub_topic_id](variables.tf#L59) | pub sub topic for status messages (GCP-->Device) | <code>string</code> | ✓ | |
|
||||
| [telemetry_pubsub_topic_id](variables.tf#L64) | pub sub topic for telemetry messages (Device-->GCP) | <code>string</code> | ✓ | |
|
||||
| [extra_telemetry_pubsub_topic_ids](variables.tf#L22) | additional pubsub topics linked to adhoc MQTT topics (Device-->GCP) in the format MQTT_TOPIC: PUBSUB_TOPIC_ID | <code>map(string)</code> | | <code>{}</code> |
|
||||
| [log_level](variables.tf#L28) | IoT Registry Log level | <code>string</code> | | <code>"INFO"</code> |
|
||||
| [protocols](variables.tf#L39) | IoT protocols (HTTP / MQTT) activation | <code title="object({ http = bool, mqtt = bool })">object({…})</code> | | <code>{ http = true, mqtt = true }</code> |
|
||||
| [registry_name](variables.tf#L53) | Name for the IoT Core Registry | <code>string</code> | | <code>"cloudiot-registry"</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
| name | description | sensitive |
|
||||
|---|---|:---:|
|
||||
| [iot_registry](outputs.tf#L17) | Cloud IoT Core Registry | |
|
||||
|
||||
<!-- END TFDOC -->
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 470 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 52 KiB |
|
|
@ -0,0 +1,95 @@
|
|||
|
||||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
locals {
|
||||
devices_config_files = [
|
||||
for config_file in fileset("${path.root}/${var.devices_config_directory}", "**/*.yaml") :
|
||||
"${path.root}/${var.devices_config_directory}/${config_file}"
|
||||
]
|
||||
|
||||
device_config = merge(
|
||||
[
|
||||
for config_file in local.devices_config_files :
|
||||
try(yamldecode(file(config_file)), {})
|
||||
]...
|
||||
)
|
||||
}
|
||||
|
||||
#---------------------------------------------------------
|
||||
# Create IoT Core Registry
|
||||
#---------------------------------------------------------
|
||||
|
||||
resource "google_cloudiot_registry" "registry" {
|
||||
|
||||
name = var.registry_name
|
||||
project = var.project_id
|
||||
region = var.region
|
||||
|
||||
dynamic "event_notification_configs" {
|
||||
for_each = var.extra_telemetry_pubsub_topic_ids
|
||||
content {
|
||||
pubsub_topic_name = event_notification_configs.value
|
||||
subfolder_matches = event_notification_configs.key
|
||||
}
|
||||
}
|
||||
|
||||
event_notification_configs {
|
||||
pubsub_topic_name = var.telemetry_pubsub_topic_id
|
||||
subfolder_matches = ""
|
||||
}
|
||||
|
||||
state_notification_config = {
|
||||
pubsub_topic_name = var.status_pubsub_topic_id
|
||||
}
|
||||
|
||||
mqtt_config = {
|
||||
mqtt_enabled_state = var.protocols.mqtt ? "MQTT_ENABLED" : "MQTT_DISABLED"
|
||||
}
|
||||
|
||||
http_config = {
|
||||
http_enabled_state = var.protocols.http ? "HTTP_ENABLED" : "HTTP_DISABLED"
|
||||
}
|
||||
|
||||
log_level = var.log_level
|
||||
|
||||
}
|
||||
|
||||
#---------------------------------------------------------
|
||||
# Create IoT Core Device
|
||||
# certificate created using: openssl req -x509 -newkey rsa:2048 -keyout rsa_private.pem -nodes -out rsa_cert.pem -subj "/CN=unused"
|
||||
#---------------------------------------------------------
|
||||
|
||||
resource "google_cloudiot_device" "device" {
|
||||
for_each = local.device_config
|
||||
name = each.key
|
||||
registry = google_cloudiot_registry.registry.id
|
||||
|
||||
credentials {
|
||||
public_key {
|
||||
format = try(each.value.certificate_format, null)
|
||||
key = try(file(each.value.certificate_file), null)
|
||||
}
|
||||
}
|
||||
|
||||
blocked = try(each.value.is_blocked, null)
|
||||
|
||||
log_level = try(each.value.log_level, null)
|
||||
|
||||
gateway_config {
|
||||
gateway_type = try(each.value.is_gateway, null) ? "GATEWAY" : "NON_GATEWAY"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
output "iot_registry" {
|
||||
description = "Cloud IoT Core Registry"
|
||||
value = google_cloudiot_registry.registry
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "devices_config_directory" {
|
||||
description = "Path to folder where devices configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "extra_telemetry_pubsub_topic_ids" {
|
||||
description = "additional pubsub topics linked to adhoc MQTT topics (Device-->GCP) in the format MQTT_TOPIC: PUBSUB_TOPIC_ID"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "log_level" {
|
||||
description = "IoT Registry Log level"
|
||||
type = string
|
||||
default = "INFO"
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Project were resources will be deployed"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "protocols" {
|
||||
description = "IoT protocols (HTTP / MQTT) activation"
|
||||
type = object({
|
||||
http = bool,
|
||||
mqtt = bool
|
||||
})
|
||||
default = { http = true, mqtt = true }
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
description = "Region were resources will be deployed"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "registry_name" {
|
||||
description = "Name for the IoT Core Registry"
|
||||
type = string
|
||||
default = "cloudiot-registry"
|
||||
}
|
||||
|
||||
variable "status_pubsub_topic_id" {
|
||||
description = "pub sub topic for status messages (GCP-->Device)"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "telemetry_pubsub_topic_id" {
|
||||
description = "pub sub topic for telemetry messages (Device-->GCP)"
|
||||
type = string
|
||||
}
|
||||
|
|
@ -153,12 +153,14 @@ resource "google_compute_network_peering" "remote" {
|
|||
}
|
||||
|
||||
resource "google_compute_shared_vpc_host_project" "shared_vpc_host" {
|
||||
provider = google-beta
|
||||
count = var.shared_vpc_host ? 1 : 0
|
||||
project = var.project_id
|
||||
depends_on = [local.network]
|
||||
}
|
||||
|
||||
resource "google_compute_shared_vpc_service_project" "service_projects" {
|
||||
provider = google-beta
|
||||
for_each = (
|
||||
var.shared_vpc_host && var.shared_vpc_service_projects != null
|
||||
? toset(var.shared_vpc_service_projects)
|
||||
|
|
|
|||
|
|
@ -17,11 +17,13 @@
|
|||
# tfdoc:file:description Shared VPC project-level configuration.
|
||||
|
||||
resource "google_compute_shared_vpc_host_project" "shared_vpc_host" {
|
||||
count = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0
|
||||
project = local.project.project_id
|
||||
provider = google-beta
|
||||
count = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0
|
||||
project = local.project.project_id
|
||||
}
|
||||
|
||||
resource "google_compute_shared_vpc_service_project" "service_projects" {
|
||||
provider = google-beta
|
||||
for_each = (
|
||||
try(var.shared_vpc_host_config.enabled, false)
|
||||
? toset(coalesce(var.shared_vpc_host_config.service_projects, []))
|
||||
|
|
@ -33,6 +35,7 @@ resource "google_compute_shared_vpc_service_project" "service_projects" {
|
|||
}
|
||||
|
||||
resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
|
||||
provider = google-beta
|
||||
count = try(var.shared_vpc_service_config.attach, false) ? 1 : 0
|
||||
host_project = var.shared_vpc_service_config.host_project
|
||||
service_project = local.project.project_id
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
'Parse and output IAM bindings from Terraform state file.'
|
||||
|
||||
import collections
|
||||
import json
|
||||
import itertools
|
||||
import re
|
||||
import sys
|
||||
|
||||
import click
|
||||
|
||||
|
||||
FIELDS = (
|
||||
'authoritative', 'resource_type', 'resource_id', 'role', 'member_type',
|
||||
'member_id', 'conditions'
|
||||
)
|
||||
ORG_IDS = {}
|
||||
RESOURCE_SORT = {'organization': 0, 'folder': 1, 'project': 2}
|
||||
RESOURCE_TYPE_RE = re.compile(r'^google_([^_]+)_iam_([^_]+)$')
|
||||
Binding = collections.namedtuple('Binding', ' '.join(FIELDS))
|
||||
|
||||
|
||||
def _org_id(resource_id):
|
||||
if resource_id not in ORG_IDS:
|
||||
ORG_IDS[resource_id] = f'[org_id #{len(ORG_IDS)}]'
|
||||
return ORG_IDS[resource_id]
|
||||
|
||||
|
||||
def get_bindings(resources, prefix=None, folders=None):
|
||||
'Parse resources and return bindings.'
|
||||
org_ids = {}
|
||||
for r in resources:
|
||||
m = RESOURCE_TYPE_RE.match(r['type'])
|
||||
if not m:
|
||||
continue
|
||||
resource_type = m.group(1)
|
||||
authoritative = m.group(2) == 'binding'
|
||||
for i in r.get('instances'):
|
||||
attrs = i['attributes']
|
||||
conditions = ' '.join(c['title'] for c in attrs.get('condition', []))
|
||||
if resource_type == 'organization':
|
||||
resource_id = _org_id(attrs['org_id'])
|
||||
else:
|
||||
resource_id = attrs[resource_type]
|
||||
if prefix and resource_id.startswith(prefix):
|
||||
resource_id = resource_id[len(prefix) + 1:]
|
||||
role = attrs['role']
|
||||
if role.startswith('organizations/'):
|
||||
org_id = role.split('/')[1]
|
||||
role = role.replace(org_id, _org_id(org_id))
|
||||
members = attrs['members'] if authoritative else [attrs['member']]
|
||||
if resource_type == 'folder' and folders:
|
||||
resource_id = folders.get(resource_id, resource_id)
|
||||
for member in members:
|
||||
member_type, _, member_id = member.partition(':')
|
||||
if member_type == 'user':
|
||||
continue
|
||||
member_id = member_id.rpartition('@')[0]
|
||||
if prefix and member_id.startswith(prefix):
|
||||
member_id = member_id[len(prefix) + 1:]
|
||||
yield Binding(authoritative, resource_type, resource_id, role,
|
||||
member_type, member_id, conditions)
|
||||
|
||||
|
||||
def get_folders(resources):
|
||||
'Parse resources and return folder id, name tuples.'
|
||||
for r in resources:
|
||||
if r['type'] != 'google_folder':
|
||||
continue
|
||||
for i in r['instances']:
|
||||
yield i['attributes']['id'], i['attributes']['display_name']
|
||||
|
||||
|
||||
def output_csv(bindings):
|
||||
'Output bindings in CSV format.'
|
||||
print(','.join(FIELDS))
|
||||
for b in bindings:
|
||||
print(','.join(str(getattr(b, f)) for f in FIELDS))
|
||||
|
||||
|
||||
def output_principals(bindings):
|
||||
'Output bindings in Markdown format by principals.'
|
||||
resource_grouper = itertools.groupby(
|
||||
bindings, key=lambda b: (b.resource_type, b.resource_id))
|
||||
print('# IAM bindings reference')
|
||||
print('\nLegend: <code>+</code> additive, <code>•</code> conditional.')
|
||||
for resource, resource_groups in resource_grouper:
|
||||
print(f'\n## {resource[0].title()} <i>{resource[1].lower()}</i>\n')
|
||||
principal_grouper = itertools.groupby(
|
||||
resource_groups, key=lambda b: (b.member_type, b.member_id))
|
||||
print('| members | roles |')
|
||||
print('|---|---|')
|
||||
for principal, principal_groups in principal_grouper:
|
||||
roles = []
|
||||
for b in principal_groups:
|
||||
additive = '<code>+</code>' if not b.authoritative else ''
|
||||
conditions = '<code>•</code>' if b.conditions else ''
|
||||
if b.role.startswith('organizations/'):
|
||||
roles.append(f'{b.role} {additive}{conditions}')
|
||||
else:
|
||||
url = (
|
||||
'https://cloud.google.com/iam/docs/understanding-roles#'
|
||||
f'{b.role.replace("roles/", "")}'
|
||||
)
|
||||
roles.append(f'[{b.role}]({url}) {additive}{conditions}')
|
||||
print((
|
||||
f'|<b>{principal[1]}</b><br><small><i>{principal[0]}</i></small>|'
|
||||
f'{"<br>".join(roles)}|'
|
||||
))
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('state-file', type=click.File('r'), default=sys.stdin)
|
||||
@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']), default='raw')
|
||||
@click.option('--prefix', default=None)
|
||||
def main(state_file, format, prefix=None):
|
||||
'Output IAM bindings parsed from Terraform state file or standard input.'
|
||||
with state_file:
|
||||
data = json.load(state_file)
|
||||
resources = data.get('resources', [])
|
||||
folders = dict(get_folders(resources))
|
||||
bindings = get_bindings(resources, prefix=prefix, folders=folders)
|
||||
bindings = sorted(bindings, key=lambda b: (
|
||||
RESOURCE_SORT.get(b.resource_type, 99), b.resource_id,
|
||||
b.member_type, b.member_id))
|
||||
if format == 'raw':
|
||||
for b in bindings:
|
||||
print(b)
|
||||
else:
|
||||
func = globals().get(f'output_{format}')
|
||||
if not func:
|
||||
raise SystemExit('Unknown format.')
|
||||
func(bindings)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue