(WIP) Read-only service accounts for automation and CI/CD (#1899)

* add design doc for the new CI/CD sa

* describe the actual implementation

* specify which files will need to be changed

* Update 0-cicd-plan-sa.md

* Update 0-cicd-plan-sa.md

* Update 0-cicd-plan-sa.md

* Update 0-cicd-plan-sa.md

* Update 0-cicd-plan-sa.md

* Update 0-cicd-plan-sa.md

* Update 0-cicd-plan-sa.md

* Fix typo

* stage 0 read-only service accounts

* stage 0 IAM map

* linting

* cicd read-only service accounts

* tweak workflow templates

* roles and github workflow fixes

* tfdoc

* Ad-hoc custom role factory for FAST bootstrap

* use factory variable for custom roles data path

* custom roles factory in org/project modules

* tfdoc

* rename custom roles factory variable, fix gitlab template

* gitlab workflow fixes

* fix merge

* output plan results on failed assertion

* update stage 0 expected values

* data platform branch

* gke

* networking

* security

* project factory

* outputs

* workflow templates

* resman apply fixes

* tfdoc

* fix stage 1 test fixture

* fix gh workflow

* read-only resman sa roles

* fix test

* read-only resman sa roles

* read-only resman sa roles

* read-only resman sa roles

* read-only resman sa roles

* fix test variables

* rename wif principal attribute names

* rename wif principal variables

* multitenant stages

---------

Co-authored-by: Wiktor Niesiobędzki <wiktorn@google.com>
Co-authored-by: Julio Castillo <jccb@google.com>

fast/assets/templates/workflow-github.yaml

@@ -24,13 +24,13 @@ on:
       - synchronize
 
 env:
-  FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+  FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
-  FAST_SERVICE_ACCOUNT: ${service_account}
+  FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_PROVIDERS_FILE: ${tf_providers_files.apply}
-  TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
+  TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan}
-  TF_VERSION: 1.4.4
+  TF_VERSION: 1.6.5
 
 jobs:
   fast-pr:
@@ -46,52 +46,74 @@ jobs:
         uses: actions/checkout@v3
 
       # set up SSH key authentication to the modules repository
+
       - id: ssh-config
         name: Configure SSH authentication
         run: |
           ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
           ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
 
-      # set up authentication via Workload identity Federation
+      # set up step variables for plan / apply
+
+      - id: vars-plan
+        if: github.event.pull_request.merged != true && success()
+        name: Set up plan variables
+        run: |
+          echo "plan_opts=-lock=false" >> "$GITHUB_ENV"
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE_PLAN}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT_PLAN}}" >> "$GITHUB_ENV"
+
+      - id: vars-apply
+        if: github.event.pull_request.merged == true && success()
+        name: Set up apply variables
+        run: |
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT}}" >> "$GITHUB_ENV"
+
+      # set up authentication via Workload identity Federation and gcloud
+
       - id: gcp-auth
         name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v0
+        uses: google-github-actions/auth@v2
         with:
-          workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+          workload_identity_provider: $${{env.FAST_WIF_PROVIDER}}
-          service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+          service_account: $${{env.service_account}}
-          access_token_lifetime: 3600s
+          access_token_lifetime: 900s
 
       - id: gcp-sdk
         name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         with:
           install_components: alpha
 
-      # copy provider and tfvars files
+      # copy provider file
-      - id: tf-config
+
-        name: Copy Terraform output files
+      - id: tf-config-provider
+        name: Copy Terraform provider file
         run: |
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+            "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./
+          %{~ for f in tf_var_files ~}
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+            "gs://${outputs_bucket}/tfvars/${f}" ./
-          for f in $${{env.TF_VAR_FILES}}; do
+          %{~ endfor ~}
-            ln -s "tfvars/$f" ./
-          done
 
       - id: tf-setup
         name: Set up Terraform
         uses: hashicorp/setup-terraform@v2.0.3
         with:
-          terraform_version: $${{ env.TF_VERSION }}
+          terraform_version: $${{env.TF_VERSION}}
 
       # run Terraform init/validate/plan
+
       - id: tf-init
         name: Terraform init
+        continue-on-error: true
         run: |
           terraform init -no-color
 
       - id: tf-validate
+        continue-on-error: true
         name: Terraform validate
         run: terraform validate -no-color
 
@@ -99,7 +121,7 @@ jobs:
         name: Terraform plan
         continue-on-error: true
         run: |
-          terraform plan -input=false -out ../plan.out -no-color
+          terraform plan -input=false -out ../plan.out -no-color $${{env.plan_opts}}
 
       - id: tf-apply
         if: github.event.pull_request.merged == true && success()
@@ -108,28 +130,31 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -no-color ../plan.out
 
+      # PR comment with Terraform result from previous steps
+      # length is checked and trimmed for length so as to stay within the limit
+
       - id: pr-comment
         name: Post comment to Pull Request
         continue-on-error: true
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request'
         env:
-          PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+          PLAN: $${{steps.tf-plan.outputs.stdout}}\n$${{steps.tf-plan.outputs.stderr}}
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
             <details><summary>Validation Output</summary>
 
             \`\`\`\n
-            $${{ steps.tf-validate.outputs.stdout }}
+            $${{steps.tf-validate.outputs.stdout}}
             \`\`\`
 
             </details>
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             <details><summary>Show Plan</summary>
 
@@ -139,9 +164,9 @@ jobs:
 
             </details>
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -151,22 +176,22 @@ jobs:
             })
 
       - id: pr-short-comment
-        name: Post comment to Pull Request
+        name: Post comment to Pull Request (abbreviated)
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request' && steps.pr-comment.outcome != 'success'
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             Plan output is in the action log.
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -175,6 +200,18 @@ jobs:
               body: output
             })
 
+      # exit on error from previous steps
+
+      - id: check-init
+        name: Check init failure
+        if: steps.tf-init.outcome != 'success'
+        run: exit 1
+
+      - id: check-validate
+        name: Check validate failure
+        if: steps.tf-validate.outcome != 'success'
+        run: exit 1
+
       - id: check-plan
         name: Check plan failure
         if: steps.tf-plan.outcome != 'success'

fast/assets/templates/workflow-gitlab.yaml

@@ -12,43 +12,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-default:
-  image:
-    name: hashicorp/terraform
-    entrypoint:
-      - "/usr/bin/env"
-      - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
 variables:
   GOOGLE_CREDENTIALS: cicd-sa-credentials.json
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
-  FAST_SERVICE_ACCOUNT: ${service_account}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  %{~ if tf_providers_file != "" ~}
+  %{~ if tf_var_files != [] ~}
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
   %{~ endif ~}
-  TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
+
+workflow:
+  rules:
+    # merge / apply
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        COMMAND: apply
+        FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
+        TF_PROVIDERS_FILE: 0-bootstrap-providers.tf
+    # pr / plan
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      variables:
+        COMMAND: plan
+        FAST_SERVICE_ACCOUNT: ${service_accounts.plan}
+        TF_PROVIDERS_FILE: 0-bootstrap-r-providers.tf
 
 stages:
-  - gcp-auth
+  - gcp-setup
-  - tf-files
+  - tf-plan-apply
-  - tf-plan
-  - tf-apply
 
-cache:
+# TODO: document project-level deploy key used to fetch modules
-  key: gcp-auth
-  paths:
-    - cicd-sa-credentials.json
-    - token.txt
-    %{~ if tf_providers_file != "" ~}
-    - ${tf_providers_file}
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - ${f}
-    %{~ endfor ~}
 
-gcp-auth:
+gcp-setup:
+  stage: gcp-setup
+  image:
+    name: google/cloud-sdk:slim
+  artifacts:
+    paths:
+      - cicd-sa-credentials.json
+      - providers.tf
+  id_tokens:
+    GITLAB_TOKEN:
+      aud:
+      %{~ for aud in audiences ~}
+        - ${aud}
+      %{~ endfor ~}
+  before_script:
+    - echo "$GITLAB_TOKEN" > token.txt
+  script:
+    - |
+      gcloud iam workload-identity-pools create-cred-config \
+        $FAST_WIF_PROVIDER \
+        --service-account=$FAST_SERVICE_ACCOUNT \
+        --service-account-token-lifetime-seconds=900 \
+        --output-file=$GOOGLE_CREDENTIALS \
+        --credential-source-file=token.txt
+    - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS
+    - gcloud alpha storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf
+
+tf-plan-apply:
+  stage: tf-plan-apply
+  dependencies:
+    - gcp-setup
   id_tokens:
     GITLAB_TOKEN:
       aud:
@@ -56,69 +80,20 @@ gcp-auth:
         - ${aud}
       %{~ endfor ~}
   image:
-    name: google/cloud-sdk:slim
+    name: hashicorp/terraform
-  stage: gcp-auth
+    entrypoint:
+      - "/usr/bin/env"
+  variables:
+    SSH_AUTH_SOCK: /tmp/ssh-agent.sock
   script:
-    - echo "$${GITLAB_TOKEN}" > token.txt
     - |
-      gcloud iam workload-identity-pools create-cred-config \
+      ssh-agent -a $SSH_AUTH_SOCK
-        $${FAST_WIF_PROVIDER} \
+      echo "$CICD_MODULES_KEY" | ssh-add -
-        --service-account=$${FAST_SERVICE_ACCOUNT} \
+      mkdir -p ~/.ssh
-        --service-account-token-lifetime-seconds=3600 \
+      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-        --output-file=$${GOOGLE_CREDENTIALS} \
+      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-        --credential-source-file=token.txt
+    - echo "$GITLAB_TOKEN" > token.txt
-
-tf-files:
-  dependencies:
-    - gcp-auth
-  image:
-    name: google/cloud-sdk:slim
-  stage: tf-files
-  script:
-    # - gcloud components install -q alpha
-    - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS}
-    %{~ if tf_providers_file != "" ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./
-    %{~ endfor ~}
-    - ls -l
-
-tf-plan:
-  dependencies:
-    - tf-files
-  stage: tf-plan
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
     - terraform init
     - terraform validate
-    - terraform plan
+    - "if [ $COMMAND == 'plan' ]; then terraform plan -input=false -no-color -lock=false; fi"
-
+    - "if [ $COMMAND == 'apply' ]; then terraform apply -input=false -no-color -auto-approve; fi"
-tf-apply:
-  dependencies:
-    - tf-files
-  stage: tf-apply
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
-    - terraform init
-    - terraform validate
-    - terraform apply -input=false -auto-approve
-  when: manual
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

fast/docs/0-cicd-plan-sa.md

@@ -0,0 +1,85 @@
+# Add new service accounts for CI/CD with plan-only permissions
+
+**authors:** [Ludo](https://github.com/ludoo) \
+**date:** December 3, 2023
+
+## Status
+
+In development.
+
+## Context
+
+The current CI/CD workflows are inherently insecure, as the same service account is used to run `terraform plan` in PR checks, and `terraform apply` in merges.
+
+The current repository configuration variable allows setting a branch which could be used to only allow using the service account in merges, but that only has the consequence of preventing PR checks to work so it's not working as desired.
+
+## Proposal
+
+The proposal is to create a separate "chain" of less privileged service accounts that can only run `plan`, used only when a repository configuration sets a branch for merges in the `cicd_repositories` variable.
+
+### Use cases
+
+#### Merge branch set in repository configuration
+
+```hcl
+cicd_repositories = {
+  bootstrap = {
+    branch            = "main"
+    identity_provider = "github-example"
+    name              = "example/bootstrap"
+    type              = "github"
+  }
+}
+# tftest skip
+```
+
+When a merge branch is set as in the example above, the CI/CD workflow will have two separate flows:
+
+- for PR checks, the OIDC token will be exchanged with credentials for the `plan`-only CI/CD service account, which can only impersonate the `plan`-only automation service account
+- for merges, the current flow that enables credential exchange and impersonation of the `apply`-enabled service account will be used
+
+#### No merge branch set in repository configuration
+
+```hcl
+cicd_repositories = {
+  bootstrap = {
+    identity_provider = "github-example"
+    name              = "example/bootstrap"
+    type              = "github"
+  }
+}
+# tftest skip
+```
+
+If no merge branch is set in the repository configuration as in the example above, the current behaviour will be preserved allowing exchange and impersonation of the `apply`-enabled service account from any branch.
+
+### Implementation
+
+No changes to variables will be needed other than a lightweight refactor with `optional`.
+
+The following resource changes will need to be implemented:
+
+- define the set of read-only roles for each stage
+- create a new automation service account in each stage and assign the identified roles
+- create a new CI/CD service account with `roles/iam.serviceAccountTokenCreator` on the new automation service account
+- if a merge branch is set in the repository configuration
+  - grant `roles/iam.workloadIdentityUser` on the new CI/CD service account to the `principalSet:` matching any branch
+  - define a new provider file that impersonates the new automation service account and use it in the workflow for checks
+  - keep the existing token exchange via `principal:`, impersonation and provider file for the `apply` part of the workflow only matching the specified merge branch
+- if a branch is not set the current behaviour will be kept
+
+Implementation will modify in stages 0 and 1
+
+- the `automation.tf` files
+- any file where IAM roles are assigned to the automation service account
+- the `cicd-*.tf` files
+- the `templates/workflow-*.yaml` files to implement the new workflow logic
+- the `outputs.tf` files to generate the additional provider files
+
+## Decision
+
+This has been surfaced a while ago and implementation was only pending actual time for development. Development has started.
+
+## Consequences
+
+Existing CI/CD workflows will need to be replaced when a merge branch is already defined in the repository configuration (unlikely to happen as the current workflow would not work).

fast/docs/1-network-ranges.md

@@ -16,6 +16,7 @@ This was not an issue when there were only a few networking stages, but as FAST
 ## Decision
 
 We adopted an IP plan based on regions and environments with the following key points:
+
 - Large ranges for the 3 environments we have out of the box (landing, dev, prod)
 - Support for 2 regions
 - Leave enough space to easily grow either the number of environments or regions
@@ -31,9 +32,10 @@ The following table summarizes the agreed IP plan:
 | Region 2, secondary ranges | 100.80.0.0/12 |                                                      100.80.0.0/14 | 100.84.0.0/16 | 100.88.0.0/14 |
 
 To allocate additional secondary ranges for GKE clusters:
+
 - For the pods range, use the next available /16 in the secondary range of its region/environment pair.
 - For the service range, use the next available /24 in the last /16 of its region/environment pair.
 
 ## Consequences
 
-Default subnets for networking stages were updated to reflect to new ranges.
+Default subnets for networking stages were updated to reflect the new ranges.

fast/stages-multitenant/0-bootstrap-tenant/README.md

@@ -196,7 +196,7 @@ This configuration is possible but unsupported and only exists for development p
 
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L20) | Automation resources created by the organization-level bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket          &#61; string&#10;  project_id              &#61; string&#10;  project_number          &#61; string&#10;  federated_identity_pool &#61; string&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    audiences        &#61; list&#40;string&#41;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_tpl    &#61; string&#10;    principalset_tpl &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
+| [automation](variables.tf#L20) | Automation resources created by the organization-level bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket          &#61; string&#10;  project_id              &#61; string&#10;  project_number          &#61; string&#10;  federated_identity_pool &#61; string&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    audiences        &#61; list&#40;string&#41;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_branch &#61; string&#10;    principal_repo   &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
 | [billing_account](variables.tf#L39) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | <code title="object&#40;&#123;&#10;  id           &#61; string&#10;  is_org_level &#61; optional&#40;bool, true&#41;&#10;  no_iam       &#61; optional&#40;bool, false&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |  |
 | [organization](variables.tf#L214) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
 | [prefix](variables.tf#L230) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>0-bootstrap</code> |

fast/stages-multitenant/0-bootstrap-tenant/cicd.tf

@@ -36,8 +36,8 @@ locals {
         issuer           = local.identity_providers[k].issuer
         issuer_uri       = try(v.oidc[0].issuer_uri, null)
         name             = v.name
-        principal_tpl    = local.identity_providers[k].principal_tpl
+        principal_branch = local.identity_providers[k].principal_branch
-        principalset_tpl = local.identity_providers[k].principalset_tpl
+        principal_repo   = local.identity_providers[k].principal_repo
       }
     }
   )
@@ -118,12 +118,12 @@ module "automation-tf-cicd-sa-bootstrap" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch
@@ -209,12 +209,12 @@ module "automation-tf-cicd-sa-resman" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch

fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf

@@ -35,8 +35,8 @@ locals {
         "attribute.ref"              = "assertion.ref"
       }
       issuer_uri       = "https://token.actions.githubusercontent.com"
-      principal_tpl    = "principal://iam.googleapis.com/%s/subject/repo:%s:ref:refs/heads/%s"
+      principal_branch = "principal://iam.googleapis.com/%s/subject/repo:%s:ref:refs/heads/%s"
-      principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
+      principal_repo   = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
     }
     # https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#token-payload
     gitlab = {
@@ -57,8 +57,8 @@ locals {
         "attribute.ref_type"              = "assertion.ref_type"
       }
       issuer_uri       = "https://gitlab.com"
-      principal_tpl    = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s"
+      principal_branch = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s"
-      principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
+      principal_repo   = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
     }
   }
 }

fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-github.yaml

@@ -24,15 +24,13 @@ on:
       - synchronize
 
 env:
-  FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+  FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
-  FAST_SERVICE_ACCOUNT: ${service_account}
+  FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_PROVIDERS_FILE: ${tf_providers_files.apply}
-  %{~ if tf_var_files != [] ~}
+  TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan}
-  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
+  TF_VERSION: 1.6.5
-  %{~ endif ~}
-  TF_VERSION: 1.5.1
 
 jobs:
   fast-pr:
@@ -48,48 +46,66 @@ jobs:
         uses: actions/checkout@v3
 
       # set up SSH key authentication to the modules repository
+
       - id: ssh-config
         name: Configure SSH authentication
         run: |
           ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
           ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
 
-      # set up authentication via Workload identity Federation
+      # set up step variables for plan / apply
+
+      - id: vars-plan
+        if: github.event.pull_request.merged != true && success()
+        name: Set up plan variables
+        run: |
+          echo "plan_opts=-lock=false" >> "$GITHUB_ENV"
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE_PLAN}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT_PLAN}}" >> "$GITHUB_ENV"
+
+      - id: vars-apply
+        if: github.event.pull_request.merged == true && success()
+        name: Set up apply variables
+        run: |
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT}}" >> "$GITHUB_ENV"
+
+      # set up authentication via Workload identity Federation and gcloud
+
       - id: gcp-auth
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
-          workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+          workload_identity_provider: $${{env.FAST_WIF_PROVIDER}}
-          service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+          service_account: $${{env.service_account}}
-          access_token_lifetime: 3600s
+          access_token_lifetime: 900s
 
       - id: gcp-sdk
         name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         with:
           install_components: alpha
 
-      # copy provider and tfvars files
+      # copy provider file
-      - id: tf-config
+
-        name: Copy Terraform output files
+      - id: tf-config-provider
+        name: Copy Terraform provider file
         run: |
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+            "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./
-          %{~ if tf_var_files != [] ~}
+          %{~ for f in tf_var_files ~}
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+            "gs://${outputs_bucket}/tfvars/${f}" ./
-          for f in $${{env.TF_VAR_FILES}}; do
+          %{~ endfor ~}
-            ln -s "tfvars/$f" ./
-          done
-          %{~ endif ~}
 
       - id: tf-setup
         name: Set up Terraform
         uses: hashicorp/setup-terraform@v2.0.3
         with:
-          terraform_version: $${{ env.TF_VERSION }}
+          terraform_version: $${{env.TF_VERSION}}
 
       # run Terraform init/validate/plan
+
       - id: tf-init
         name: Terraform init
         continue-on-error: true
@@ -105,7 +121,7 @@ jobs:
         name: Terraform plan
         continue-on-error: true
         run: |
-          terraform plan -input=false -out ../plan.out -no-color
+          terraform plan -input=false -out ../plan.out -no-color $${{env.plan_opts}}
 
       - id: tf-apply
         if: github.event.pull_request.merged == true && success()
@@ -114,28 +130,31 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -no-color ../plan.out
 
+      # PR comment with Terraform result from previous steps
+      # length is checked and trimmed for length so as to stay within the limit
+
       - id: pr-comment
         name: Post comment to Pull Request
         continue-on-error: true
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request'
         env:
-          PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+          PLAN: $${{steps.tf-plan.outputs.stdout}}\n$${{steps.tf-plan.outputs.stderr}}
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
             <details><summary>Validation Output</summary>
 
             \`\`\`\n
-            $${{ steps.tf-validate.outputs.stdout }}
+            $${{steps.tf-validate.outputs.stdout}}
             \`\`\`
 
             </details>
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             <details><summary>Show Plan</summary>
 
@@ -145,9 +164,9 @@ jobs:
 
             </details>
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -157,22 +176,22 @@ jobs:
             })
 
       - id: pr-short-comment
-        name: Post comment to Pull Request
+        name: Post comment to Pull Request (abbreviated)
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request' && steps.pr-comment.outcome != 'success'
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             Plan output is in the action log.
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -181,6 +200,8 @@ jobs:
               body: output
             })
 
+      # exit on error from previous steps
+
       - id: check-init
         name: Check init failure
         if: steps.tf-init.outcome != 'success'

fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-gitlab.yaml

@@ -12,43 +12,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-default:
-  image:
-    name: hashicorp/terraform
-    entrypoint:
-      - "/usr/bin/env"
-      - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
 variables:
   GOOGLE_CREDENTIALS: cicd-sa-credentials.json
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
-  FAST_SERVICE_ACCOUNT: ${service_account}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  %{~ if tf_providers_file != "" ~}
+  %{~ if tf_var_files != [] ~}
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
   %{~ endif ~}
-  TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
+
+workflow:
+  rules:
+    # merge / apply
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        COMMAND: apply
+        FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
+        TF_PROVIDERS_FILE: 0-bootstrap-providers.tf
+    # pr / plan
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      variables:
+        COMMAND: plan
+        FAST_SERVICE_ACCOUNT: ${service_accounts.plan}
+        TF_PROVIDERS_FILE: 0-bootstrap-r-providers.tf
 
 stages:
-  - gcp-auth
+  - gcp-setup
-  - tf-files
+  - tf-plan-apply
-  - tf-plan
-  - tf-apply
 
-cache:
+# TODO: document project-level deploy key used to fetch modules
-  key: gcp-auth
-  paths:
-    - cicd-sa-credentials.json
-    - token.txt
-    %{~ if tf_providers_file != "" ~}
-    - ${tf_providers_file}
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - ${f}
-    %{~ endfor ~}
 
-gcp-auth:
+gcp-setup:
+  stage: gcp-setup
+  image:
+    name: google/cloud-sdk:slim
+  artifacts:
+    paths:
+      - cicd-sa-credentials.json
+      - providers.tf
+  id_tokens:
+    GITLAB_TOKEN:
+      aud:
+      %{~ for aud in audiences ~}
+        - ${aud}
+      %{~ endfor ~}
+  before_script:
+    - echo "$GITLAB_TOKEN" > token.txt
+  script:
+    - |
+      gcloud iam workload-identity-pools create-cred-config \
+        $FAST_WIF_PROVIDER \
+        --service-account=$FAST_SERVICE_ACCOUNT \
+        --service-account-token-lifetime-seconds=900 \
+        --output-file=$GOOGLE_CREDENTIALS \
+        --credential-source-file=token.txt
+    - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS
+    - gcloud alpha storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf
+
+tf-plan-apply:
+  stage: tf-plan-apply
+  dependencies:
+    - gcp-setup
   id_tokens:
     GITLAB_TOKEN:
       aud:
@@ -56,69 +80,20 @@ gcp-auth:
         - ${aud}
       %{~ endfor ~}
   image:
-    name: google/cloud-sdk:slim
+    name: hashicorp/terraform
-  stage: gcp-auth
+    entrypoint:
+      - "/usr/bin/env"
+  variables:
+    SSH_AUTH_SOCK: /tmp/ssh-agent.sock
   script:
-    - echo "$${GITLAB_TOKEN}" > token.txt
     - |
-      gcloud iam workload-identity-pools create-cred-config \
+      ssh-agent -a $SSH_AUTH_SOCK
-        $${FAST_WIF_PROVIDER} \
+      echo "$CICD_MODULES_KEY" | ssh-add -
-        --service-account=$${FAST_SERVICE_ACCOUNT} \
+      mkdir -p ~/.ssh
-        --service-account-token-lifetime-seconds=3600 \
+      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-        --output-file=$${GOOGLE_CREDENTIALS} \
+      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-        --credential-source-file=token.txt
+    - echo "$GITLAB_TOKEN" > token.txt
-
-tf-files:
-  dependencies:
-    - gcp-auth
-  image:
-    name: google/cloud-sdk:slim
-  stage: tf-files
-  script:
-    # - gcloud components install -q alpha
-    - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS}
-    %{~ if tf_providers_file != "" ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./
-    %{~ endfor ~}
-    - ls -l
-
-tf-plan:
-  dependencies:
-    - tf-files
-  stage: tf-plan
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
     - terraform init
     - terraform validate
-    - terraform plan
+    - "if [ $COMMAND == 'plan' ]; then terraform plan -input=false -no-color -lock=false; fi"
-
+    - "if [ $COMMAND == 'apply' ]; then terraform apply -input=false -no-color -auto-approve; fi"
-tf-apply:
-  dependencies:
-    - tf-files
-  stage: tf-apply
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
-    - terraform init
-    - terraform validate
-    - terraform apply -input=false -auto-approve
-  when: manual
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

fast/stages-multitenant/0-bootstrap-tenant/variables.tf

@@ -30,8 +30,8 @@ variable "automation" {
       issuer           = string
       issuer_uri       = string
       name             = string
-      principal_tpl    = string
+      principal_branch = string
-      principalset_tpl = string
+      principal_repo   = string
     }))
   })
 }

fast/stages-multitenant/1-resman-tenant/README.md

@@ -153,7 +153,7 @@ Once the configuration is done just go through the usual `init/apply` cycle. On
 
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket           &#61; string&#10;  project_id               &#61; string&#10;  project_number           &#61; string&#10;  federated_identity_pools &#61; list&#40;string&#41;&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    audiences        &#61; list&#40;string&#41;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_tpl    &#61; string&#10;    principalset_tpl &#61; string&#10;  &#125;&#41;&#41;&#10;  service_accounts &#61; object&#40;&#123;&#10;    networking &#61; string&#10;    resman     &#61; string&#10;    security   &#61; string&#10;    dp-dev     &#61; optional&#40;string&#41;&#10;    dp-prod    &#61; optional&#40;string&#41;&#10;    gke-dev    &#61; optional&#40;string&#41;&#10;    gke-prod   &#61; optional&#40;string&#41;&#10;    pf-dev     &#61; optional&#40;string&#41;&#10;    pf-prod    &#61; optional&#40;string&#41;&#10;    sandbox    &#61; optional&#40;string&#41;&#10;    teams      &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
+| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket           &#61; string&#10;  project_id               &#61; string&#10;  project_number           &#61; string&#10;  federated_identity_pools &#61; list&#40;string&#41;&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    audiences        &#61; list&#40;string&#41;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_branch &#61; string&#10;    principal_repo   &#61; string&#10;  &#125;&#41;&#41;&#10;  service_accounts &#61; object&#40;&#123;&#10;    networking &#61; string&#10;    resman     &#61; string&#10;    security   &#61; string&#10;    dp-dev     &#61; optional&#40;string&#41;&#10;    dp-prod    &#61; optional&#40;string&#41;&#10;    gke-dev    &#61; optional&#40;string&#41;&#10;    gke-prod   &#61; optional&#40;string&#41;&#10;    pf-dev     &#61; optional&#40;string&#41;&#10;    pf-prod    &#61; optional&#40;string&#41;&#10;    sandbox    &#61; optional&#40;string&#41;&#10;    teams      &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
 | [billing_account](variables.tf#L52) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | <code title="object&#40;&#123;&#10;  id           &#61; string&#10;  is_org_level &#61; optional&#40;bool, true&#41;&#10;  no_iam       &#61; optional&#40;bool, false&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
 | [organization](variables.tf#L214) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
 | [prefix](variables.tf#L230) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>0-bootstrap</code> |

fast/stages-multitenant/1-resman-tenant/cicd-data-platform.tf

@@ -108,12 +108,12 @@ module "branch-dp-dev-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch
@@ -151,12 +151,12 @@ module "branch-dp-prod-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch

fast/stages-multitenant/1-resman-tenant/cicd-gke.tf

@@ -110,12 +110,12 @@ module "branch-gke-dev-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch
@@ -153,12 +153,12 @@ module "branch-gke-prod-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch

fast/stages-multitenant/1-resman-tenant/cicd-networking.tf

@@ -72,12 +72,12 @@ module "branch-network-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch

fast/stages-multitenant/1-resman-tenant/cicd-project-factory.tf

@@ -121,12 +121,12 @@ module "branch-pf-dev-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch
@@ -169,12 +169,12 @@ module "branch-pf-prod-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch

fast/stages-multitenant/1-resman-tenant/cicd-security.tf

@@ -72,12 +72,12 @@ module "branch-security-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_repo,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name
         )
         : format(
-          local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+          local.cicd_identity_providers[each.value.identity_provider].principal_branch,
           local.cicd_identity_pools[each.value.identity_provider],
           each.value.name,
           each.value.branch

fast/stages-multitenant/1-resman-tenant/templates/workflow-github.yaml

@@ -24,15 +24,13 @@ on:
       - synchronize
 
 env:
-  FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+  FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
-  FAST_SERVICE_ACCOUNT: ${service_account}
+  FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_PROVIDERS_FILE: ${tf_providers_files.apply}
-  %{~ if tf_var_files != [] ~}
+  TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan}
-  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
+  TF_VERSION: 1.6.5
-  %{~ endif ~}
-  TF_VERSION: 1.5.1
 
 jobs:
   fast-pr:
@@ -48,48 +46,66 @@ jobs:
         uses: actions/checkout@v3
 
       # set up SSH key authentication to the modules repository
+
       - id: ssh-config
         name: Configure SSH authentication
         run: |
           ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
           ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
 
-      # set up authentication via Workload identity Federation
+      # set up step variables for plan / apply
+
+      - id: vars-plan
+        if: github.event.pull_request.merged != true && success()
+        name: Set up plan variables
+        run: |
+          echo "plan_opts=-lock=false" >> "$GITHUB_ENV"
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE_PLAN}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT_PLAN}}" >> "$GITHUB_ENV"
+
+      - id: vars-apply
+        if: github.event.pull_request.merged == true && success()
+        name: Set up apply variables
+        run: |
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT}}" >> "$GITHUB_ENV"
+
+      # set up authentication via Workload identity Federation and gcloud
+
       - id: gcp-auth
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
-          workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+          workload_identity_provider: $${{env.FAST_WIF_PROVIDER}}
-          service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+          service_account: $${{env.service_account}}
-          access_token_lifetime: 3600s
+          access_token_lifetime: 900s
 
       - id: gcp-sdk
         name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         with:
           install_components: alpha
 
-      # copy provider and tfvars files
+      # copy provider file
-      - id: tf-config
+
-        name: Copy Terraform output files
+      - id: tf-config-provider
+        name: Copy Terraform provider file
         run: |
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+            "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./
-          %{~ if tf_var_files != [] ~}
+          %{~ for f in tf_var_files ~}
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+            "gs://${outputs_bucket}/tfvars/${f}" ./
-          for f in $${{env.TF_VAR_FILES}}; do
+          %{~ endfor ~}
-            ln -s "tfvars/$f" ./
-          done
-          %{~ endif ~}
 
       - id: tf-setup
         name: Set up Terraform
         uses: hashicorp/setup-terraform@v2.0.3
         with:
-          terraform_version: $${{ env.TF_VERSION }}
+          terraform_version: $${{env.TF_VERSION}}
 
       # run Terraform init/validate/plan
+
       - id: tf-init
         name: Terraform init
         continue-on-error: true
@@ -105,7 +121,7 @@ jobs:
         name: Terraform plan
         continue-on-error: true
         run: |
-          terraform plan -input=false -out ../plan.out -no-color
+          terraform plan -input=false -out ../plan.out -no-color $${{env.plan_opts}}
 
       - id: tf-apply
         if: github.event.pull_request.merged == true && success()
@@ -114,28 +130,31 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -no-color ../plan.out
 
+      # PR comment with Terraform result from previous steps
+      # length is checked and trimmed for length so as to stay within the limit
+
       - id: pr-comment
         name: Post comment to Pull Request
         continue-on-error: true
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request'
         env:
-          PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+          PLAN: $${{steps.tf-plan.outputs.stdout}}\n$${{steps.tf-plan.outputs.stderr}}
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
             <details><summary>Validation Output</summary>
 
             \`\`\`\n
-            $${{ steps.tf-validate.outputs.stdout }}
+            $${{steps.tf-validate.outputs.stdout}}
             \`\`\`
 
             </details>
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             <details><summary>Show Plan</summary>
 
@@ -145,9 +164,9 @@ jobs:
 
             </details>
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -157,22 +176,22 @@ jobs:
             })
 
       - id: pr-short-comment
-        name: Post comment to Pull Request
+        name: Post comment to Pull Request (abbreviated)
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request' && steps.pr-comment.outcome != 'success'
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             Plan output is in the action log.
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -181,6 +200,8 @@ jobs:
               body: output
             })
 
+      # exit on error from previous steps
+
       - id: check-init
         name: Check init failure
         if: steps.tf-init.outcome != 'success'

fast/stages-multitenant/1-resman-tenant/templates/workflow-gitlab.yaml

@@ -12,43 +12,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-default:
-  image:
-    name: hashicorp/terraform
-    entrypoint:
-      - "/usr/bin/env"
-      - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
 variables:
   GOOGLE_CREDENTIALS: cicd-sa-credentials.json
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
-  FAST_SERVICE_ACCOUNT: ${service_account}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  %{~ if tf_providers_file != "" ~}
+  %{~ if tf_var_files != [] ~}
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
   %{~ endif ~}
-  TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
+
+workflow:
+  rules:
+    # merge / apply
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        COMMAND: apply
+        FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
+        TF_PROVIDERS_FILE: 0-bootstrap-providers.tf
+    # pr / plan
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      variables:
+        COMMAND: plan
+        FAST_SERVICE_ACCOUNT: ${service_accounts.plan}
+        TF_PROVIDERS_FILE: 0-bootstrap-r-providers.tf
 
 stages:
-  - gcp-auth
+  - gcp-setup
-  - tf-files
+  - tf-plan-apply
-  - tf-plan
-  - tf-apply
 
-cache:
+# TODO: document project-level deploy key used to fetch modules
-  key: gcp-auth
-  paths:
-    - cicd-sa-credentials.json
-    - token.txt
-    %{~ if tf_providers_file != "" ~}
-    - ${tf_providers_file}
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - ${f}
-    %{~ endfor ~}
 
-gcp-auth:
+gcp-setup:
+  stage: gcp-setup
+  image:
+    name: google/cloud-sdk:slim
+  artifacts:
+    paths:
+      - cicd-sa-credentials.json
+      - providers.tf
+  id_tokens:
+    GITLAB_TOKEN:
+      aud:
+      %{~ for aud in audiences ~}
+        - ${aud}
+      %{~ endfor ~}
+  before_script:
+    - echo "$GITLAB_TOKEN" > token.txt
+  script:
+    - |
+      gcloud iam workload-identity-pools create-cred-config \
+        $FAST_WIF_PROVIDER \
+        --service-account=$FAST_SERVICE_ACCOUNT \
+        --service-account-token-lifetime-seconds=900 \
+        --output-file=$GOOGLE_CREDENTIALS \
+        --credential-source-file=token.txt
+    - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS
+    - gcloud alpha storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf
+
+tf-plan-apply:
+  stage: tf-plan-apply
+  dependencies:
+    - gcp-setup
   id_tokens:
     GITLAB_TOKEN:
       aud:
@@ -56,69 +80,20 @@ gcp-auth:
         - ${aud}
       %{~ endfor ~}
   image:
-    name: google/cloud-sdk:slim
+    name: hashicorp/terraform
-  stage: gcp-auth
+    entrypoint:
+      - "/usr/bin/env"
+  variables:
+    SSH_AUTH_SOCK: /tmp/ssh-agent.sock
   script:
-    - echo "$${GITLAB_TOKEN}" > token.txt
     - |
-      gcloud iam workload-identity-pools create-cred-config \
+      ssh-agent -a $SSH_AUTH_SOCK
-        $${FAST_WIF_PROVIDER} \
+      echo "$CICD_MODULES_KEY" | ssh-add -
-        --service-account=$${FAST_SERVICE_ACCOUNT} \
+      mkdir -p ~/.ssh
-        --service-account-token-lifetime-seconds=3600 \
+      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-        --output-file=$${GOOGLE_CREDENTIALS} \
+      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-        --credential-source-file=token.txt
+    - echo "$GITLAB_TOKEN" > token.txt
-
-tf-files:
-  dependencies:
-    - gcp-auth
-  image:
-    name: google/cloud-sdk:slim
-  stage: tf-files
-  script:
-    # - gcloud components install -q alpha
-    - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS}
-    %{~ if tf_providers_file != "" ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./
-    %{~ endfor ~}
-    - ls -l
-
-tf-plan:
-  dependencies:
-    - tf-files
-  stage: tf-plan
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
     - terraform init
     - terraform validate
-    - terraform plan
+    - "if [ $COMMAND == 'plan' ]; then terraform plan -input=false -no-color -lock=false; fi"
-
+    - "if [ $COMMAND == 'apply' ]; then terraform apply -input=false -no-color -auto-approve; fi"
-tf-apply:
-  dependencies:
-    - tf-files
-  stage: tf-apply
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
-    - terraform init
-    - terraform validate
-    - terraform apply -input=false -auto-approve
-  when: manual
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

fast/stages-multitenant/1-resman-tenant/variables.tf

@@ -30,8 +30,8 @@ variable "automation" {
       issuer           = string
       issuer_uri       = string
       name             = string
-      principal_tpl    = string
+      principal_branch = string
-      principalset_tpl = string
+      principal_repo   = string
     }))
     service_accounts = object({
       networking = string

fast/stages/0-bootstrap/IAM.md

@@ -12,13 +12,16 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 |<b>gcp-organization-admins</b><br><small><i>group</i></small>|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner) <br>[roles/cloudsupport.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.admin) <br>[roles/compute.osAdminLogin](https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin) <br>[roles/compute.osLoginExternalUser](https://cloud.google.com/iam/docs/understanding-roles#compute.osLoginExternalUser) <br>[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) <br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) <br>[roles/resourcemanager.tagAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagAdmin) <br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code><br>[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) <code>+</code>|
 |<b>gcp-security-admins</b><br><small><i>group</i></small>|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner) <br>[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor) <br>[roles/iam.securityReviewer](https://cloud.google.com/iam/docs/understanding-roles#iam.securityReviewer) <br>[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/securitycenter.admin](https://cloud.google.com/iam/docs/understanding-roles#securitycenter.admin) <br>[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) <code>+</code><br>[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) <code>+</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
 |<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) <br>[roles/resourcemanager.projectMover](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectMover) <br>[roles/resourcemanager.tagAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagAdmin) <br>[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) <code>+</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
+|<b>prod-bootstrap-0r</b><br><small><i>serviceAccount</i></small>|organizations/[org_id #0]/roles/organizationAdminViewer <code>+</code><br>[roles/logging.viewer](https://cloud.google.com/iam/docs/understanding-roles#logging.viewer) <br>[roles/resourcemanager.folderViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderViewer) <br>[roles/resourcemanager.tagViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagViewer) <br>[roles/iam.organizationRoleViewer](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleViewer) <code>+</code><br>[roles/orgpolicy.policyViewer](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyViewer) <code>+</code>|
 |<b>prod-resman-0</b><br><small><i>serviceAccount</i></small>|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin) <br>[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin) <br>[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) <br>[roles/resourcemanager.tagAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagAdmin) <br>[roles/resourcemanager.tagUser](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagUser) <br>organizations/[org_id #0]/roles/organizationIamAdmin <code>•</code><br>[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) <code>+</code>|
+|<b>prod-resman-0r</b><br><small><i>serviceAccount</i></small>|[roles/logging.viewer](https://cloud.google.com/iam/docs/understanding-roles#logging.viewer) <br>[roles/resourcemanager.folderViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderViewer) <br>[roles/resourcemanager.tagViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagViewer) <br>[roles/orgpolicy.policyViewer](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyViewer) <code>+</code>|
 
 ## Project <i>prod-audit-logs-0</i>
 
 | members | roles |
 |---|---|
 |<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
+|<b>prod-bootstrap-0r</b><br><small><i>serviceAccount</i></small>|[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
 
 ## Project <i>prod-iac-core-0</i>
 
@@ -28,6 +31,8 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 |<b>gcp-organization-admins</b><br><small><i>group</i></small>|[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) <br>[roles/iam.workloadIdentityPoolAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.workloadIdentityPoolAdmin) |
 |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
 |<b>prod-bootstrap-0</b><br><small><i>serviceAccount</i></small>|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) |
+|<b>prod-bootstrap-0r</b><br><small><i>serviceAccount</i></small>|[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
 |<b>prod-bootstrap-1</b><br><small><i>serviceAccount</i></small>|[roles/logging.logWriter](https://cloud.google.com/iam/docs/understanding-roles#logging.logWriter) <code>+</code>|
 |<b>prod-resman-0</b><br><small><i>serviceAccount</i></small>|[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin) <br>[roles/iam.workloadIdentityPoolAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.workloadIdentityPoolAdmin) <br>[roles/source.admin](https://cloud.google.com/iam/docs/understanding-roles#source.admin) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/resourcemanager.projectIamAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) <code>•</code><br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <code>+</code>|
+|<b>prod-resman-0r</b><br><small><i>serviceAccount</i></small>|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser) <br>[roles/cloudbuild.builds.viewer](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.viewer) <br>[roles/iam.serviceAccountViewer](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountViewer) <br>[roles/iam.workloadIdentityPoolViewer](https://cloud.google.com/iam/docs/understanding-roles#iam.workloadIdentityPoolViewer) <br>[roles/source.reader](https://cloud.google.com/iam/docs/understanding-roles#source.reader) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) <br>[roles/serviceusage.serviceUsageViewer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageViewer) <code>+</code>|
 |<b>prod-resman-1</b><br><small><i>serviceAccount</i></small>|[roles/logging.logWriter](https://cloud.google.com/iam/docs/understanding-roles#logging.logWriter) <code>+</code>|

fast/stages/0-bootstrap/README.md

@@ -616,14 +616,14 @@ The `fast_features` variable consists of 4 toggles:
 
 | name | description | sensitive | consumers |
 |---|---|:---:|---|
-| [automation](outputs.tf#L102) | Automation resources. |  |  |
+| [automation](outputs.tf#L124) | Automation resources. |  |  |
-| [billing_dataset](outputs.tf#L107) | BigQuery dataset prepared for billing export. |  |  |
+| [billing_dataset](outputs.tf#L129) | BigQuery dataset prepared for billing export. |  |  |
-| [cicd_repositories](outputs.tf#L112) | CI/CD repository configurations. |  |  |
+| [cicd_repositories](outputs.tf#L134) | CI/CD repository configurations. |  |  |
-| [custom_roles](outputs.tf#L124) | Organization-level custom roles. |  |  |
+| [custom_roles](outputs.tf#L146) | Organization-level custom roles. |  |  |
-| [federated_identity](outputs.tf#L129) | Workload Identity Federation pool and providers. |  |  |
+| [federated_identity](outputs.tf#L151) | Workload Identity Federation pool and providers. |  |  |
-| [outputs_bucket](outputs.tf#L139) | GCS bucket where generated output files are stored. |  |  |
+| [outputs_bucket](outputs.tf#L161) | GCS bucket where generated output files are stored. |  |  |
-| [project_ids](outputs.tf#L144) | Projects created by this stage. |  |  |
+| [project_ids](outputs.tf#L166) | Projects created by this stage. |  |  |
-| [providers](outputs.tf#L154) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
+| [providers](outputs.tf#L176) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
-| [service_accounts](outputs.tf#L161) | Automation service accounts created by this stage. |  |  |
+| [service_accounts](outputs.tf#L183) | Automation service accounts created by this stage. |  |  |
-| [tfvars](outputs.tf#L170) | Terraform variable files for the following stages. | ✓ |  |
+| [tfvars](outputs.tf#L192) | Terraform variable files for the following stages. | ✓ |  |
 <!-- END TFDOC -->

fast/stages/0-bootstrap/automation.tf

@@ -17,7 +17,8 @@
 # tfdoc:file:description Automation project and resources.
 
 locals {
-  cicd_resman_sa = try(module.automation-tf-cicd-sa["resman"].iam_email, "")
+  cicd_resman_sa   = try(module.automation-tf-cicd-sa["resman"].iam_email, "")
+  cicd_resman_r_sa = try(module.automation-tf-cicd-r-sa["resman"].iam_email, "")
 }
 
 module "automation-project" {
@@ -41,24 +42,47 @@ module "automation-project" {
   }
   # machine (service accounts) IAM bindings
   iam = {
+    "roles/browser" = [
+      module.automation-tf-resman-r-sa.iam_email
+    ]
     "roles/owner" = [
       module.automation-tf-bootstrap-sa.iam_email
     ]
     "roles/cloudbuild.builds.editor" = [
       module.automation-tf-resman-sa.iam_email
     ]
+    "roles/cloudbuild.builds.viewer" = [
+      module.automation-tf-resman-r-sa.iam_email
+    ]
     "roles/iam.serviceAccountAdmin" = [
       module.automation-tf-resman-sa.iam_email
     ]
+    "roles/iam.serviceAccountViewer" = [
+      module.automation-tf-resman-r-sa.iam_email
+    ]
     "roles/iam.workloadIdentityPoolAdmin" = [
       module.automation-tf-resman-sa.iam_email
     ]
+    "roles/iam.workloadIdentityPoolViewer" = [
+      module.automation-tf-resman-r-sa.iam_email
+    ]
     "roles/source.admin" = [
       module.automation-tf-resman-sa.iam_email
     ]
+    "roles/source.reader" = [
+      module.automation-tf-resman-r-sa.iam_email
+    ]
     "roles/storage.admin" = [
       module.automation-tf-resman-sa.iam_email
     ]
+    (module.organization.custom_role_id["storage_viewer"]) = [
+      module.automation-tf-bootstrap-r-sa.iam_email,
+      module.automation-tf-resman-r-sa.iam_email
+    ]
+    "roles/viewer" = [
+      module.automation-tf-bootstrap-r-sa.iam_email,
+      module.automation-tf-resman-r-sa.iam_email
+    ]
   }
   iam_bindings = {
     delegated_grants_resman = {
@@ -79,6 +103,10 @@ module "automation-project" {
       member = module.automation-tf-resman-sa.iam_email
       role   = "roles/serviceusage.serviceUsageConsumer"
     }
+    serviceusage_resman_r = {
+      member = module.automation-tf-resman-r-sa.iam_email
+      role   = "roles/serviceusage.serviceUsageViewer"
+    }
   }
   services = [
     "accesscontextmanager.googleapis.com",
@@ -151,6 +179,31 @@ module "automation-tf-bootstrap-sa" {
   }
 }
 
+module "automation-tf-bootstrap-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  project_id   = module.automation-project.project_id
+  name         = "bootstrap-0r"
+  display_name = "Terraform organization bootstrap service account (read-only)."
+  prefix       = local.prefix
+  # allow SA used by CI/CD workflow to impersonate this SA
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.automation-tf-cicd-r-sa["bootstrap"].iam_email, null)
+    ])
+  }
+  # we grant organization roles here as IAM bindings have precedence over
+  # custom roles in the organization module, so these need to depend on it
+  iam_organization_roles = {
+    (var.organization.id) = [
+      module.organization.custom_role_id["organization_admin_viewer"],
+      module.organization.custom_role_id["tag_viewer"]
+    ]
+  }
+  iam_storage_roles = {
+    (module.automation-tf-output-gcs.name) = [module.organization.custom_role_id["storage_viewer"]]
+  }
+}
+
 # resource hierarchy stage's bucket and service account
 
 module "automation-tf-resman-gcs" {
@@ -162,7 +215,8 @@ module "automation-tf-resman-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.automation-tf-resman-sa.iam_email]
+    "roles/storage.objectAdmin"  = [module.automation-tf-resman-sa.iam_email]
+    "roles/storage.objectViewer" = [module.automation-tf-resman-r-sa.iam_email]
   }
   depends_on = [module.organization]
 }
@@ -187,3 +241,32 @@ module "automation-tf-resman-sa" {
     (module.automation-tf-output-gcs.name) = ["roles/storage.admin"]
   }
 }
+
+module "automation-tf-resman-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  project_id   = module.automation-project.project_id
+  name         = "resman-0r"
+  display_name = "Terraform stage 1 resman service account (read-only)."
+  prefix       = local.prefix
+  # allow SA used by CI/CD workflow to impersonate this SA
+  # we use additive IAM to allow tenant CI/CD SAs to impersonate it
+  iam_bindings_additive = (
+    local.cicd_resman_r_sa == "" ? {} : {
+      cicd_token_creator = {
+        member = local.cicd_resman_r_sa
+        role   = "roles/iam.serviceAccountTokenCreator"
+      }
+    }
+  )
+  # we grant organization roles here as IAM bindings have precedence over
+  # custom roles in the organization module, so these need to depend on it
+  iam_organization_roles = {
+    (var.organization.id) = [
+      module.organization.custom_role_id["organization_admin_viewer"],
+      module.organization.custom_role_id["tag_viewer"]
+    ]
+  }
+  iam_storage_roles = {
+    (module.automation-tf-output-gcs.name) = [module.organization.custom_role_id["storage_viewer"]]
+  }
+}

fast/stages/0-bootstrap/billing.tf

@@ -24,6 +24,10 @@ locals {
     module.automation-tf-bootstrap-sa.iam_email,
     module.automation-tf-resman-sa.iam_email
   ]
+  billing_ext_viewers = [
+    module.automation-tf-bootstrap-r-sa.iam_email,
+    module.automation-tf-resman-r-sa.iam_email
+  ]
   billing_mode = (
     var.billing_account.no_iam
     ? null
@@ -43,7 +47,8 @@ module "billing-export-project" {
   )
   prefix = local.prefix
   iam = {
-    "roles/owner" = [module.automation-tf-bootstrap-sa.iam_email]
+    "roles/owner"  = [module.automation-tf-bootstrap-sa.iam_email]
+    "roles/viewer" = [module.automation-tf-bootstrap-r-sa.iam_email]
   }
   services = [
     # "cloudresourcemanager.googleapis.com",
@@ -74,3 +79,12 @@ resource "google_billing_account_iam_member" "billing_ext_admin" {
   role               = "roles/billing.admin"
   member             = each.key
 }
+
+resource "google_billing_account_iam_member" "billing_ext_viewer" {
+  for_each = toset(
+    local.billing_mode == "resource" ? local.billing_ext_viewers : []
+  )
+  billing_account_id = var.billing_account.id
+  role               = "roles/billing.viewer"
+  member             = each.key
+}

fast/stages/0-bootstrap/cicd.tf

@@ -27,8 +27,8 @@ locals {
       issuer           = local.identity_providers[k].issuer
       issuer_uri       = try(v.oidc[0].issuer_uri, null)
       name             = v.name
-      principal_tpl    = local.identity_providers[k].principal_tpl
+      principal_branch = local.identity_providers[k].principal_branch
-      principalset_tpl = local.identity_providers[k].principalset_tpl
+      principal_repo   = local.identity_providers[k].principal_repo
     }
   }
   cicd_repositories = {
@@ -51,8 +51,10 @@ locals {
     )
   }
   cicd_workflow_providers = {
-    bootstrap = "0-bootstrap-providers.tf"
+    bootstrap   = "0-bootstrap-providers.tf"
-    resman    = "1-resman-providers.tf"
+    bootstrap_r = "0-bootstrap-r-providers.tf"
+    resman      = "1-resman-providers.tf"
+    resman_r    = "1-resman-r-providers.tf"
   }
   cicd_workflow_var_files = {
     bootstrap = []
@@ -78,9 +80,12 @@ module "automation-tf-cicd-repo" {
       ? module.automation-tf-bootstrap-sa.iam_email
       : module.automation-tf-resman-sa.iam_email
     ]
-    "roles/source.reader" = [
+    "roles/source.reader" = concat(
-      module.automation-tf-cicd-sa[each.key].iam_email
+      [module.automation-tf-cicd-sa[each.key].iam_email],
-    ]
+      each.key == "bootstrap"
+      ? module.automation-tf-bootstrap-r-sa.iam_email
+      : module.automation-tf-resman-r-sa.iam_email
+    )
   }
   triggers = {
     "fast-0-${each.key}" = {
@@ -116,12 +121,12 @@ module "automation-tf-cicd-sa" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers_defs[each.value.type].principalset_tpl,
+          local.identity_providers_defs[each.value.type].principal_repo,
           google_iam_workload_identity_pool.default.0.name,
           each.value.name
         )
         : format(
-          local.identity_providers_defs[each.value.type].principal_tpl,
+          local.identity_providers_defs[each.value.type].principal_branch,
           google_iam_workload_identity_pool.default.0.name,
           each.value.name,
           each.value.branch
@@ -136,3 +141,33 @@ module "automation-tf-cicd-sa" {
     (module.automation-tf-output-gcs.name) = ["roles/storage.objectViewer"]
   }
 }
+
+module "automation-tf-cicd-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  for_each     = local.cicd_repositories
+  project_id   = module.automation-project.project_id
+  name         = "${each.key}-1r"
+  display_name = "Terraform CI/CD ${each.key} service account (read-only)."
+  prefix       = local.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers_defs[each.value.type].principal_repo,
+          google_iam_workload_identity_pool.default.0.name,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (module.automation-project.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (module.automation-tf-output-gcs.name) = ["roles/storage.objectViewer"]
+  }
+}

fast/stages/0-bootstrap/data/custom-roles/organization_admin_viewer.yaml

@@ -0,0 +1,30 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this is used by the plan-only admin SA
+name: organizationAdminViewer
+includedPermissions:
+  - essentialcontacts.contacts.get
+  - essentialcontacts.contacts.list
+  - orgpolicy.constraints.list
+  - orgpolicy.policies.list
+  - orgpolicy.policy.get
+  - resourcemanager.folders.get
+  - resourcemanager.folders.getIamPolicy
+  - resourcemanager.folders.list
+  - resourcemanager.organizations.get
+  - resourcemanager.organizations.getIamPolicy
+  - resourcemanager.projects.get
+  - resourcemanager.projects.getIamPolicy
+  - resourcemanager.projects.list

fast/stages/0-bootstrap/data/custom-roles/storage_viewer.yaml

@@ -0,0 +1,32 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# the following permissions are a descoped version of storage.admin
+name: storageViewer
+includedPermissions:
+  - storage.buckets.get
+  - storage.buckets.getIamPolicy
+  - storage.buckets.getObjectInsights
+  - storage.buckets.list
+  - storage.buckets.listEffectiveTags
+  - storage.buckets.listTagBindings
+  - storage.managedFolders.get
+  - storage.managedFolders.getIamPolicy
+  - storage.managedFolders.list
+  - storage.multipartUploads.list
+  - storage.multipartUploads.listParts
+  - storage.objects.create
+  - storage.objects.get
+  - storage.objects.getIamPolicy
+  - storage.objects.list

fast/stages/0-bootstrap/data/custom-roles/tag_viewer.yaml

@@ -0,0 +1,24 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# the following permissions are a descoped version of tagAdm
+name: tagViewer
+includedPermissions:
+  - resourcemanager.tagHolds.list
+  - resourcemanager.tagKeys.get
+  - resourcemanager.tagKeys.getIamPolicy
+  - resourcemanager.tagKeys.list
+  - resourcemanager.tagValues.get
+  - resourcemanager.tagValues.getIamPolicy
+  - resourcemanager.tagValues.list

fast/stages/0-bootstrap/identity-providers.tf

@@ -36,8 +36,8 @@ locals {
         "attribute.fast_sub"         = "\"repo:\" + assertion.repository + \":ref:\" + assertion.ref"
       }
       issuer_uri       = "https://token.actions.githubusercontent.com"
-      principal_tpl    = "principalSet://iam.googleapis.com/%s/attribute.fast_sub/repo:%s:ref:refs/heads/%s"
+      principal_branch = "principalSet://iam.googleapis.com/%s/attribute.fast_sub/repo:%s:ref:refs/heads/%s"
-      principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
+      principal_repo   = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
     }
     # https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#token-payload
     gitlab = {
@@ -58,8 +58,8 @@ locals {
         "attribute.ref_type"              = "assertion.ref_type"
       }
       issuer_uri       = "https://gitlab.com"
-      principal_tpl    = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s"
+      principal_branch = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s"
-      principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
+      principal_repo   = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
     }
   }
 }

fast/stages/0-bootstrap/log-export.tf

@@ -39,7 +39,8 @@ module "log-export-project" {
   prefix          = local.prefix
   billing_account = var.billing_account.id
   iam = {
-    "roles/owner" = [module.automation-tf-bootstrap-sa.iam_email]
+    "roles/owner"  = [module.automation-tf-bootstrap-sa.iam_email]
+    "roles/viewer" = [module.automation-tf-bootstrap-r-sa.iam_email]
   }
   services = [
     # "cloudresourcemanager.googleapis.com",

fast/stages/0-bootstrap/organization-iam.tf

@@ -113,6 +113,23 @@ locals {
         ]
       )
     }
+    (module.automation-tf-bootstrap-r-sa.iam_email) = {
+      authoritative = [
+        "roles/logging.viewer",
+        "roles/resourcemanager.folderViewer",
+        "roles/resourcemanager.tagViewer"
+      ]
+      additive = concat(
+        [
+          # the organizationAdminViewer custom role is granted via the SA module
+          "roles/iam.organizationRoleViewer",
+          "roles/orgpolicy.policyViewer"
+        ],
+        local.billing_mode != "org" ? [] : [
+          "roles/billing.viewer"
+        ]
+      )
+    }
     (module.automation-tf-resman-sa.iam_email) = {
       authoritative = [
         "roles/logging.admin",
@@ -130,6 +147,23 @@ locals {
         ]
       )
     }
+    (module.automation-tf-resman-r-sa.iam_email) = {
+      authoritative = [
+        "roles/logging.viewer",
+        "roles/resourcemanager.folderViewer",
+        "roles/resourcemanager.tagViewer",
+        "roles/serviceusage.serviceUsageViewer"
+      ]
+      additive = concat(
+        [
+          # the organizationAdminViewer custom role is granted via the SA module
+          "roles/orgpolicy.policyViewer"
+        ],
+        local.billing_mode != "org" ? [] : [
+          "roles/billing.viewer"
+        ]
+      )
+    }
   }
   # bootstrap user bindings
   iam_user_bootstrap_bindings = var.bootstrap_user == null ? {} : {

fast/stages/0-bootstrap/organization.tf

@@ -50,6 +50,14 @@ locals {
     var.org_policies_config.constraints.allowed_policy_member_domains
   )
   drs_tag_name = "${var.organization.id}/${var.org_policies_config.tag_name}"
+  fast_custom_roles = [
+    "organization_admin_viewer",
+    "organization_iam_admin",
+    "service_project_network_admin",
+    "storage_viewer",
+    "tag_viewer",
+    "tenant_network_admin",
+  ]
   group_iam = {
     for k, v in local.iam_group_bindings : k => v.authoritative
   }
@@ -69,6 +77,8 @@ locals {
   }
 }
 
+# TODO: add a check block to ensure our custom roles exist in the factory files
+
 module "organization" {
   source          = "../../../modules/organization"
   organization_id = "organizations/${var.organization.id}"

fast/stages/0-bootstrap/outputs.tf

@@ -29,12 +29,16 @@ locals {
           local.cicd_providers[v["identity_provider"]].name, ""
         )
         outputs_bucket = module.automation-tf-output-gcs.name
-        service_account = try(
+        service_accounts = {
-          module.automation-tf-cicd-sa[k].email, ""
+          apply = try(module.automation-tf-cicd-sa[k].email, "")
-        )
+          plan  = try(module.automation-tf-cicd-r-sa[k].email, "")
-        stage_name        = k
+        }
-        tf_providers_file = local.cicd_workflow_providers[k]
+        stage_name = k
-        tf_var_files      = local.cicd_workflow_var_files[k]
+        tf_providers_files = {
+          apply = local.cicd_workflow_providers[k]
+          plan  = local.cicd_workflow_providers["${k}_r"]
+        }
+        tf_var_files = local.cicd_workflow_var_files[k]
       }
     )
   }
@@ -45,12 +49,24 @@ locals {
       name          = "bootstrap"
       sa            = module.automation-tf-bootstrap-sa.email
     })
+    "0-bootstrap-r" = templatefile(local._tpl_providers, {
+      backend_extra = null
+      bucket        = module.automation-tf-bootstrap-gcs.name
+      name          = "bootstrap"
+      sa            = module.automation-tf-bootstrap-r-sa.email
+    })
     "1-resman" = templatefile(local._tpl_providers, {
       backend_extra = null
       bucket        = module.automation-tf-resman-gcs.name
       name          = "resman"
       sa            = module.automation-tf-resman-sa.email
     })
+    "1-resman-r" = templatefile(local._tpl_providers, {
+      backend_extra = null
+      bucket        = module.automation-tf-resman-gcs.name
+      name          = "resman"
+      sa            = module.automation-tf-resman-r-sa.email
+    })
     "0-bootstrap-tenant" = templatefile(local._tpl_providers, {
       backend_extra = join("\n", [
         "# remove the newline between quotes and set the tenant name as prefix",
@@ -71,6 +87,12 @@ locals {
       outputs_bucket               = module.automation-tf-output-gcs.name
       project_id                   = module.automation-project.project_id
       project_number               = module.automation-project.number
+      service_accounts = {
+        bootstrap   = module.automation-tf-bootstrap-sa.email
+        bootstrap-r = module.automation-tf-bootstrap-r-sa.email
+        resman      = module.automation-tf-resman-sa.email
+        resman-r    = module.automation-tf-resman-r-sa.email
+      }
     }
     custom_roles = module.organization.custom_role_id
     logging = {

fast/stages/0-bootstrap/templates/workflow-github.yaml

@@ -24,15 +24,13 @@ on:
       - synchronize
 
 env:
-  FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+  FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
-  FAST_SERVICE_ACCOUNT: ${service_account}
+  FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_PROVIDERS_FILE: ${tf_providers_files.apply}
-  %{~ if tf_var_files != [] ~}
+  TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan}
-  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
+  TF_VERSION: 1.6.5
-  %{~ endif ~}
-  TF_VERSION: 1.5.1
 
 jobs:
   fast-pr:
@@ -48,48 +46,66 @@ jobs:
         uses: actions/checkout@v3
 
       # set up SSH key authentication to the modules repository
+
       - id: ssh-config
         name: Configure SSH authentication
         run: |
           ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
           ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
 
-      # set up authentication via Workload identity Federation
+      # set up step variables for plan / apply
+
+      - id: vars-plan
+        if: github.event.pull_request.merged != true && success()
+        name: Set up plan variables
+        run: |
+          echo "plan_opts=-lock=false" >> "$GITHUB_ENV"
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE_PLAN}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT_PLAN}}" >> "$GITHUB_ENV"
+
+      - id: vars-apply
+        if: github.event.pull_request.merged == true && success()
+        name: Set up apply variables
+        run: |
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT}}" >> "$GITHUB_ENV"
+
+      # set up authentication via Workload identity Federation and gcloud
+
       - id: gcp-auth
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
-          workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+          workload_identity_provider: $${{env.FAST_WIF_PROVIDER}}
-          service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+          service_account: $${{env.service_account}}
-          access_token_lifetime: 3600s
+          access_token_lifetime: 900s
 
       - id: gcp-sdk
         name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         with:
           install_components: alpha
 
-      # copy provider and tfvars files
+      # copy provider file
-      - id: tf-config
+
-        name: Copy Terraform output files
+      - id: tf-config-provider
+        name: Copy Terraform provider file
         run: |
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+            "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./
-          %{~ if tf_var_files != [] ~}
+          %{~ for f in tf_var_files ~}
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+            "gs://${outputs_bucket}/tfvars/${f}" ./
-          for f in $${{env.TF_VAR_FILES}}; do
+          %{~ endfor ~}
-            ln -s "tfvars/$f" ./
-          done
-          %{~ endif ~}
 
       - id: tf-setup
         name: Set up Terraform
         uses: hashicorp/setup-terraform@v2.0.3
         with:
-          terraform_version: $${{ env.TF_VERSION }}
+          terraform_version: $${{env.TF_VERSION}}
 
       # run Terraform init/validate/plan
+
       - id: tf-init
         name: Terraform init
         continue-on-error: true
@@ -105,7 +121,7 @@ jobs:
         name: Terraform plan
         continue-on-error: true
         run: |
-          terraform plan -input=false -out ../plan.out -no-color
+          terraform plan -input=false -out ../plan.out -no-color $${{env.plan_opts}}
 
       - id: tf-apply
         if: github.event.pull_request.merged == true && success()
@@ -114,28 +130,31 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -no-color ../plan.out
 
+      # PR comment with Terraform result from previous steps
+      # length is checked and trimmed for length so as to stay within the limit
+
       - id: pr-comment
         name: Post comment to Pull Request
         continue-on-error: true
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request'
         env:
-          PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+          PLAN: $${{steps.tf-plan.outputs.stdout}}\n$${{steps.tf-plan.outputs.stderr}}
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
             <details><summary>Validation Output</summary>
 
             \`\`\`\n
-            $${{ steps.tf-validate.outputs.stdout }}
+            $${{steps.tf-validate.outputs.stdout}}
             \`\`\`
 
             </details>
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             <details><summary>Show Plan</summary>
 
@@ -145,9 +164,9 @@ jobs:
 
             </details>
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -157,22 +176,22 @@ jobs:
             })
 
       - id: pr-short-comment
-        name: Post comment to Pull Request
+        name: Post comment to Pull Request (abbreviated)
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request' && steps.pr-comment.outcome != 'success'
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             Plan output is in the action log.
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -181,6 +200,8 @@ jobs:
               body: output
             })
 
+      # exit on error from previous steps
+
       - id: check-init
         name: Check init failure
         if: steps.tf-init.outcome != 'success'

fast/stages/0-bootstrap/templates/workflow-gitlab.yaml

@@ -12,43 +12,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-default:
-  image:
-    name: hashicorp/terraform
-    entrypoint:
-      - "/usr/bin/env"
-      - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
 variables:
   GOOGLE_CREDENTIALS: cicd-sa-credentials.json
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
-  FAST_SERVICE_ACCOUNT: ${service_account}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  %{~ if tf_providers_file != "" ~}
+  %{~ if tf_var_files != [] ~}
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
   %{~ endif ~}
-  TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
+
+workflow:
+  rules:
+    # merge / apply
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        COMMAND: apply
+        FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
+        TF_PROVIDERS_FILE: 0-bootstrap-providers.tf
+    # pr / plan
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      variables:
+        COMMAND: plan
+        FAST_SERVICE_ACCOUNT: ${service_accounts.plan}
+        TF_PROVIDERS_FILE: 0-bootstrap-r-providers.tf
 
 stages:
-  - gcp-auth
+  - gcp-setup
-  - tf-files
+  - tf-plan-apply
-  - tf-plan
-  - tf-apply
 
-cache:
+# TODO: document project-level deploy key used to fetch modules
-  key: gcp-auth
-  paths:
-    - cicd-sa-credentials.json
-    - token.txt
-    %{~ if tf_providers_file != "" ~}
-    - ${tf_providers_file}
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - ${f}
-    %{~ endfor ~}
 
-gcp-auth:
+gcp-setup:
+  stage: gcp-setup
+  image:
+    name: google/cloud-sdk:slim
+  artifacts:
+    paths:
+      - cicd-sa-credentials.json
+      - providers.tf
+  id_tokens:
+    GITLAB_TOKEN:
+      aud:
+      %{~ for aud in audiences ~}
+        - ${aud}
+      %{~ endfor ~}
+  before_script:
+    - echo "$GITLAB_TOKEN" > token.txt
+  script:
+    - |
+      gcloud iam workload-identity-pools create-cred-config \
+        $FAST_WIF_PROVIDER \
+        --service-account=$FAST_SERVICE_ACCOUNT \
+        --service-account-token-lifetime-seconds=900 \
+        --output-file=$GOOGLE_CREDENTIALS \
+        --credential-source-file=token.txt
+    - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS
+    - gcloud alpha storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf
+
+tf-plan-apply:
+  stage: tf-plan-apply
+  dependencies:
+    - gcp-setup
   id_tokens:
     GITLAB_TOKEN:
       aud:
@@ -56,69 +80,20 @@ gcp-auth:
         - ${aud}
       %{~ endfor ~}
   image:
-    name: google/cloud-sdk:slim
+    name: hashicorp/terraform
-  stage: gcp-auth
+    entrypoint:
+      - "/usr/bin/env"
+  variables:
+    SSH_AUTH_SOCK: /tmp/ssh-agent.sock
   script:
-    - echo "$${GITLAB_TOKEN}" > token.txt
     - |
-      gcloud iam workload-identity-pools create-cred-config \
+      ssh-agent -a $SSH_AUTH_SOCK
-        $${FAST_WIF_PROVIDER} \
+      echo "$CICD_MODULES_KEY" | ssh-add -
-        --service-account=$${FAST_SERVICE_ACCOUNT} \
+      mkdir -p ~/.ssh
-        --service-account-token-lifetime-seconds=3600 \
+      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-        --output-file=$${GOOGLE_CREDENTIALS} \
+      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-        --credential-source-file=token.txt
+    - echo "$GITLAB_TOKEN" > token.txt
-
-tf-files:
-  dependencies:
-    - gcp-auth
-  image:
-    name: google/cloud-sdk:slim
-  stage: tf-files
-  script:
-    # - gcloud components install -q alpha
-    - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS}
-    %{~ if tf_providers_file != "" ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./
-    %{~ endfor ~}
-    - ls -l
-
-tf-plan:
-  dependencies:
-    - tf-files
-  stage: tf-plan
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
     - terraform init
     - terraform validate
-    - terraform plan
+    - "if [ $COMMAND == 'plan' ]; then terraform plan -input=false -no-color -lock=false; fi"
-
+    - "if [ $COMMAND == 'apply' ]; then terraform apply -input=false -no-color -auto-approve; fi"
-tf-apply:
-  dependencies:
-    - tf-files
-  stage: tf-apply
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
-    - terraform init
-    - terraform validate
-    - terraform apply -input=false -auto-approve
-  when: manual
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

fast/stages/1-resman/README.md

@@ -335,7 +335,7 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
 | [branch-teams.tf](./branch-teams.tf) | Team stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> |  |
 | [branch-tenants.tf](./branch-tenants.tf) | Lightweight tenant resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> · <code>project</code> |  |
 | [cicd-data-platform.tf](./cicd-data-platform.tf) | CI/CD resources for the data platform branch. | <code>iam-service-account</code> · <code>source-repository</code> |  |
-| [cicd-gke.tf](./cicd-gke.tf) | CI/CD resources for the data platform branch. | <code>iam-service-account</code> · <code>source-repository</code> |  |
+| [cicd-gke.tf](./cicd-gke.tf) | CI/CD resources for the GKE multitenant branch. | <code>iam-service-account</code> · <code>source-repository</code> |  |
 | [cicd-networking.tf](./cicd-networking.tf) | CI/CD resources for the networking branch. | <code>iam-service-account</code> · <code>source-repository</code> |  |
 | [cicd-project-factory.tf](./cicd-project-factory.tf) | CI/CD resources for the teams branch. | <code>iam-service-account</code> · <code>source-repository</code> |  |
 | [cicd-security.tf](./cicd-security.tf) | CI/CD resources for the security branch. | <code>iam-service-account</code> · <code>source-repository</code> |  |
@@ -353,36 +353,36 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
 
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket          &#61; string&#10;  project_id              &#61; string&#10;  project_number          &#61; string&#10;  federated_identity_pool &#61; string&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    audiences        &#61; list&#40;string&#41;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_tpl    &#61; string&#10;    principalset_tpl &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
+| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket          &#61; string&#10;  project_id              &#61; string&#10;  project_number          &#61; string&#10;  federated_identity_pool &#61; string&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    audiences        &#61; list&#40;string&#41;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_branch &#61; string&#10;    principal_repo   &#61; string&#10;  &#125;&#41;&#41;&#10;  service_accounts &#61; object&#40;&#123;&#10;    resman-r &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
-| [billing_account](variables.tf#L39) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | <code title="object&#40;&#123;&#10;  id           &#61; string&#10;  is_org_level &#61; optional&#40;bool, true&#41;&#10;  no_iam       &#61; optional&#40;bool, false&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
+| [billing_account](variables.tf#L42) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | <code title="object&#40;&#123;&#10;  id           &#61; string&#10;  is_org_level &#61; optional&#40;bool, true&#41;&#10;  no_iam       &#61; optional&#40;bool, false&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
-| [organization](variables.tf#L198) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
+| [organization](variables.tf#L202) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
-| [prefix](variables.tf#L214) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>0-bootstrap</code> |
+| [prefix](variables.tf#L218) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>0-bootstrap</code> |
-| [cicd_repositories](variables.tf#L50) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10;  data_platform_dev &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  data_platform_prod &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  gke_dev &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  gke_prod &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  networking &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  project_factory_dev &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  project_factory_prod &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  security &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
+| [cicd_repositories](variables.tf#L53) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10;  data_platform_dev &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  data_platform_prod &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  gke_dev &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  gke_prod &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  networking &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  project_factory_dev &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  project_factory_prod &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  security &#61; optional&#40;object&#40;&#123;&#10;    name              &#61; string&#10;    type              &#61; string&#10;    branch            &#61; optional&#40;string&#41;&#10;    identity_provider &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
-| [custom_roles](variables.tf#L132) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10;  service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>0-bootstrap</code> |
+| [custom_roles](variables.tf#L135) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10;  service_project_network_admin &#61; string&#10;  storage_viewer                &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>0-bootstrap</code> |
-| [fast_features](variables.tf#L141) | Selective control for top-level FAST features. | <code title="object&#40;&#123;&#10;  data_platform   &#61; optional&#40;bool, false&#41;&#10;  gke             &#61; optional&#40;bool, false&#41;&#10;  project_factory &#61; optional&#40;bool, false&#41;&#10;  sandbox         &#61; optional&#40;bool, false&#41;&#10;  teams           &#61; optional&#40;bool, false&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> | <code>0-0-bootstrap</code> |
+| [fast_features](variables.tf#L145) | Selective control for top-level FAST features. | <code title="object&#40;&#123;&#10;  data_platform   &#61; optional&#40;bool, false&#41;&#10;  gke             &#61; optional&#40;bool, false&#41;&#10;  project_factory &#61; optional&#40;bool, false&#41;&#10;  sandbox         &#61; optional&#40;bool, false&#41;&#10;  teams           &#61; optional&#40;bool, false&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> | <code>0-0-bootstrap</code> |
-| [groups](variables.tf#L155) | Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed. | <code title="object&#40;&#123;&#10;  gcp-devops          &#61; optional&#40;string&#41;&#10;  gcp-network-admins  &#61; optional&#40;string&#41;&#10;  gcp-security-admins &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> | <code>0-bootstrap</code> |
+| [groups](variables.tf#L159) | Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed. | <code title="object&#40;&#123;&#10;  gcp-devops          &#61; optional&#40;string&#41;&#10;  gcp-network-admins  &#61; optional&#40;string&#41;&#10;  gcp-security-admins &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> | <code>0-bootstrap</code> |
-| [locations](variables.tf#L168) | Optional locations for GCS, BigQuery, and logging buckets created here. | <code title="object&#40;&#123;&#10;  bq      &#61; string&#10;  gcs     &#61; string&#10;  logging &#61; string&#10;  pubsub  &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  bq      &#61; &#34;EU&#34;&#10;  gcs     &#61; &#34;EU&#34;&#10;  logging &#61; &#34;global&#34;&#10;  pubsub  &#61; &#91;&#93;&#10;&#125;">&#123;&#8230;&#125;</code> | <code>0-bootstrap</code> |
+| [locations](variables.tf#L172) | Optional locations for GCS, BigQuery, and logging buckets created here. | <code title="object&#40;&#123;&#10;  bq      &#61; string&#10;  gcs     &#61; string&#10;  logging &#61; string&#10;  pubsub  &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  bq      &#61; &#34;EU&#34;&#10;  gcs     &#61; &#34;EU&#34;&#10;  logging &#61; &#34;global&#34;&#10;  pubsub  &#61; &#91;&#93;&#10;&#125;">&#123;&#8230;&#125;</code> | <code>0-bootstrap</code> |
-| [org_policy_tags](variables.tf#L186) | Resource management tags for organization policy exceptions. | <code title="object&#40;&#123;&#10;  key_id   &#61; optional&#40;string&#41;&#10;  key_name &#61; optional&#40;string&#41;&#10;  values   &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> | <code>0-bootstrap</code> |
+| [org_policy_tags](variables.tf#L190) | Resource management tags for organization policy exceptions. | <code title="object&#40;&#123;&#10;  key_id   &#61; optional&#40;string&#41;&#10;  key_name &#61; optional&#40;string&#41;&#10;  values   &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> | <code>0-bootstrap</code> |
-| [outputs_location](variables.tf#L208) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | <code>string</code> |  | <code>null</code> |  |
+| [outputs_location](variables.tf#L212) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | <code>string</code> |  | <code>null</code> |  |
-| [tag_names](variables.tf#L225) | Customized names for resource management tags. | <code title="object&#40;&#123;&#10;  context     &#61; optional&#40;string, &#34;context&#34;&#41;&#10;  environment &#61; optional&#40;string, &#34;environment&#34;&#41;&#10;  tenant      &#61; optional&#40;string, &#34;tenant&#34;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [tag_names](variables.tf#L229) | Customized names for resource management tags. | <code title="object&#40;&#123;&#10;  context     &#61; optional&#40;string, &#34;context&#34;&#41;&#10;  environment &#61; optional&#40;string, &#34;environment&#34;&#41;&#10;  tenant      &#61; optional&#40;string, &#34;tenant&#34;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |  |
-| [tags](variables.tf#L240) | Custome secure tags by key name. The `iam` attribute behaves like the similarly named one at module level. | <code title="map&#40;object&#40;&#123;&#10;  description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;  iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  values &#61; optional&#40;map&#40;object&#40;&#123;&#10;    description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;    iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;    id          &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [tags](variables.tf#L244) | Custome secure tags by key name. The `iam` attribute behaves like the similarly named one at module level. | <code title="map&#40;object&#40;&#123;&#10;  description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;  iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  values &#61; optional&#40;map&#40;object&#40;&#123;&#10;    description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;    iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;    id          &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
-| [team_folders](variables.tf#L261) | Team folders to be created. Format is described in a code comment. | <code title="map&#40;object&#40;&#123;&#10;  descriptive_name     &#61; string&#10;  group_iam            &#61; map&#40;list&#40;string&#41;&#41;&#10;  impersonation_groups &#61; list&#40;string&#41;&#10;  cicd &#61; optional&#40;object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>null</code> |  |
+| [team_folders](variables.tf#L265) | Team folders to be created. Format is described in a code comment. | <code title="map&#40;object&#40;&#123;&#10;  descriptive_name     &#61; string&#10;  group_iam            &#61; map&#40;list&#40;string&#41;&#41;&#10;  impersonation_groups &#61; list&#40;string&#41;&#10;  cicd &#61; optional&#40;object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>null</code> |  |
-| [tenants](variables.tf#L277) | Lightweight tenant definitions. | <code title="map&#40;object&#40;&#123;&#10;  admin_group_email &#61; string&#10;  descriptive_name  &#61; string&#10;  billing_account   &#61; optional&#40;string&#41;&#10;  organization &#61; optional&#40;object&#40;&#123;&#10;    customer_id &#61; string&#10;    domain      &#61; string&#10;    id          &#61; number&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [tenants](variables.tf#L281) | Lightweight tenant definitions. | <code title="map&#40;object&#40;&#123;&#10;  admin_group_email &#61; string&#10;  descriptive_name  &#61; string&#10;  billing_account   &#61; optional&#40;string&#41;&#10;  organization &#61; optional&#40;object&#40;&#123;&#10;    customer_id &#61; string&#10;    domain      &#61; string&#10;    id          &#61; number&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
-| [tenants_config](variables.tf#L293) | Lightweight tenants shared configuration. Roles will be assigned to tenant admin group and service accounts. | <code title="object&#40;&#123;&#10;  core_folder_roles   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  tenant_folder_roles &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  top_folder_roles    &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [tenants_config](variables.tf#L297) | Lightweight tenants shared configuration. Roles will be assigned to tenant admin group and service accounts. | <code title="object&#40;&#123;&#10;  core_folder_roles   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  tenant_folder_roles &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  top_folder_roles    &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |  |
 
 ## Outputs
 
 | name | description | sensitive | consumers |
 |---|---|:---:|---|
-| [cicd_repositories](outputs.tf#L232) | WIF configuration for CI/CD repositories. |  |  |
+| [cicd_repositories](outputs.tf#L336) | WIF configuration for CI/CD repositories. |  |  |
-| [dataplatform](outputs.tf#L246) | Data for the Data Platform stage. |  |  |
+| [dataplatform](outputs.tf#L350) | Data for the Data Platform stage. |  |  |
-| [gke_multitenant](outputs.tf#L262) | Data for the GKE multitenant stage. |  | <code>03-gke-multitenant</code> |
+| [gke_multitenant](outputs.tf#L366) | Data for the GKE multitenant stage. |  | <code>03-gke-multitenant</code> |
-| [networking](outputs.tf#L283) | Data for the networking stage. |  |  |
+| [networking](outputs.tf#L387) | Data for the networking stage. |  |  |
-| [project_factories](outputs.tf#L292) | Data for the project factories stage. |  |  |
+| [project_factories](outputs.tf#L396) | Data for the project factories stage. |  |  |
-| [providers](outputs.tf#L307) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
+| [providers](outputs.tf#L411) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
-| [sandbox](outputs.tf#L314) | Data for the sandbox stage. |  | <code>xx-sandbox</code> |
+| [sandbox](outputs.tf#L418) | Data for the sandbox stage. |  | <code>xx-sandbox</code> |
-| [security](outputs.tf#L328) | Data for the networking stage. |  | <code>02-security</code> |
+| [security](outputs.tf#L432) | Data for the networking stage. |  | <code>02-security</code> |
-| [team_cicd_repositories](outputs.tf#L338) | WIF configuration for Team CI/CD repositories. |  |  |
+| [team_cicd_repositories](outputs.tf#L442) | WIF configuration for Team CI/CD repositories. |  |  |
-| [teams](outputs.tf#L352) | Data for the teams stage. |  |  |
+| [teams](outputs.tf#L456) | Data for the teams stage. |  |  |
-| [tfvars](outputs.tf#L364) | Terraform variable files for the following stages. | ✓ |  |
+| [tfvars](outputs.tf#L468) | Terraform variable files for the following stages. | ✓ |  |
 <!-- END TFDOC -->

fast/stages/1-resman/branch-data-platform.tf

@@ -34,15 +34,20 @@ module "branch-dp-dev-folder" {
   parent    = module.branch-dp-folder.0.id
   name      = "Development"
   group_iam = {}
+  # owner and viewer roles are broad and might grant unwanted access
+  # replace them with more selective custom roles for production deployments
   iam = {
+    # read-write (apply) automation service account
     (local.custom_roles.service_project_network_admin) = [
       module.branch-dp-dev-sa.0.iam_email
     ]
-    # remove owner here and at project level if SA does not manage project resources
     "roles/owner"                          = [module.branch-dp-dev-sa.0.iam_email]
     "roles/logging.admin"                  = [module.branch-dp-dev-sa.0.iam_email]
     "roles/resourcemanager.folderAdmin"    = [module.branch-dp-dev-sa.0.iam_email]
     "roles/resourcemanager.projectCreator" = [module.branch-dp-dev-sa.0.iam_email]
+    # read-only (plan) automation service account
+    "roles/viewer"                       = [module.branch-dp-dev-r-sa.0.iam_email]
+    "roles/resourcemanager.folderViewer" = [module.branch-dp-dev-r-sa.0.iam_email]
   }
   tag_bindings = {
     context = try(
@@ -58,13 +63,18 @@ module "branch-dp-prod-folder" {
   parent    = module.branch-dp-folder.0.id
   name      = "Production"
   group_iam = {}
+  # owner and viewer roles are broad and might grant unwanted access
+  # replace them with more selective custom roles for production deployments
   iam = {
+    # read-write (apply) automation service account
     (local.custom_roles.service_project_network_admin) = [module.branch-dp-prod-sa.0.iam_email]
-    # remove owner here and at project level if SA does not manage project resources
+    "roles/owner"                                      = [module.branch-dp-prod-sa.0.iam_email]
-    "roles/owner"                          = [module.branch-dp-prod-sa.0.iam_email]
+    "roles/logging.admin"                              = [module.branch-dp-prod-sa.0.iam_email]
-    "roles/logging.admin"                  = [module.branch-dp-prod-sa.0.iam_email]
+    "roles/resourcemanager.folderAdmin"                = [module.branch-dp-prod-sa.0.iam_email]
-    "roles/resourcemanager.folderAdmin"    = [module.branch-dp-prod-sa.0.iam_email]
+    "roles/resourcemanager.projectCreator"             = [module.branch-dp-prod-sa.0.iam_email]
-    "roles/resourcemanager.projectCreator" = [module.branch-dp-prod-sa.0.iam_email]
+    # read-only (plan) automation service account
+    "roles/viewer"                       = [module.branch-dp-prod-r-sa.0.iam_email]
+    "roles/resourcemanager.folderViewer" = [module.branch-dp-prod-r-sa.0.iam_email]
   }
   tag_bindings = {
     context = try(
@@ -74,7 +84,7 @@ module "branch-dp-prod-folder" {
   }
 }
 
-# automation service accounts and buckets
+# automation service accounts
 
 module "branch-dp-dev-sa" {
   source       = "../../../modules/iam-service-account"
@@ -113,6 +123,50 @@ module "branch-dp-prod-sa" {
   }
 }
 
+# automation read-only service accounts
+
+module "branch-dp-dev-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  count        = var.fast_features.data_platform ? 1 : 0
+  project_id   = var.automation.project_id
+  name         = "dev-resman-dp-0r"
+  display_name = "Terraform data platform development service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-dp-dev-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+module "branch-dp-prod-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  count        = var.fast_features.data_platform ? 1 : 0
+  project_id   = var.automation.project_id
+  name         = "prod-resman-dp-0r"
+  display_name = "Terraform data platform production service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-dp-prod-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+# automation buckets
+
 module "branch-dp-dev-gcs" {
   source        = "../../../modules/gcs"
   count         = var.fast_features.data_platform ? 1 : 0
@@ -123,7 +177,8 @@ module "branch-dp-dev-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-dp-dev-sa.0.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-dp-dev-sa.0.iam_email]
+    "roles/storage.objectViewer" = [module.branch-dp-dev-r-sa.0.iam_email]
   }
 }
 
@@ -137,6 +192,7 @@ module "branch-dp-prod-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-dp-prod-sa.0.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-dp-prod-sa.0.iam_email]
+    "roles/storage.objectViewer" = [module.branch-dp-prod-r-sa.0.iam_email]
   }
 }

fast/stages/1-resman/branch-gke.tf

@@ -34,11 +34,15 @@ module "branch-gke-dev-folder" {
   parent = module.branch-gke-folder.0.id
   name   = "Development"
   iam = {
+    # read-write (apply) automation service account
     "roles/owner"                          = [module.branch-gke-dev-sa.0.iam_email]
     "roles/logging.admin"                  = [module.branch-gke-dev-sa.0.iam_email]
     "roles/resourcemanager.folderAdmin"    = [module.branch-gke-dev-sa.0.iam_email]
     "roles/resourcemanager.projectCreator" = [module.branch-gke-dev-sa.0.iam_email]
     "roles/compute.xpnAdmin"               = [module.branch-gke-dev-sa.0.iam_email]
+    # read-only (plan) automation service account
+    "roles/viewer"                       = [module.branch-gke-dev-r-sa.0.iam_email]
+    "roles/resourcemanager.folderViewer" = [module.branch-gke-dev-r-sa.0.iam_email]
   }
   tag_bindings = {
     context = try(
@@ -54,11 +58,15 @@ module "branch-gke-prod-folder" {
   parent = module.branch-gke-folder.0.id
   name   = "Production"
   iam = {
+    # read-write (apply) automation service account
     "roles/owner"                          = [module.branch-gke-prod-sa.0.iam_email]
     "roles/logging.admin"                  = [module.branch-gke-prod-sa.0.iam_email]
     "roles/resourcemanager.folderAdmin"    = [module.branch-gke-prod-sa.0.iam_email]
     "roles/resourcemanager.projectCreator" = [module.branch-gke-prod-sa.0.iam_email]
     "roles/compute.xpnAdmin"               = [module.branch-gke-prod-sa.0.iam_email]
+    # read-only (plan) automation service account
+    "roles/viewer"                       = [module.branch-gke-prod-r-sa.0.iam_email]
+    "roles/resourcemanager.folderViewer" = [module.branch-gke-prod-r-sa.0.iam_email]
   }
   tag_bindings = {
     context = try(
@@ -68,6 +76,8 @@ module "branch-gke-prod-folder" {
   }
 }
 
+# automation service accounts
+
 module "branch-gke-dev-sa" {
   source       = "../../../modules/iam-service-account"
   count        = var.fast_features.gke ? 1 : 0
@@ -122,6 +132,50 @@ module "branch-gke-prod-sa" {
   }
 }
 
+# automation read-only service accounts
+
+module "branch-gke-dev-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  count        = var.fast_features.gke ? 1 : 0
+  project_id   = var.automation.project_id
+  name         = "dev-resman-gke-0r"
+  display_name = "Terraform gke multitenant development service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-gke-dev-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+module "branch-gke-prod-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  count        = var.fast_features.gke ? 1 : 0
+  project_id   = var.automation.project_id
+  name         = "prod-resman-gke-0r"
+  display_name = "Terraform gke multitenant production service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-gke-prod-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+# automation buckets
+
 module "branch-gke-dev-gcs" {
   source        = "../../../modules/gcs"
   count         = var.fast_features.gke ? 1 : 0
@@ -132,7 +186,8 @@ module "branch-gke-dev-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-gke-dev-sa.0.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-gke-dev-sa.0.iam_email]
+    "roles/storage.objectViewer" = [module.branch-gke-dev-r-sa.0.iam_email]
   }
 }
 
@@ -146,6 +201,7 @@ module "branch-gke-prod-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-gke-prod-sa.0.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-gke-prod-sa.0.iam_email]
+    "roles/storage.objectViewer" = [module.branch-gke-prod-r-sa.0.iam_email]
   }
 }

fast/stages/1-resman/branch-networking.tf

@@ -22,21 +22,21 @@ module "branch-network-folder" {
   name   = "Networking"
   group_iam = local.groups.gcp-network-admins == null ? {} : {
     (local.groups.gcp-network-admins) = [
-      # add any needed roles for resources/services not managed via Terraform,
+      # owner and viewer roles are broad and might grant unwanted access
-      # or replace editor with ~viewer if no broad resource management needed
+      # replace them with more selective custom roles for production deployments
-      # e.g.
-      #   "roles/compute.networkAdmin",
-      #   "roles/dns.admin",
-      #   "roles/compute.securityAdmin",
       "roles/editor",
     ]
   }
   iam = {
+    # read-write (apply) automation service account
     "roles/logging.admin"                  = [module.branch-network-sa.iam_email]
     "roles/owner"                          = [module.branch-network-sa.iam_email]
     "roles/resourcemanager.folderAdmin"    = [module.branch-network-sa.iam_email]
     "roles/resourcemanager.projectCreator" = [module.branch-network-sa.iam_email]
     "roles/compute.xpnAdmin"               = [module.branch-network-sa.iam_email]
+    # read-only (plan) automation service account
+    "roles/viewer"                       = [module.branch-network-r-sa.iam_email]
+    "roles/resourcemanager.folderViewer" = [module.branch-network-r-sa.iam_email]
   }
   tag_bindings = {
     context = try(
@@ -50,11 +50,18 @@ module "branch-network-prod-folder" {
   parent = module.branch-network-folder.id
   name   = "Production"
   iam = {
+    # read-write (apply) automation service accounts
     (local.custom_roles.service_project_network_admin) = concat(
       local.branch_optional_sa_lists.dp-prod,
       local.branch_optional_sa_lists.gke-prod,
       local.branch_optional_sa_lists.pf-prod,
     )
+    # read-only (plan) automation service accounts
+    "roles/compute.networkViewer" = concat(
+      local.branch_optional_r_sa_lists.dp-prod,
+      local.branch_optional_r_sa_lists.gke-prod,
+      local.branch_optional_r_sa_lists.pf-prod,
+    )
   }
   tag_bindings = {
     environment = try(
@@ -69,11 +76,18 @@ module "branch-network-dev-folder" {
   parent = module.branch-network-folder.id
   name   = "Development"
   iam = {
+    # read-write (apply) automation service accounts
     (local.custom_roles.service_project_network_admin) = concat(
       local.branch_optional_sa_lists.dp-dev,
       local.branch_optional_sa_lists.gke-dev,
       local.branch_optional_sa_lists.pf-dev,
     )
+    # read-only (plan) automation service accounts
+    "roles/compute.networkViewer" = concat(
+      local.branch_optional_r_sa_lists.dp-prod,
+      local.branch_optional_r_sa_lists.gke-prod,
+      local.branch_optional_r_sa_lists.pf-prod,
+    )
   }
   tag_bindings = {
     environment = try(
@@ -83,7 +97,7 @@ module "branch-network-dev-folder" {
   }
 }
 
-# automation service account and bucket
+# automation service account
 
 module "branch-network-sa" {
   source       = "../../../modules/iam-service-account"
@@ -104,6 +118,29 @@ module "branch-network-sa" {
   }
 }
 
+# automation read-only service account
+
+module "branch-network-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  project_id   = var.automation.project_id
+  name         = "prod-resman-net-0r"
+  display_name = "Terraform resman networking service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-network-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+# automation bucket
+
 module "branch-network-gcs" {
   source        = "../../../modules/gcs"
   project_id    = var.automation.project_id
@@ -113,6 +150,7 @@ module "branch-network-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-network-sa.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-network-sa.iam_email]
+    "roles/storage.objectViewer" = [module.branch-network-r-sa.iam_email]
   }
 }

fast/stages/1-resman/branch-project-factory.tf

@@ -16,12 +16,13 @@
 
 # tfdoc:file:description Project factory stage resources.
 
+# automation service accounts
+
 module "branch-pf-dev-sa" {
-  source     = "../../../modules/iam-service-account"
+  source       = "../../../modules/iam-service-account"
-  count      = var.fast_features.project_factory ? 1 : 0
+  count        = var.fast_features.project_factory ? 1 : 0
-  project_id = var.automation.project_id
+  project_id   = var.automation.project_id
-  name       = "dev-resman-pf-0"
+  name         = "dev-resman-pf-0"
-  # naming: environment in description
   display_name = "Terraform project factory development service account."
   prefix       = var.prefix
   iam = {
@@ -38,11 +39,10 @@ module "branch-pf-dev-sa" {
 }
 
 module "branch-pf-prod-sa" {
-  source     = "../../../modules/iam-service-account"
+  source       = "../../../modules/iam-service-account"
-  count      = var.fast_features.project_factory ? 1 : 0
+  count        = var.fast_features.project_factory ? 1 : 0
-  project_id = var.automation.project_id
+  project_id   = var.automation.project_id
-  name       = "prod-resman-pf-0"
+  name         = "prod-resman-pf-0"
-  # naming: environment in description
   display_name = "Terraform project factory production service account."
   prefix       = var.prefix
   iam = {
@@ -58,6 +58,50 @@ module "branch-pf-prod-sa" {
   }
 }
 
+# automation read-only service accounts
+
+module "branch-pf-dev-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  count        = var.fast_features.project_factory ? 1 : 0
+  project_id   = var.automation.project_id
+  name         = "dev-resman-pf-0r"
+  display_name = "Terraform project factory development service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-pf-dev-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+module "branch-pf-prod-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  count        = var.fast_features.project_factory ? 1 : 0
+  project_id   = var.automation.project_id
+  name         = "prod-resman-pf-0r"
+  display_name = "Terraform project factory production service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-pf-prod-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+# automation buckets
+
 module "branch-pf-dev-gcs" {
   source        = "../../../modules/gcs"
   count         = var.fast_features.project_factory ? 1 : 0
@@ -68,7 +112,8 @@ module "branch-pf-dev-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-pf-dev-sa.0.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-pf-dev-sa.0.iam_email]
+    "roles/storage.objectViewer" = [module.branch-pf-dev-r-sa.0.iam_email]
   }
 }
 
@@ -82,6 +127,7 @@ module "branch-pf-prod-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-pf-prod-sa.0.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-pf-prod-sa.0.iam_email]
+    "roles/storage.objectViewer" = [module.branch-pf-prod-r-sa.0.iam_email]
   }
 }

fast/stages/1-resman/branch-security.tf

@@ -22,22 +22,20 @@ module "branch-security-folder" {
   name   = "Security"
   group_iam = local.groups.gcp-security-admins == null ? {} : {
     (local.groups.gcp-security-admins) = [
-      # add any needed roles for resources/services not managed via Terraform,
+      # owner and viewer roles are broad and might grant unwanted access
-      # e.g.
+      # replace them with more selective custom roles for production deployments
-      # "roles/bigquery.admin",
+      "roles/editor"
-      # "roles/cloudasset.owner",
-      # "roles/cloudkms.admin",
-      # "roles/logging.admin",
-      # "roles/secretmanager.admin",
-      # "roles/storage.admin",
-      "roles/viewer"
     ]
   }
   iam = {
+    # read-write (apply) automation service account
     "roles/logging.admin"                  = [module.branch-security-sa.iam_email]
     "roles/owner"                          = [module.branch-security-sa.iam_email]
     "roles/resourcemanager.folderAdmin"    = [module.branch-security-sa.iam_email]
     "roles/resourcemanager.projectCreator" = [module.branch-security-sa.iam_email]
+    # read-only (plan) automation service account
+    "roles/viewer"                       = [module.branch-network-r-sa.iam_email]
+    "roles/resourcemanager.folderViewer" = [module.branch-network-r-sa.iam_email]
   }
   tag_bindings = {
     context = try(
@@ -46,7 +44,7 @@ module "branch-security-folder" {
   }
 }
 
-# automation service account and bucket
+# automation service account
 
 module "branch-security-sa" {
   source       = "../../../modules/iam-service-account"
@@ -67,6 +65,29 @@ module "branch-security-sa" {
   }
 }
 
+# automation read-only service account
+
+module "branch-security-r-sa" {
+  source       = "../../../modules/iam-service-account"
+  project_id   = var.automation.project_id
+  name         = "prod-resman-sec-0r"
+  display_name = "Terraform resman security service account (read-only)."
+  prefix       = var.prefix
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.branch-security-r-sa-cicd.0.iam_email, null)
+    ])
+  }
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/serviceusage.serviceUsageConsumer"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = [var.custom_roles["storage_viewer"]]
+  }
+}
+
+# automation bucket
+
 module "branch-security-gcs" {
   source        = "../../../modules/gcs"
   project_id    = var.automation.project_id
@@ -76,6 +97,7 @@ module "branch-security-gcs" {
   storage_class = local.gcs_storage_class
   versioning    = true
   iam = {
-    "roles/storage.objectAdmin" = [module.branch-security-sa.iam_email]
+    "roles/storage.objectAdmin"  = [module.branch-security-sa.iam_email]
+    "roles/storage.objectViewer" = [module.branch-security-r-sa.iam_email]
   }
 }

fast/stages/1-resman/branch-teams.tf

@@ -18,8 +18,6 @@
 
 # TODO(ludo): add support for CI/CD
 
-############### top-level Teams branch and automation resources ###############
-
 module "branch-teams-folder" {
   source = "../../../modules/folder"
   count  = var.fast_features.teams ? 1 : 0
@@ -68,7 +66,6 @@ module "branch-teams-gcs" {
   }
 }
 
-################## per-team folders and automation resources ##################
 
 module "branch-teams-team-folder" {
   source   = "../../../modules/folder"

fast/stages/1-resman/branch-tenants.tf

@@ -16,6 +16,8 @@
 
 # tfdoc:file:description Lightweight tenant resources.
 
+# TODO(ludo): add support for CI/CD
+
 locals {
   tenant_iam = {
     for k, v in var.tenants : k => [
@@ -174,6 +176,14 @@ module "tenant-self-iac-project" {
       "roles/iam.workloadIdentityPoolAdmin"
     ]
   }
+  iam = {
+    (var.custom_roles.storage_viewer) = [
+      "serviceAccount:${var.automation.service_accounts.resman-r}"
+    ]
+    "roles/viewer" = [
+      "serviceAccount:${var.automation.service_accounts.resman-r}"
+    ]
+  }
   services = [
     "accesscontextmanager.googleapis.com",
     "bigquery.googleapis.com",

fast/stages/1-resman/cicd-data-platform.tf

@@ -86,7 +86,7 @@ module "branch-dp-prod-cicd-repo" {
   depends_on = [module.branch-dp-prod-sa-cicd]
 }
 
-# SAs used by CI/CD workflows to impersonate automation SAs
+# read-write (apply) SAs used by CI/CD workflows to impersonate automation SAs
 
 module "branch-dp-dev-sa-cicd" {
   source = "../../../modules/iam-service-account"
@@ -110,12 +110,12 @@ module "branch-dp-dev-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -153,12 +153,12 @@ module "branch-dp-prod-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -173,3 +173,73 @@ module "branch-dp-prod-sa-cicd" {
     (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
   }
 }
+
+# read-only (plan) SAs used by CI/CD workflows to impersonate automation SAs
+
+module "branch-dp-dev-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.data_platform_dev.name, null) != null
+    ? { 0 = local.cicd_repositories.data_platform_dev }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "dev-resman-dp-1r"
+  display_name = "Terraform CI/CD data platform development service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}
+
+module "branch-dp-prod-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.data_platform_prod.name, null) != null
+    ? { 0 = local.cicd_repositories.data_platform_prod }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "prod-resman-dp-1r"
+  display_name = "Terraform CI/CD data platform production service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}

fast/stages/1-resman/cicd-gke.tf

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-# tfdoc:file:description CI/CD resources for the data platform branch.
+# tfdoc:file:description CI/CD resources for the GKE multitenant branch.
 
 # source repositories
 
@@ -86,7 +86,7 @@ module "branch-gke-prod-cicd-repo" {
   depends_on = [module.branch-gke-prod-sa-cicd]
 }
 
-# SAs used by CI/CD workflows to impersonate automation SAs
+# read-write (apply) SAs used by CI/CD workflows to impersonate automation SAs
 
 module "branch-gke-dev-sa-cicd" {
   source = "../../../modules/iam-service-account"
@@ -110,12 +110,12 @@ module "branch-gke-dev-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -153,12 +153,12 @@ module "branch-gke-prod-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -173,3 +173,73 @@ module "branch-gke-prod-sa-cicd" {
     (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
   }
 }
+
+# read-only (plan) SAs used by CI/CD workflows to impersonate automation SAs
+
+module "branch-gke-dev-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.gke_dev.name, null) != null
+    ? { 0 = local.cicd_repositories.gke_dev }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "dev-resman-gke-1r"
+  display_name = "Terraform CI/CD gke multitenant development service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}
+
+module "branch-gke-prod-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.gke_prod.name, null) != null
+    ? { 0 = local.cicd_repositories.gke_prod }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "prod-resman-gke-1r"
+  display_name = "Terraform CI/CD gke multitenant production service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}

fast/stages/1-resman/cicd-networking.tf

@@ -48,7 +48,7 @@ module "branch-network-cicd-repo" {
   depends_on = [module.branch-network-sa-cicd]
 }
 
-# SA used by CI/CD workflows to impersonate automation SAs
+# read-write (apply) SA used by CI/CD workflows to impersonate automation SA
 
 module "branch-network-sa-cicd" {
   source = "../../../modules/iam-service-account"
@@ -72,12 +72,12 @@ module "branch-network-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -92,3 +92,39 @@ module "branch-network-sa-cicd" {
     (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
   }
 }
+
+# read-only (plan) SA used by CI/CD workflows to impersonate automation SA
+
+module "branch-network-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.networking.name, null) != null
+    ? { 0 = local.cicd_repositories.networking }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "prod-resman-net-1r"
+  display_name = "Terraform CI/CD stage 2 networking service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}

fast/stages/1-resman/cicd-project-factory.tf

@@ -18,11 +18,6 @@
 
 # source repositories
 
-moved {
-  from = module.branch-teams-dev-pf-cicd-repo
-  to   = module.branch-pf-dev-cicd-repo
-}
-
 module "branch-pf-dev-cicd-repo" {
   source = "../../../modules/source-repository"
   for_each = (
@@ -55,11 +50,6 @@ module "branch-pf-dev-cicd-repo" {
   depends_on = [module.branch-pf-dev-sa-cicd]
 }
 
-moved {
-  from = module.branch-teams-prod-pf-cicd-repo
-  to   = module.branch-pf-prod-cicd-repo
-}
-
 module "branch-pf-prod-cicd-repo" {
   source = "../../../modules/source-repository"
   for_each = (
@@ -92,12 +82,7 @@ module "branch-pf-prod-cicd-repo" {
   depends_on = [module.branch-pf-prod-sa-cicd]
 }
 
-# SAs used by CI/CD workflows to impersonate automation SAs
+# read-write (apply) SAs used by CI/CD workflows to impersonate automation SAs
-
-moved {
-  from = module.branch-teams-dev-pf-sa-cicd
-  to   = module.branch-pf-dev-sa-cicd
-}
 
 module "branch-pf-dev-sa-cicd" {
   source = "../../../modules/iam-service-account"
@@ -121,12 +106,12 @@ module "branch-pf-dev-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -142,11 +127,6 @@ module "branch-pf-dev-sa-cicd" {
   }
 }
 
-moved {
-  from = module.branch-teams-prod-pf-sa-cicd
-  to   = module.branch-pf-prod-sa-cicd
-}
-
 module "branch-pf-prod-sa-cicd" {
   source = "../../../modules/iam-service-account"
   for_each = (
@@ -169,12 +149,12 @@ module "branch-pf-prod-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -189,3 +169,73 @@ module "branch-pf-prod-sa-cicd" {
     (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
   }
 }
+
+# read-only (plan) SAs used by CI/CD workflows to impersonate automation SAs
+
+module "branch-pf-dev-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.project_factory_dev.name, null) != null
+    ? { 0 = local.cicd_repositories.project_factory_dev }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "dev-resman-pf-1r"
+  display_name = "Terraform CI/CD project factory development service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}
+
+module "branch-pf-prod-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.project_factory_prod.name, null) != null
+    ? { 0 = local.cicd_repositories.project_factory_prod }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "prod-resman-pf-1r"
+  display_name = "Terraform CI/CD project factory production service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}

fast/stages/1-resman/cicd-security.tf

@@ -48,7 +48,7 @@ module "branch-security-cicd-repo" {
   depends_on = [module.branch-security-sa-cicd]
 }
 
-# SA used by CI/CD workflows to impersonate automation SAs
+# read-write (apply) SA used by CI/CD workflows to impersonate automation SA
 
 module "branch-security-sa-cicd" {
   source = "../../../modules/iam-service-account"
@@ -72,12 +72,12 @@ module "branch-security-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.branch == null
         ? format(
-          local.identity_providers[each.value.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.name
         )
         : format(
-          local.identity_providers[each.value.identity_provider].principal_tpl,
+          local.identity_providers[each.value.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.name,
           each.value.branch
@@ -92,3 +92,39 @@ module "branch-security-sa-cicd" {
     (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
   }
 }
+
+# read-only (plan) SA used by CI/CD workflows to impersonate automation SA
+
+module "branch-security-r-sa-cicd" {
+  source = "../../../modules/iam-service-account"
+  for_each = (
+    try(local.cicd_repositories.security.name, null) != null
+    ? { 0 = local.cicd_repositories.security }
+    : {}
+  )
+  project_id   = var.automation.project_id
+  name         = "prod-resman-sec-1r"
+  display_name = "Terraform CI/CD stage 2 security service account (read-only)."
+  prefix       = var.prefix
+  iam = (
+    each.value.type == "sourcerepo"
+    # build trigger for read-only SA is optionally defined by users
+    ? {}
+    # impersonated via workload identity federation for external repos
+    : {
+      "roles/iam.workloadIdentityUser" = [
+        format(
+          local.identity_providers[each.value.identity_provider].principal_repo,
+          var.automation.federated_identity_pool,
+          each.value.name
+        )
+      ]
+    }
+  )
+  iam_project_roles = {
+    (var.automation.project_id) = ["roles/logging.logWriter"]
+  }
+  iam_storage_roles = {
+    (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+  }
+}

fast/stages/1-resman/cicd-teams.tf

@@ -71,12 +71,12 @@ module "branch-teams-team-sa-cicd" {
       "roles/iam.workloadIdentityUser" = [
         each.value.cicd.branch == null
         ? format(
-          local.identity_providers[each.value.cicd.identity_provider].principalset_tpl,
+          local.identity_providers[each.value.cicd.identity_provider].principal_repo,
           var.automation.federated_identity_pool,
           each.value.cicd.name
         )
         : format(
-          local.identity_providers[each.value.cicd.identity_provider].principal_tpl,
+          local.identity_providers[each.value.cicd.identity_provider].principal_branch,
           var.automation.federated_identity_pool,
           each.value.cicd.name,
           each.value.cicd.branch

fast/stages/1-resman/main.tf

@@ -24,6 +24,7 @@ locals {
     ? []
     : ["serviceAccount:${local.automation_resman_sa}"]
   )
+  # service accounts that receive additional grants on networking/security
   branch_optional_sa_lists = {
     dp-dev   = compact([try(module.branch-dp-dev-sa.0.iam_email, "")])
     dp-prod  = compact([try(module.branch-dp-prod-sa.0.iam_email, "")])
@@ -32,6 +33,15 @@ locals {
     pf-dev   = compact([try(module.branch-pf-dev-sa.0.iam_email, "")])
     pf-prod  = compact([try(module.branch-pf-prod-sa.0.iam_email, "")])
   }
+  branch_optional_r_sa_lists = {
+    dp-dev   = compact([try(module.branch-dp-dev-r-sa.0.iam_email, "")])
+    dp-prod  = compact([try(module.branch-dp-prod-r-sa.0.iam_email, "")])
+    gke-dev  = compact([try(module.branch-gke-dev-r-sa.0.iam_email, "")])
+    gke-prod = compact([try(module.branch-gke-prod-r-sa.0.iam_email, "")])
+    pf-dev   = compact([try(module.branch-pf-dev-r-sa.0.iam_email, "")])
+    pf-prod  = compact([try(module.branch-pf-prod-r-sa.0.iam_email, "")])
+  }
+  # normalize CI/CD repositories
   cicd_repositories = {
     for k, v in coalesce(var.cicd_repositories, {}) : k => v
     if(

fast/stages/1-resman/outputs.tf

@@ -18,44 +18,92 @@ locals {
   _tpl_providers = "${path.module}/templates/providers.tf.tpl"
   cicd_workflow_attrs = {
     data_platform_dev = {
-      service_account   = try(module.branch-dp-dev-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "3-data-platform-dev-providers.tf"
+        apply = try(module.branch-dp-dev-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_3
+        plan  = try(module.branch-dp-dev-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "3-data-platform-dev-providers.tf"
+        plan  = "3-data-platform-dev-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_3
     }
     data_platform_prod = {
-      service_account   = try(module.branch-dp-prod-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "3-data-platform-prod-providers.tf"
+        apply = try(module.branch-dp-prod-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_3
+        plan  = try(module.branch-dp-prod-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "3-data-platform-prod-providers.tf"
+        plan  = "3-data-platform-prod-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_3
     }
     gke_dev = {
-      service_account   = try(module.branch-gke-dev-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "3-gke-dev-providers.tf"
+        apply = try(module.branch-gke-dev-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_3
+        plan  = try(module.branch-gke-dev-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "3-gke-dev-providers.tf"
+        plan  = "3-gke-dev-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_3
     }
     gke_prod = {
-      service_account   = try(module.branch-gke-prod-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "3-gke-prod-providers.tf"
+        apply = try(module.branch-gke-prod-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_3
+        plan  = try(module.branch-gke-prod-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "3-gke-prod-providers.tf"
+        plan  = "3-gke-prod-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_3
     }
     networking = {
-      service_account   = try(module.branch-network-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "2-networking-providers.tf"
+        apply = try(module.branch-network-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_2
+        plan  = try(module.branch-network-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "2-networking-providers.tf"
+        plan  = "2-networking-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_2
     }
     project_factory_dev = {
-      service_account   = try(module.branch-pf-dev-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "3-project-factory-dev-providers.tf"
+        apply = try(module.branch-pf-dev-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_3
+        plan  = try(module.branch-pf-dev-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "3-project-factory-dev-providers.tf"
+        plan  = "3-project-factory-dev-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_3
     }
     project_factory_prod = {
-      service_account   = try(module.branch-pf-prod-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "3-project-factory-prod-providers.tf"
+        apply = try(module.branch-pf-prod-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_3
+        plan  = try(module.branch-pf-prod-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "3-project-factory-prod-providers.tf"
+        plan  = "3-project-factory-prod-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_3
     }
     security = {
-      service_account   = try(module.branch-security-sa-cicd.0.email, null)
+      service_accounts = {
-      tf_providers_file = "2-security-providers.tf"
+        apply = try(module.branch-security-sa-cicd.0.email, null)
-      tf_var_files      = local.cicd_workflow_var_files.stage_2
+        plan  = try(module.branch-security-r-sa-cicd.0.email, null)
+      }
+      tf_providers_files = {
+        apply = "2-security-providers.tf"
+        plan  = "2-security-r-providers.tf"
+      }
+      tf_var_files = local.cicd_workflow_var_files.stage_2
     }
   }
   cicd_workflows = {
@@ -107,12 +155,24 @@ locals {
         name          = "networking"
         sa            = module.branch-network-sa.email
       })
+      "2-networking-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-network-gcs.name
+        name          = "networking"
+        sa            = module.branch-network-r-sa.email
+      })
       "2-security" = templatefile(local._tpl_providers, {
         backend_extra = null
         bucket        = module.branch-security-gcs.name
         name          = "security"
         sa            = module.branch-security-sa.email
       })
+      "2-security-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-security-gcs.name
+        name          = "security"
+        sa            = module.branch-security-r-sa.email
+      })
     },
     !var.fast_features.data_platform ? {} : {
       "3-data-platform-dev" = templatefile(local._tpl_providers, {
@@ -121,12 +181,24 @@ locals {
         name          = "dp-dev"
         sa            = module.branch-dp-dev-sa.0.email
       })
+      "3-data-platform-dev-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-dp-dev-gcs.0.name
+        name          = "dp-dev"
+        sa            = module.branch-dp-dev-r-sa.0.email
+      })
       "3-data-platform-prod" = templatefile(local._tpl_providers, {
         backend_extra = null
         bucket        = module.branch-dp-prod-gcs.0.name
         name          = "dp-prod"
         sa            = module.branch-dp-prod-sa.0.email
       })
+      "3-data-platform-prod-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-dp-prod-gcs.0.name
+        name          = "dp-prod"
+        sa            = module.branch-dp-prod-r-sa.0.email
+      })
     },
     !var.fast_features.gke ? {} : {
       "3-gke-dev" = templatefile(local._tpl_providers, {
@@ -135,12 +207,24 @@ locals {
         name          = "gke-dev"
         sa            = module.branch-gke-dev-sa.0.email
       })
+      "3-gke-dev-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-gke-dev-gcs.0.name
+        name          = "gke-dev"
+        sa            = module.branch-gke-dev-r-sa.0.email
+      })
       "3-gke-prod" = templatefile(local._tpl_providers, {
         backend_extra = null
         bucket        = module.branch-gke-prod-gcs.0.name
         name          = "gke-prod"
         sa            = module.branch-gke-prod-sa.0.email
       })
+      "3-gke-prod-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-gke-prod-gcs.0.name
+        name          = "gke-prod"
+        sa            = module.branch-gke-prod-r-sa.0.email
+      })
     },
     !var.fast_features.project_factory ? {} : {
       "3-project-factory-dev" = templatefile(local._tpl_providers, {
@@ -149,12 +233,24 @@ locals {
         name          = "team-dev"
         sa            = module.branch-pf-dev-sa.0.email
       })
+      "3-project-factory-dev-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-pf-dev-gcs.0.name
+        name          = "team-dev"
+        sa            = module.branch-pf-dev-r-sa.0.email
+      })
       "3-project-factory-prod" = templatefile(local._tpl_providers, {
         backend_extra = null
         bucket        = module.branch-pf-prod-gcs.0.name
         name          = "team-prod"
         sa            = module.branch-pf-prod-sa.0.email
       })
+      "3-project-factory-prod-r" = templatefile(local._tpl_providers, {
+        backend_extra = null
+        bucket        = module.branch-pf-prod-gcs.0.name
+        name          = "team-prod"
+        sa            = module.branch-pf-prod-r-sa.0.email
+      })
     },
     !var.fast_features.sandbox ? {} : {
       "9-sandbox" = templatefile(local._tpl_providers, {
@@ -186,16 +282,24 @@ locals {
   )
   service_accounts = merge(
     {
-      data-platform-dev    = try(module.branch-dp-dev-sa.0.email, null)
+      data-platform-dev      = try(module.branch-dp-dev-sa.0.email, null)
-      data-platform-prod   = try(module.branch-dp-prod-sa.0.email, null)
+      data-platform-dev-r    = try(module.branch-dp-dev-r-sa.0.email, null)
-      gke-dev              = try(module.branch-gke-dev-sa.0.email, null)
+      data-platform-prod     = try(module.branch-dp-prod-sa.0.email, null)
-      gke-prod             = try(module.branch-gke-prod-sa.0.email, null)
+      data-platform-prod-r   = try(module.branch-dp-prod-r-sa.0.email, null)
-      networking           = module.branch-network-sa.email
+      gke-dev                = try(module.branch-gke-dev-sa.0.email, null)
-      project-factory-dev  = try(module.branch-pf-dev-sa.0.email, null)
+      gke-dev-r              = try(module.branch-gke-dev-r-sa.0.email, null)
-      project-factory-prod = try(module.branch-pf-prod-sa.0.email, null)
+      gke-prod               = try(module.branch-gke-prod-sa.0.email, null)
-      sandbox              = try(module.branch-sandbox-sa.0.email, null)
+      gke-prod-r             = try(module.branch-gke-prod-r-sa.0.email, null)
-      security             = module.branch-security-sa.email
+      networking             = module.branch-network-sa.email
-      teams                = try(module.branch-teams-sa.0.email, null)
+      networking-r           = module.branch-network-r-sa.email
+      project-factory-dev    = try(module.branch-pf-dev-sa.0.email, null)
+      project-factory-dev-r  = try(module.branch-pf-dev-r-sa.0.email, null)
+      project-factory-prod   = try(module.branch-pf-prod-sa.0.email, null)
+      project-factory-prod-r = try(module.branch-pf-prod-r-sa.0.email, null)
+      sandbox                = try(module.branch-sandbox-sa.0.email, null)
+      security               = module.branch-security-sa.email
+      security-r             = module.branch-security-r-sa.email
+      teams                  = try(module.branch-teams-sa.0.email, null)
     },
     {
       for k, v in module.branch-teams-team-sa : "team-${k}" => v.email
@@ -238,7 +342,7 @@ output "cicd_repositories" {
       provider = try(
         local.identity_providers[v.identity_provider].name, null
       )
-      service_account = local.cicd_workflow_attrs[k].service_account
+      service_account = local.cicd_workflow_attrs[k].service_accounts
     } if v != null
   }
 }

fast/stages/1-resman/templates/workflow-github.yaml

@@ -24,15 +24,13 @@ on:
       - synchronize
 
 env:
-  FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+  FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
-  FAST_SERVICE_ACCOUNT: ${service_account}
+  FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_PROVIDERS_FILE: ${tf_providers_files.apply}
-  %{~ if tf_var_files != [] ~}
+  TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan}
-  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
+  TF_VERSION: 1.6.5
-  %{~ endif ~}
-  TF_VERSION: 1.5.1
 
 jobs:
   fast-pr:
@@ -48,48 +46,66 @@ jobs:
         uses: actions/checkout@v3
 
       # set up SSH key authentication to the modules repository
+
       - id: ssh-config
         name: Configure SSH authentication
         run: |
           ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
           ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
 
-      # set up authentication via Workload identity Federation
+      # set up step variables for plan / apply
+
+      - id: vars-plan
+        if: github.event.pull_request.merged != true && success()
+        name: Set up plan variables
+        run: |
+          echo "plan_opts=-lock=false" >> "$GITHUB_ENV"
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE_PLAN}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT_PLAN}}" >> "$GITHUB_ENV"
+
+      - id: vars-apply
+        if: github.event.pull_request.merged == true && success()
+        name: Set up apply variables
+        run: |
+          echo "provider_file=$${{env.TF_PROVIDERS_FILE}}" >> "$GITHUB_ENV"
+          echo "service_account=$${{env.FAST_SERVICE_ACCOUNT}}" >> "$GITHUB_ENV"
+
+      # set up authentication via Workload identity Federation and gcloud
+
       - id: gcp-auth
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
-          workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+          workload_identity_provider: $${{env.FAST_WIF_PROVIDER}}
-          service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+          service_account: $${{env.service_account}}
-          access_token_lifetime: 3600s
+          access_token_lifetime: 900s
 
       - id: gcp-sdk
         name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         with:
           install_components: alpha
 
-      # copy provider and tfvars files
+      # copy provider file
-      - id: tf-config
+
-        name: Copy Terraform output files
+      - id: tf-config-provider
+        name: Copy Terraform provider file
         run: |
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+            "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./
-          %{~ if tf_var_files != [] ~}
+          %{~ for f in tf_var_files ~}
           gcloud alpha storage cp -r \
-            "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+            "gs://${outputs_bucket}/tfvars/${f}" ./
-          for f in $${{env.TF_VAR_FILES}}; do
+          %{~ endfor ~}
-            ln -s "tfvars/$f" ./
-          done
-          %{~ endif ~}
 
       - id: tf-setup
         name: Set up Terraform
         uses: hashicorp/setup-terraform@v2.0.3
         with:
-          terraform_version: $${{ env.TF_VERSION }}
+          terraform_version: $${{env.TF_VERSION}}
 
       # run Terraform init/validate/plan
+
       - id: tf-init
         name: Terraform init
         continue-on-error: true
@@ -105,7 +121,7 @@ jobs:
         name: Terraform plan
         continue-on-error: true
         run: |
-          terraform plan -input=false -out ../plan.out -no-color
+          terraform plan -input=false -out ../plan.out -no-color $${{env.plan_opts}}
 
       - id: tf-apply
         if: github.event.pull_request.merged == true && success()
@@ -114,28 +130,31 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -no-color ../plan.out
 
+      # PR comment with Terraform result from previous steps
+      # length is checked and trimmed for length so as to stay within the limit
+
       - id: pr-comment
         name: Post comment to Pull Request
         continue-on-error: true
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request'
         env:
-          PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+          PLAN: $${{steps.tf-plan.outputs.stdout}}\n$${{steps.tf-plan.outputs.stderr}}
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
             <details><summary>Validation Output</summary>
 
             \`\`\`\n
-            $${{ steps.tf-validate.outputs.stdout }}
+            $${{steps.tf-validate.outputs.stdout}}
             \`\`\`
 
             </details>
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             <details><summary>Show Plan</summary>
 
@@ -145,9 +164,9 @@ jobs:
 
             </details>
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -157,22 +176,22 @@ jobs:
             })
 
       - id: pr-short-comment
-        name: Post comment to Pull Request
+        name: Post comment to Pull Request (abbreviated)
         uses: actions/github-script@v6
         if: github.event_name == 'pull_request' && steps.pr-comment.outcome != 'success'
         with:
           script: |
-            const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+            const output = `### Terraform Initialization \`$${{steps.tf-init.outcome}}\`
 
-            ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+            ### Terraform Validation \`$${{steps.tf-validate.outcome}}\`
 
-            ### Terraform Plan \`$${{ steps.tf-plan.outcome }}\`
+            ### Terraform Plan \`$${{steps.tf-plan.outcome}}\`
 
             Plan output is in the action log.
 
-            ### Terraform Apply \`$${{ steps.tf-apply.outcome }}\`
+            ### Terraform Apply \`$${{steps.tf-apply.outcome}}\`
 
-            *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`;
+            *Pusher: @$${{github.actor}}, Action: \`$${{github.event_name}}\`, Working Directory: \`$${{env.tf_actions_working_dir}}\`, Workflow: \`$${{github.workflow}}\`*`;
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
@@ -181,6 +200,8 @@ jobs:
               body: output
             })
 
+      # exit on error from previous steps
+
       - id: check-init
         name: Check init failure
         if: steps.tf-init.outcome != 'success'

fast/stages/1-resman/templates/workflow-gitlab.yaml

@@ -12,43 +12,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-default:
-  image:
-    name: hashicorp/terraform
-    entrypoint:
-      - "/usr/bin/env"
-      - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
 variables:
   GOOGLE_CREDENTIALS: cicd-sa-credentials.json
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
-  FAST_SERVICE_ACCOUNT: ${service_account}
   FAST_WIF_PROVIDER: ${identity_provider}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  %{~ if tf_providers_file != "" ~}
+  %{~ if tf_var_files != [] ~}
-  TF_PROVIDERS_FILE: ${tf_providers_file}
+  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
   %{~ endif ~}
-  TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n    ", tf_var_files)}
+
+workflow:
+  rules:
+    # merge / apply
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        COMMAND: apply
+        FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
+        TF_PROVIDERS_FILE: 0-bootstrap-providers.tf
+    # pr / plan
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      variables:
+        COMMAND: plan
+        FAST_SERVICE_ACCOUNT: ${service_accounts.plan}
+        TF_PROVIDERS_FILE: 0-bootstrap-r-providers.tf
 
 stages:
-  - gcp-auth
+  - gcp-setup
-  - tf-files
+  - tf-plan-apply
-  - tf-plan
-  - tf-apply
 
-cache:
+# TODO: document project-level deploy key used to fetch modules
-  key: gcp-auth
-  paths:
-    - cicd-sa-credentials.json
-    - token.txt
-    %{~ if tf_providers_file != "" ~}
-    - ${tf_providers_file}
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - ${f}
-    %{~ endfor ~}
 
-gcp-auth:
+gcp-setup:
+  stage: gcp-setup
+  image:
+    name: google/cloud-sdk:slim
+  artifacts:
+    paths:
+      - cicd-sa-credentials.json
+      - providers.tf
+  id_tokens:
+    GITLAB_TOKEN:
+      aud:
+      %{~ for aud in audiences ~}
+        - ${aud}
+      %{~ endfor ~}
+  before_script:
+    - echo "$GITLAB_TOKEN" > token.txt
+  script:
+    - |
+      gcloud iam workload-identity-pools create-cred-config \
+        $FAST_WIF_PROVIDER \
+        --service-account=$FAST_SERVICE_ACCOUNT \
+        --service-account-token-lifetime-seconds=900 \
+        --output-file=$GOOGLE_CREDENTIALS \
+        --credential-source-file=token.txt
+    - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS
+    - gcloud alpha storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf
+
+tf-plan-apply:
+  stage: tf-plan-apply
+  dependencies:
+    - gcp-setup
   id_tokens:
     GITLAB_TOKEN:
       aud:
@@ -56,69 +80,20 @@ gcp-auth:
         - ${aud}
       %{~ endfor ~}
   image:
-    name: google/cloud-sdk:slim
+    name: hashicorp/terraform
-  stage: gcp-auth
+    entrypoint:
+      - "/usr/bin/env"
+  variables:
+    SSH_AUTH_SOCK: /tmp/ssh-agent.sock
   script:
-    - echo "$${GITLAB_TOKEN}" > token.txt
     - |
-      gcloud iam workload-identity-pools create-cred-config \
+      ssh-agent -a $SSH_AUTH_SOCK
-        $${FAST_WIF_PROVIDER} \
+      echo "$CICD_MODULES_KEY" | ssh-add -
-        --service-account=$${FAST_SERVICE_ACCOUNT} \
+      mkdir -p ~/.ssh
-        --service-account-token-lifetime-seconds=3600 \
+      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-        --output-file=$${GOOGLE_CREDENTIALS} \
+      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-        --credential-source-file=token.txt
+    - echo "$GITLAB_TOKEN" > token.txt
-
-tf-files:
-  dependencies:
-    - gcp-auth
-  image:
-    name: google/cloud-sdk:slim
-  stage: tf-files
-  script:
-    # - gcloud components install -q alpha
-    - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS}
-    %{~ if tf_providers_file != "" ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./
-    %{~ endif ~}
-    %{~ for f in tf_var_files ~}
-    - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./
-    %{~ endfor ~}
-    - ls -l
-
-tf-plan:
-  dependencies:
-    - tf-files
-  stage: tf-plan
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
     - terraform init
     - terraform validate
-    - terraform plan
+    - "if [ $COMMAND == 'plan' ]; then terraform plan -input=false -no-color -lock=false; fi"
-
+    - "if [ $COMMAND == 'apply' ]; then terraform apply -input=false -no-color -auto-approve; fi"
-tf-apply:
-  dependencies:
-    - tf-files
-  stage: tf-apply
-  # uncomment the following lines and set the SSH key secret for private modules repo
-  # before_script:
-  #   - |
-  #     ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-  #     echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
-  #     mkdir -p ~/.ssh
-  #     ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-  #     ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-  script:
-    - terraform init
-    - terraform validate
-    - terraform apply -input=false -auto-approve
-  when: manual
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

fast/stages/1-resman/variables.tf

@@ -30,9 +30,12 @@ variable "automation" {
       issuer           = string
       issuer_uri       = string
       name             = string
-      principal_tpl    = string
+      principal_branch = string
-      principalset_tpl = string
+      principal_repo   = string
     }))
+    service_accounts = object({
+      resman-r = string
+    })
   })
 }
 
@@ -134,6 +137,7 @@ variable "custom_roles" {
   description = "Custom roles defined at the org level, in key => id format."
   type = object({
     service_project_network_admin = string
+    storage_viewer                = string
   })
   default = null
 }

tests/fast/stages/s0_bootstrap/simple.yaml

@@ -16,27 +16,36 @@ counts:
   google_bigquery_dataset: 1
   google_bigquery_default_service_account: 3
   google_logging_organization_sink: 3
-  google_organization_iam_binding: 20
+  google_logging_project_bucket_config: 3
-  google_organization_iam_custom_role: 3
+  google_org_policy_policy: 13
-  google_organization_iam_member: 13
+  google_organization_iam_binding: 23
+  google_organization_iam_custom_role: 6
+  google_organization_iam_member: 22
   google_project: 3
-  google_project_iam_binding: 10
+  google_project_iam_binding: 19
-  google_project_iam_member: 5
+  google_project_iam_member: 6
   google_project_service: 29
   google_project_service_identity: 3
-  google_service_account: 2
+  google_service_account: 4
-  google_service_account_iam_binding: 1
+  google_service_account_iam_binding: 2
   google_storage_bucket: 3
-  google_storage_bucket_iam_binding: 1
+  google_storage_bucket_iam_binding: 2
-  google_storage_bucket_iam_member: 2
+  google_storage_bucket_iam_member: 4
-  google_storage_bucket_object: 5
+  google_storage_bucket_object: 7
   google_storage_project_service_account: 3
-  local_file: 5
+  google_tags_tag_key: 1
+  google_tags_tag_value: 1
+  local_file: 7
+  modules: 15
+  resources: 168
 
 outputs:
   custom_roles:
+    organization_admin_viewer: organizations/123456789012/roles/organizationAdminViewer
     organization_iam_admin: organizations/123456789012/roles/organizationIamAdmin
     service_project_network_admin: organizations/123456789012/roles/serviceProjectNetworkAdmin
+    storage_viewer: organizations/123456789012/roles/storageViewer
+    tag_viewer: organizations/123456789012/roles/tagViewer
     tenant_network_admin: organizations/123456789012/roles/tenantNetworkAdmin
   outputs_bucket: fast-prod-iac-core-outputs-0
   project_ids:

tests/fast/stages/s1_resman/common.tfvars

@@ -4,6 +4,9 @@ automation = {
   project_id                   = "fast-prod-automation"
   project_number               = 123456
   outputs_bucket               = "test"
+  service_accounts = {
+    resman-r = "ldj-prod-resman-0r@fast2-prod-iac-core-0.iam.gserviceaccount.com"
+  }
 }
 billing_account = {
   id = "000000-111111-222222"
@@ -11,6 +14,7 @@ billing_account = {
 custom_roles = {
   # organization_iam_admin = "organizations/123456789012/roles/organizationIamAdmin",
   service_project_network_admin = "organizations/123456789012/roles/xpnServiceAdmin"
+  storage_viewer                = "organizations/123456789012/roles/storageViewer"
 }
 groups = {
   gcp-billing-admins      = "gcp-billing-admins",

tests/fixtures.py

@@ -202,25 +202,36 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
                            "")
 
     if 'counts' in inventory:
-      expected_counts = inventory['counts']
+      try:
-      for type_, expected_count in expected_counts.items():
+        expected_counts = inventory['counts']
-        assert type_ in summary.counts, \
+        for type_, expected_count in expected_counts.items():
-            f'{relative_path}: module does not create any resources of type `{type_}`'
+          assert type_ in summary.counts, \
-        plan_count = summary.counts[type_]
+              f'{relative_path}: module does not create any resources of type `{type_}`'
-        assert plan_count == expected_count, \
+          plan_count = summary.counts[type_]
-            f'{relative_path}: count of {type_} resources failed. Got {plan_count}, expected {expected_count}'
+          assert plan_count == expected_count, \
+              f'{relative_path}: count of {type_} resources failed. Got {plan_count}, expected {expected_count}'
+      except AssertionError:
+        print(yaml.dump({'counts': summary.counts}))
+        raise
 
     if 'outputs' in inventory:
-      expected_outputs = inventory['outputs']
+      _buffer = None
-      for output_name, expected_output in expected_outputs.items():
+      try:
-        assert output_name in summary.outputs, \
+        expected_outputs = inventory['outputs']
-            f'{relative_path}: module does not output `{output_name}`'
+        for output_name, expected_output in expected_outputs.items():
-        output = summary.outputs[output_name]
+          assert output_name in summary.outputs, \
-        # assert 'value' in output, \
+              f'{relative_path}: module does not output `{output_name}`'
-        #   f'output `{output_name}` does not have a value (is it sensitive or dynamic?)'
+          output = summary.outputs[output_name]
-        plan_output = output.get('value', '__missing__')
+          # assert 'value' in output, \
-        assert plan_output == expected_output, \
+          #   f'output `{output_name}` does not have a value (is it sensitive or dynamic?)'
-            f'{relative_path}: output {output_name} failed. Got `{plan_output}`, expected `{expected_output}`'
+          plan_output = output.get('value', '__missing__')
+          _buffer = {output_name: plan_output}
+          assert plan_output == expected_output, \
+              f'{relative_path}: output {output_name} failed. Got `{plan_output}`, expected `{expected_output}`'
+      except AssertionError:
+        if _buffer:
+          print(yaml.dump(_buffer))
+        raise
 
   return summary
 
@@ -282,7 +293,10 @@ def plan_validator_fixture(request):
 
 
 def get_tfvars_for_e2e():
-  _variables = ["billing_account", "group_email", "organization_id", "parent", "prefix", "region"]
+  _variables = [
+      "billing_account", "group_email", "organization_id", "parent", "prefix",
+      "region"
+  ]
   tf_vars = {k: os.environ.get(f"TFTEST_E2E_{k}") for k in _variables}
   return tf_vars
 
@@ -308,7 +322,10 @@ def e2e_validator(module_path, extra_files, tf_var_files, basedir=None):
     prefix = get_tfvars_for_e2e()["prefix"]
     # to allow different tests to create projects (or other globally unique resources) with the same name
     # bump prefix forward on each test execution
-    tf_vars = {"prefix": f'{prefix}-{int(time.time())}{os.environ.get("PYTEST_XDIST_WORKER", "0")[-2:]}'}
+    tf_vars = {
+        "prefix":
+            f'{prefix}-{int(time.time())}{os.environ.get("PYTEST_XDIST_WORKER", "0")[-2:]}'
+    }
     try:
       apply = tf.apply(tf_var_file=tf_var_files, tf_vars=tf_vars)
       plan = tf.plan(output=True, tf_var_file=tf_var_files, tf_vars=tf_vars)
@@ -382,7 +399,9 @@ def e2e_tfvars_path():
       tf = tftest.TerraformTest(test_path, binary=binary)
       tf_vars_file = None
       tf_vars = get_tfvars_for_e2e()
-      tf_vars['suffix'] = os.environ.get("PYTEST_XDIST_WORKER", "0")[-2:]  # take at most 2 last chars for suffix
+      tf_vars['suffix'] = os.environ.get(
+          "PYTEST_XDIST_WORKER",
+          "0")[-2:]  # take at most 2 last chars for suffix
       tf_vars['timestamp'] = str(int(time.time()))
 
       if 'TFTEST_E2E_SETUP_TFVARS_PATH' in os.environ: