Merge remote-tracking branch 'origin/master' into fast/gke2

.github/workflows/merge-pr.yml

@@ -0,0 +1,51 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Post-merge tasks
+
+on:
+  pull_request:
+    branches:
+      - master
+    types:
+      - closed
+
+env:
+  PYTHON_VERSION: 3.10
+
+jobs:
+  if_merged:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install dependencies
+        run: |
+          pip install -r tools/requirements.txt
+      - name: Update Changelog
+        run: |
+          python3 tools/changelog.py --token secrets.GITHUB_TOKEN CHANGELOG.md
+      - name: Commit and push Changelog
+        env:
+          CI_COMMIT_MESSAGE: Update Changelog
+          CI_COMMIT_AUTHOR: Fabric Repo Workflows
+        run: |
+          git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
+          git config --global user.email "username@users.noreply.github.com"
+          git commit -a -m "${{ env.CI_COMMIT_MESSAGE }}"
+          git push

CHANGELOG.md

@@ -2,64 +2,85 @@
 
 All notable changes to this project will be documented in this file.
 
-## Unreleased
+## [Unreleased]
+
+<!-- None < 2022-06-06 13:42:51+00:00 -->
+
+
+###  FAST
+
+- [[#759](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/759)] FAST: fix missing value to format principalSet ([imp14a](https://github.com/imp14a)) <!-- 2022-07-27 06:18:27+00:00 -->
+- [[#753](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/753)] Add support for IAM bindings on service accounts to project factory ([ludoo](https://github.com/ludoo)) <!-- 2022-07-21 13:13:40+00:00 -->
+- [[#745](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/745)] FAST: specify gitlab / github providers in CI/CD stage ([imp14a](https://github.com/imp14a)) <!-- 2022-07-19 21:03:33+00:00 -->
+- [[#734](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/734)] FAST: Use spot VMs for test VM and for NVAs ([sruffilli](https://github.com/sruffilli)) <!-- 2022-07-13 11:57:03+00:00 -->
+- [[#733](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/733)] FAST: fix data platform drop BQ dataset name ([juliocc](https://github.com/juliocc)) <!-- 2022-07-12 12:45:57+00:00 -->
+- [[#730](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/730)] FAST: add billing IAM for billing group ([ludoo](https://github.com/ludoo)) <!-- 2022-07-11 06:26:13+00:00 -->
+- [[#721](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/721)] FAST: add billing.costManager role to project factory SAs ([sruffilli](https://github.com/sruffilli)) <!-- 2022-07-06 13:10:14+00:00 -->
+- [[#716](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/716)] FAST: added missing format argument to project factory CI/CD IAM bindings ([mgfeller](https://github.com/mgfeller)) <!-- 2022-07-05 10:43:32+00:00 -->
+- [[#715](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/715)] FAST: fix optional service accounts in networking stages ([ludoo](https://github.com/ludoo)) <!-- 2022-07-05 07:46:54+00:00 -->
+- [[#711](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/711)] FAST: update several stage READMEs about usage of *.auto.tfvars files ([mgfeller](https://github.com/mgfeller)) <!-- 2022-06-29 15:32:02+00:00 -->
+- [[#703](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/703)] FAST: configuration switches for features ([ludoo](https://github.com/ludoo)) <!-- 2022-06-28 15:33:38+00:00 -->
+- [[#706](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/706)] Bump providers versions and pin versions for tests ([juliocc](https://github.com/juliocc)) <!-- 2022-06-28 08:33:42+00:00 -->
+- [[#702](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/702)] FAST: also trigger GitHub workflow on PR synchronize event ([mgfeller](https://github.com/mgfeller)) <!-- 2022-06-27 08:13:42+00:00 -->
+- [[#692](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/692)] FAST: fix KMS delegation role in security stage ([lcaggio](https://github.com/lcaggio)) <!-- 2022-06-23 07:13:37+00:00 -->
+- [[#699](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/699)] FAST: add `repository_owner` to GitHub identity attributes ([ludoo](https://github.com/ludoo)) <!-- 2022-06-23 06:06:25+00:00 -->
+- [[#694](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/694)] FAST: add 00-cicd stage to allow managing repositories in Gitlab/GitHub, other CI/CD improvements ([rosmo](https://github.com/rosmo)) <!-- 2022-06-21 13:37:01+00:00 -->
+- [[#690](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/690)] FAST: fix stage tfvars link paths in documentation ([lcaggio](https://github.com/lcaggio)) <!-- 2022-06-21 06:20:31+00:00 -->
+- [[#676](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/676)] FAST: add group creation GIF to documentation ([amgoogle](https://github.com/amgoogle)) <!-- 2022-06-21 05:19:52+00:00 -->
+- [[#687](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/687)] FAST: fix service identity/SA mismatch in project factory ([dosti-tee](https://github.com/dosti-tee)) <!-- 2022-06-17 11:25:30+00:00 -->
+- [[#668](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/668)] FAST: add cleanup instructions to documentation ([ajlopezn](https://github.com/ajlopezn)) <!-- 2022-06-17 09:16:13+00:00 -->
+- [[#682](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/682)] FAST: fix CI/CD source repositories in stage 01 ([imp14a](https://github.com/imp14a)) <!-- 2022-06-16 22:17:28+00:00 -->
+- [[#675](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/675)] FAST: fix audit logs when using pubsub as destination ([juliocc](https://github.com/juliocc)) <!-- 2022-06-10 11:53:18+00:00 -->
+- [[#674](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/674)] FAST: remove team folders comment from 01 variables, clarify README ([ludoo](https://github.com/ludoo)) <!-- 2022-06-10 08:51:26+00:00 -->
+- [[#671](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/671)] FAST: fix Gitlab WIF attributes ([ludoo](https://github.com/ludoo)) <!-- 2022-06-09 06:31:50+00:00 -->
+- [[#669](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/669)] FAST: CI/CD support for Source Repository and Cloud Build ([ludoo](https://github.com/ludoo)) <!-- 2022-06-08 09:34:08+00:00 -->
+
+### EXAMPLES
+
+- [[#743](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/743)] Update Readme.md: gcs to bq + cloud armor / glb ([bensadikgoogle](https://github.com/bensadikgoogle)) <!-- 2022-08-01 15:22:04+00:00 -->
+- [[#757](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/757)] Remove key_algorithm from glb/ilb-l7 examples ([ludoo](https://github.com/ludoo)) <!-- 2022-07-25 14:00:14+00:00 -->
+- [[#753](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/753)] Add support for IAM bindings on service accounts to project factory ([ludoo](https://github.com/ludoo)) <!-- 2022-07-21 13:13:40+00:00 -->
+- [[#746](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/746)] Update multi region cloud SQL documentation ([bensadikgoogle](https://github.com/bensadikgoogle)) <!-- 2022-07-20 19:13:57+00:00 -->
+- [[#733](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/733)] FAST: fix data platform drop BQ dataset name ([juliocc](https://github.com/juliocc)) <!-- 2022-07-12 12:45:57+00:00 -->
+- [[#712](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/712)] New AD FS example ([apichick](https://github.com/apichick)) <!-- 2022-07-11 08:16:43+00:00 -->
+- [[#655](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/655)] New example for a data playground Terraform setup ([aymanfarhat](https://github.com/aymanfarhat)) <!-- 2022-07-10 07:27:18+00:00 -->
+- [[#706](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/706)] Bump providers versions and pin versions for tests ([juliocc](https://github.com/juliocc)) <!-- 2022-06-28 08:33:42+00:00 -->
+
+### MODULES
+
+- [[#764](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/764)] Add dependency on shared vpc service project attachment to project module outputs ([apichick](https://github.com/apichick)) <!-- 2022-08-02 16:38:01+00:00 -->
+- [[#761](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/761)] Fix gke hub module features condition ([ludoo](https://github.com/ludoo)) <!-- 2022-07-30 13:53:05+00:00 -->
+- [[#760](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/760)] **incompatible change:** GKE hub module refactor ([ludoo](https://github.com/ludoo)) <!-- 2022-07-29 06:39:25+00:00 -->
+- [[#756](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/756)] Set cluster id output to sensitive in GKE module ([apichick](https://github.com/apichick)) <!-- 2022-07-25 14:13:05+00:00 -->
+- [[#752](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/752)] Also depend on shared vpc host in project module ([apichick](https://github.com/apichick)) <!-- 2022-07-21 12:51:38+00:00 -->
+- [[#747](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/747)] Added gkehub.googleapis.com to jit services ([apichick](https://github.com/apichick)) <!-- 2022-07-21 12:09:12+00:00 -->
+- [[#744](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/744)] Fixed issue with missing project reference in Cloud DNS data source  ([rosmo](https://github.com/rosmo)) <!-- 2022-07-19 09:26:36+00:00 -->
+- [[#741](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/741)] Added servicemesh feature to GKE hub and included fleet robot service… ([apichick](https://github.com/apichick)) <!-- 2022-07-17 19:59:52+00:00 -->
+- [[#737](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/737)] Move Cloud Run VPC Connector annotations to template metadata (#735) ([sethmoon](https://github.com/sethmoon)) <!-- 2022-07-13 19:06:28+00:00 -->
+- [[#732](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/732)] Add support for topic message duration to pubsub module ([ludoo](https://github.com/ludoo)) <!-- 2022-07-12 07:23:24+00:00 -->
+- [[#731](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/731)] Avoid setting empty IAM binding in subnet factory ([ludoo](https://github.com/ludoo)) <!-- 2022-07-11 19:11:52+00:00 -->
+- [[#729](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/729)] Fix connector create logic in cloud run module ([ludoo](https://github.com/ludoo)) <!-- 2022-07-10 09:34:42+00:00 -->
+- [[#726](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/726)] Fix documentation for organization-policy module ([averbuks](https://github.com/averbuks)) <!-- 2022-07-10 07:12:47+00:00 -->
+- [[#722](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/722)] OrgPolicy module (factory) using new org-policy API, #698 ([averbuks](https://github.com/averbuks)) <!-- 2022-07-08 13:38:42+00:00 -->
+- [[#695](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/695)] Modified reserved IP address outputs in net-glb module ([apichick](https://github.com/apichick)) <!-- 2022-07-01 17:13:10+00:00 -->
+- [[#709](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/709)] Fix incompatibility between logging and monitor config/service arguments in GKE module ([psabhishekgoogle](https://github.com/psabhishekgoogle)) <!-- 2022-06-29 12:34:13+00:00 -->
+- [[#708](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/708)] Fix incompatibility between backup and autopilot in GKE module ([ludoo](https://github.com/ludoo)) <!-- 2022-06-28 16:53:55+00:00 -->
+- [[#707](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/707)] Fix addons for autopilot clusters and add specific tests in GKE module ([juliocc](https://github.com/juliocc)) <!-- 2022-06-28 10:41:46+00:00 -->
+- [[#706](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/706)] Bump providers versions and pin versions for tests ([juliocc](https://github.com/juliocc)) <!-- 2022-06-28 08:33:42+00:00 -->
+- [[#704](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/704)] Add `consumer_accept_list` to `apigee-x-instance` ([juliocc](https://github.com/juliocc)) <!-- 2022-06-27 09:52:16+00:00 -->
+- [[#696](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/696)] Added missing image in GLB and Cloud Armor example ([apichick](https://github.com/apichick)) <!-- 2022-06-23 06:08:56+00:00 -->
+- [[#689](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/689)] New binary authorization module and example ([apichick](https://github.com/apichick)) <!-- 2022-06-18 10:18:58+00:00 -->
+- [[#686](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/686)] Revert "Binary authorization module and example" ([ludoo](https://github.com/ludoo)) <!-- 2022-06-17 10:32:42+00:00 -->
+- [[#683](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/683)] Binary authorization module and example ([apichick](https://github.com/apichick)) <!-- 2022-06-17 09:36:26+00:00 -->
+- [[#684](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/684)] Cloud function module: add support for secrets ([ludoo](https://github.com/ludoo)) <!-- 2022-06-16 14:34:47+00:00 -->
+
+### TOOLS
+
+- [[#763](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/763)] Changelog generator ([ludoo](https://github.com/ludoo)) <!-- 2022-08-02 09:45:06+00:00 -->
+- [[#762](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/762)] Update changelog on pull request merge ([ludoo](https://github.com/ludoo)) <!-- 2022-07-30 17:04:00+00:00 -->
+- [[#680](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/680)] Toos: fix Raise ValueError when check_names detects overlong names ([27Bslash6](https://github.com/27Bslash6)) <!-- 2022-06-16 08:01:59+00:00 -->
+- [[#672](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/672)] Module attribution and version updater tool, plus release automation ([rosmo](https://github.com/rosmo)) <!-- 2022-06-09 11:40:50+00:00 -->
 
-<!-- BEGIN CHANGELOG -->
-- [[#761](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/761)] Fix gke hub module features condition (ludoo)
-- [[#760](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/760)] GKE hub module refactor (ludoo)
-- [[#759](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/759)] FIX: Missing value to format principalSet (imp14a)
-- [[#756](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/756)] Changed cluster id sensitivity (apichick)
-- [[#757](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/757)] Remove key_algorithm from glb/ilb-l7 examples (ludoo)
-- [[#753](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/753)] Add support for IAM bindings on service accounts to project factory (ludoo)
-- [[#752](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/752)] Added dependency on google_compute_shared_vpc_host_project.shared_vpc… (apichick)
-- [[#747](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/747)] Added gkehub.googleapis.com to jit services (apichick)
-- [[#746](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/746)] Update multi region cloud SQL markdown file (bensadikgoogle)
-- [[#745](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/745)] FIX: 00-cicd stage - Gitlab and Github providers (imp14a)
-- [[#744](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/744)] Fixed issue with missing project reference in Cloud DNS data source  (rosmo)
-- [[#741](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/741)] Added servicemesh feature to GKE hub and included fleet robot service… (apichick)
-- [[#737](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/737)] Move Cloud Run VPC Connector annotations to template metadata (#735) (sethmoon)
-- [[#734](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/734)] FAST: Use spot VMs for test VM and for NVAs (sruffilli)
-- [[#733](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/733)] Fix dataset name (juliocc)
-- [[#732](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/732)] Add support for topic message duration to pubsub module (ludoo)
-- [[#731](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/731)] Avoid setting empty IAM binding in subnet factory (ludoo)
-- [[#712](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/712)] AD FS example (apichick)
-- [[#730](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/730)] FAST - add billing IAM for billing group (ludoo)
-- [[#729](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/729)] Fix connector create logic in cloud run module (ludoo)
-- [[#655](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/655)] New example for a data playground Terraform setup (aymanfarhat)
-- [[#726](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/726)] Fix documentation for organization-policy module (averbuks)
-- [[#722](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/722)] OrgPolicy module (factory) using new org-policy API, #698 (averbuks)
-- [[#721](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/721)] FAST: Resman: Update billing.tf (sruffilli)
-- [[#716](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/716)] FAST - added missing format argument in branch-pf-dev-sa-cicd (mgfeller)
-- [[#715](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/715)] Fix optional service accounts in networking stages (ludoo)
-- [[#695](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/695)] Modified reserved IP address outputs (apichick)
-- [[#711](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/711)] FAST - updated several stage READMEs about usage of *.auto.tfvars files (mgfeller)
-- [[#709](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/709)] when using managed prometheus or passing monitoring config there is e… (psabhishekgoogle)
-- [[#702](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/702)] FAST - trigger GitHub workflow also on PR synchronize event (mgfeller)
-- [[#708](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/708)] Fix #705 (ludoo)
-- [[#703](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/703)] FAST: configuration switches for features (ludoo)
-- [[#707](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/707)] Fix addons for autopilot clusters, add tests for gke-cluster. (juliocc)
-- [[#706](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/706)] Bump providers versions and pin versions for tests (juliocc)
-- [[#704](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/704)] Add `consumer_accept_list` to `apigee-x-instance` (juliocc)
-- [[#692](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/692)] Fix KMS delegation role (lcaggio)
-- [[#696](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/696)] Added missing image in GLB and Cloud Armor example (apichick)
-- [[#699](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/699)] Add repository_owner to GitHub identity attributes (ludoo)
-- [[#694](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/694)] Allow creating repositories in Gitlab/GitHub via Terraform and other CI/CD improvements (rosmo)
-- [[#690](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/690)] Fix stages tfvars links (lcaggio)
-- [[#676](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/676)] Fast/group creation gif (amgoogle)
-- [[#689](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/689)] Binary authorization module and example (apichick)
-- [[#687](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/687)] Fix for fast project factory (dosti-tee)
-- [[#683](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/683)] Binary authorization module and example (apichick)
-- [[#686](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/686)] Revert "Binary authorization module and example" (ludoo)
-- [[#668](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/668)] FAST cleanup instructions (ajlopezn)
-- [[#680](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/680)] fix: Raise ValueError when check_names detects overlong names (27Bslash6)
-- [[#682](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/682)] FAST: fix CI/CD source repositories in stage 01 (imp14a)
-- [[#684](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/684)] Cloud function module: add support for secrets (ludoo)
-- [[#669](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/669)] FAST: CI/CD support for Source Repository and Cloud Build (ludoo)
-- [[#671](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/671)] FAST: Fix Gitlab WIF attributes (ludoo)
-- [[#675](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/675)] FAST: Fix audit logs when using pubsub as destination (juliocc)
-- [[#674](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/674)] FAST: Remove team folders comment from 01 variables, clarify README (ludoo)
-- [[#672](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/672)] Module attribution and version updater tool, plus release automation (rosmo)
-<!-- END CHANGELOG -->
 
 ## [16.0.0] - 2022-06-06
 
@@ -82,9 +103,6 @@ All notable changes to this project will be documented in this file.
 - fix condition in subnet factory flow logs
 - added new example on GLB and Cloud Armor
 - revamped and expanded Contributing Guide
-
-**FAST**
-
 - add support for Workload Identity Federation and CI/CD repositories
 - simplify VPN tunnel configuration in the Hub and Spoke VPN network stage
 - fix subnet YAML schema
@@ -101,9 +119,6 @@ All notable changes to this project will be documented in this file.
 - fix `tag` output on `data-catalog-policy-tag` module
 - add shared-vpc support on `gcs-to-bq-with-least-privileges`
 - new `net-ilb-l7` module
-
-**FAST**
-
 - new [02-networking-peering](fast/stages/02-networking-peering) networking stage
 - **incompatible change** the variable for PSA ranges in networking stages have changed
 
@@ -123,9 +138,6 @@ All notable changes to this project will be documented in this file.
 - **incompatible change** removed `ingress_settings` configuration option in the `cloud-functions` module.
 - new [m4ce VM example](examples/cloud-operations/vm-migration/)
 - Support for resource management tags in the `organization`, `folder`, `project`, `compute-vm`, and `kms` modules
-
-**FAST**
-
 - new [data platform](fast/stages/03-data-platform) stage 3
 - new [02-networking-nva](fast/stages/02-networking-nva) networking stage
 - allow customizing the names of custom roles
@@ -139,7 +151,7 @@ All notable changes to this project will be documented in this file.
 
 ## [13.0.0] - 2022-01-27
 
-- **initial Fabric Fast implementation**
+- **initial Fabric FAST implementation**
 - new `net-glb` module for Global External Load balancer
 - new `project-factory` module in [`examples/factories`](./examples/factories)
 - add missing service identity accounts (artifactregistry, composer) in project module
@@ -173,9 +185,9 @@ All notable changes to this project will be documented in this file.
 
 - fix cases where bridge perimeter status resources are `null` in `vpc-sc` module
 - re-release 9.0.3 as a major release as it contains breaking changes
-  - update hierarchical firewall resources to use the newer `google_compute_firewall_*` resources
-  - **incompatible change** rename `firewall_policy_attachments` to `firewall_policy_association` in the `organization` and `folder` modules
-  - **incompatible change** updated API for the `net-vpc-sc` module
+- update hierarchical firewall resources to use the newer `google_compute_firewall_*` resources
+- **incompatible change** rename `firewall_policy_attachments` to `firewall_policy_association` in the `organization` and `folder` modules
+- **incompatible change** updated API for the `net-vpc-sc` module
 
 ## [9.0.3] - 2021-12-31
 
@@ -282,7 +294,6 @@ All notable changes to this project will be documented in this file.
 ## [4.6.1] - 2021-04-01
 
 - **incompatible change** support one group per zone in the `compute-vm` module
-  the `group` output is now renamed to `groups`
 
 ## [4.6.0] - 2021-03-31
 
@@ -402,8 +413,7 @@ All notable changes to this project will be documented in this file.
 ## [3.1.1] - 2020-08-26
 
 - fix error in `project` module
-
-- **incompatible change** make HA VPN Gateway creation optional for `net-vpn-ha` module. Now an existing HA VPN Gateway can be used. Updating to the new version of the module will cause VPN Gateway recreation which can be handled by `terraform state rm/terraform import` operations.  
+- **incompatible change** make HA VPN Gateway creation optional for `net-vpn-ha` module. Now an existing HA VPN Gateway can be used. Updating to the new version of the module will cause VPN Gateway recreation which can be handled by `terraform state rm/terraform import` operations.
 
 ## [3.1.0] - 2020-08-16
 
@@ -537,14 +547,10 @@ All notable changes to this project will be documented in this file.
 - fix Cloud NAT module internal router name lookup
 - re-enable and update outputs for the foundations environments example
 - add peering route configuration for private clusters to GKE cluster module
-- **incompatible changes** in the GKE nodepool module
-  - rename `node_config_workload_metadata_config` variable to `workload_metadata_config`
-  - new default for `workload_metadata_config` is `GKE_METADATA_SERVER`
-- **incompatible change** in the `compute-vm` module
-  - removed support for MIG and the `group_manager` variable
+- **incompatible changes** in the GKE nodepool module: rename `node_config_workload_metadata_config` variable to `workload_metadata_config`, new default for `workload_metadata_config` is `GKE_METADATA_SERVER`
+- **incompatible change** in the `compute-vm` module: removed support for MIG and the `group_manager` variable
 - add `compute-mig` and `net-ilb` modules
-- **incompatible change** in `net-vpc`
-  - a new `name` attribute has been added to the `subnets` variable, allowing to directly set subnet name, to update to the new module add an extra `name = false` attribute to each subnet
+- **incompatible change** in `net-vpc`: a new `name` attribute has been added to the `subnets` variable, allowing to directly set subnet name, to update to the new module add an extra `name = false` attribute to each subnet
 
 ## [1.3.0] - 2020-04-08
 
@@ -567,72 +573,69 @@ All notable changes to this project will be documented in this file.
 
 - merge development branch with suite of new modules and end-to-end examples
 
+
 <!-- markdown-link-check-disable -->
-[Unreleased]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v16.0.0...HEAD
-[16.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v15.0.0...v16.0.0
-[15.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v14.0.0...v15.0.0
-[14.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v13.0.0...v14.0.0
-[13.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v12.0.0...v13.0.0
-[12.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v11.2.0...v12.0.0
-[11.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v11.1.0...v11.2.0
-[11.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v11.0.0...v11.1.0
-[11.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v10.0.1...v11.0.0
-[10.0.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v10.0.0...v10.0.1
-[10.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v9.0.3...v10.0.0
-[9.0.3]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v9.0.2...v9.0.3
-[9.0.2]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v9.0.0...v9.0.2
-[9.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v8.0.0...v9.0.0
-[8.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v7.0.0...v8.0.0
-[7.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v6.0.0...v7.0.0
-[6.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v5.1.0...v6.0.0
-[5.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v5.0.0...v5.1.0
-[5.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.9.0...v5.0.0
-[4.9.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.8.0...v4.9.0
-[4.8.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.7.0...v4.8.0
-[4.7.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.6.1...v4.7.0
-[4.6.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.6.0...v4.6.1
-[4.6.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.5.1...v4.6.0
-[4.5.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.5.0...v4.5.1
-[4.5.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.4.2...v4.5.0
-[4.4.2]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.4.1...v4.4.2
-[4.4.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.4.0...v4.4.1
-[4.4.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.3.0...v4.4.0
-[4.3.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.2.0...v4.3.0
-[4.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.1.0...v4.2.0
-[4.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.0.0...v4.1.0
-[4.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.5.0...v4.0.0
-[3.5.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.4.0...v3.5.0
-[3.4.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.3.0...v3.4.0
-[3.3.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.2.0...v3.3.0
-[3.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.1.1...v3.2.0
-[3.1.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.1.0...v3.1.1
-[3.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v3.0.0...v3.1.0
-[3.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.8.0...v3.0.0
-[2.8.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.7.1...v2.8.0
-[2.7.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.7.0...v2.7.1
-[2.7.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.6.0...v2.7.0
-[2.6.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.5.0...v2.6.0
-[2.5.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.4.2...v2.5.0
-[2.4.2]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.4.1...v2.4.2
-[2.4.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.4.0...v2.4.1
-[2.4.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.3.0...v2.4.0
-[2.3.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.2.0...v2.3.0
-[2.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.1.0...v2.2.0
-[2.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.0.0...v2.1.0
-[2.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.9.0...v2.0.0
-[1.9.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.8.1...v1.9.0
-[1.8.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.8.0...v1.8.1
-[1.8.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.7.0...v1.8.0
-[1.7.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.6.0...v1.7.0
-[1.6.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.5.0...v1.6.0
-[1.5.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.4.1...v1.5.0
-[1.4.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.4.0...v1.4.1
-[1.4.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.3.0...v1.4.0
-[1.3.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.2...v1.3.0
-[1.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.1...v1.2
-[1.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.0...v1.1
-[1.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v0.1...v1.0
-[#82]: https://github.com/terraform-google-modules/cloud-foundation-fabric/pull/82
-[#103]: https://github.com/terraform-google-modules/cloud-foundation-fabric/pull/103
-[#156]: https://github.com/terraform-google-modules/cloud-foundation-fabric/pull/156
-<!-- markdown-link-check-enable -->
+[Unreleased]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v16.0.0...HEAD
+[16.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v15.0.0...v16.0.0
+[15.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v14.0.0...v15.0.0
+[14.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v13.0.0...v14.0.0
+[13.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v12.0.0...v13.0.0
+[12.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v11.2.0...v12.0.0
+[11.2.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v11.1.0...v11.2.0
+[11.1.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v11.0.0...v11.1.0
+[11.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v10.0.1...v11.0.0
+[10.0.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v10.0.0...v10.0.1
+[10.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v9.0.3...v10.0.0
+[9.0.3]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v9.0.2...v9.0.3
+[9.0.2]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v9.0.0...v9.0.2
+[9.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v8.0.0...v9.0.0
+[8.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v7.0.0...v8.0.0
+[7.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v6.0.0...v7.0.0
+[6.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v5.1.0...v6.0.0
+[5.1.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v5.0.0...v5.1.0
+[5.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.9.0...v5.0.0
+[4.9.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.8.0...v4.9.0
+[4.8.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.7.0...v4.8.0
+[4.7.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.6.1...v4.7.0
+[4.6.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.6.0...v4.6.1
+[4.6.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.5.1...v4.6.0
+[4.5.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.5.0...v4.5.1
+[4.5.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.4.2...v4.5.0
+[4.4.2]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.4.1...v4.4.2
+[4.4.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.4.0...v4.4.1
+[4.4.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.3.0...v4.4.0
+[4.3.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.2.0...v4.3.0
+[4.2.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.1.0...v4.2.0
+[4.1.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v4.0.0...v4.1.0
+[4.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.5.0...v4.0.0
+[3.5.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.4.0...v3.5.0
+[3.4.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.3.0...v3.4.0
+[3.3.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.2.0...v3.3.0
+[3.2.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.1.1...v3.2.0
+[3.1.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.1.0...v3.1.1
+[3.1.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v3.0.0...v3.1.0
+[3.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.8.0...v3.0.0
+[2.8.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.7.1...v2.8.0
+[2.7.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.7.0...v2.7.1
+[2.7.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.6.0...v2.7.0
+[2.6.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.5.0...v2.6.0
+[2.5.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.4.2...v2.5.0
+[2.4.2]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.4.1...v2.4.2
+[2.4.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.4.0...v2.4.1
+[2.4.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.3.0...v2.4.0
+[2.3.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.2.0...v2.3.0
+[2.2.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.1.0...v2.2.0
+[2.1.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v2.0.0...v2.1.0
+[2.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.9.0...v2.0.0
+[1.9.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.8.1...v1.9.0
+[1.8.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.8.0...v1.8.1
+[1.8.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.7.0...v1.8.0
+[1.7.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.6.0...v1.7.0
+[1.6.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.5.0...v1.6.0
+[1.5.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.4.1...v1.5.0
+[1.4.1]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.4.0...v1.4.1
+[1.4.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.3.0...v1.4.0
+[1.3.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.2.0...v1.3.0
+[1.2.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.1.0...v1.2.0
+[1.1.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v1.0.0...v1.1.0
+[1.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v0.1...v1.0.0

examples/README.md

@@ -4,7 +4,7 @@ This section contains **[foundational examples](./foundations/)** that bootstrap
 
 Currently available examples:
 
-- **cloud operations** - [Resource tracking and remediation via Cloud Asset feeds](./cloud-operations/asset-inventory-feed-remediation), [Granular Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Granular Cloud DNS IAM for Shared VPC](./cloud-operations/dns-shared-vpc), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Packer image builder](./cloud-operations/packer-image-builder), [On-prem SA key management](./cloud-operations/onprem-sa-key-management), [TCP healthcheck for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck)
+- **cloud operations** - [Resource tracking and remediation via Cloud Asset feeds](./cloud-operations/asset-inventory-feed-remediation), [Granular Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Granular Cloud DNS IAM for Shared VPC](./cloud-operations/dns-shared-vpc), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Packer image builder](./cloud-operations/packer-image-builder), [On-prem SA key management](./cloud-operations/onprem-sa-key-management), [TCP healthcheck for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [HTTP Load Balancer with Cloud Armor](./cloud-operations/glb_and_armor)
 - **data solutions** - [GCE/GCS CMEK via centralized Cloud KMS](./data-solutions/gcs-to-bq-with-least-privileges/), [Cloud Storage to Bigquery with Cloud Dataflow with least privileges](./data-solutions/gcs-to-bq-with-least-privileges/), [Data Platform Foundations](./data-solutions/data-platform-foundations/), [SQL Server AlwaysOn availability groups example](./data-solutions/sqlserver-alwayson), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion/)
 - **factories** - [The why and the how of resource factories](./factories/README.md)
 - **foundations** - [single level hierarchy](./foundations/environments/) (environments), [multiple level hierarchy](./foundations/business-units/) (business units + environments)

examples/cloud-operations/glb_and_armor/README.md

@@ -1,48 +1,124 @@
 # HTTP Load Balancer with Cloud Armor
 
-Google Cloud HTTP(S) load balancing is implemented at the edge of Google's network in Google's points of presence (POP) around the world. User traffic directed to an HTTP(S) load balancer enters the POP closest to the user and is then load balanced over Google's global network to the closest backend that has sufficient capacity available.
+## Introduction
 
-Cloud Armor IP allowlist/denylist enable you to restrict or allow access to your HTTP(S) load balancer at the edge of the Google Cloud, as close as possible to the user and to malicious traffic. This prevents malicious users or traffic from consuming resources or entering your virtual private cloud (VPC) networks.
+This repository contains all necessary Terraform modules to build a multi-regional infrastructure with horizontally scalable managed instance group backends, HTTP load balancing and Google’s advanced WAF security tool (Cloud Armor) on top to securely deploy an application at global scale.
 
-In this lab, you configure an HTTP Load Balancer with global backends, as shown in the diagram below. Then, you stress test the Load Balancer and denylist the stress test IP with Cloud Armor.
+This tutorial is general enough to fit in a variety of use-cases, from hosting a mobile app's backend to deploy proprietary workloads at scale.
 
-![Architecture](architecture.png)
+## Use cases
 
-## Running the example
+Even though there are many ways to implement an architecture, some workloads require high compute power or specific licenses while making sure the services are secured by a managed service and highly available across multiple regions. An architecture consisting of Managed Instance Groups in multiple regions available through an HTTP Load Balancer with Cloud Armor enabled is suitable for such use-cases.
 
-Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fcloud-foundation-fabric&cloudshell_print=cloud-shell-readme.txt&cloudshell_working_dir=examples%2Fcloud-operations%2Fglb-and-armor), then go through the following steps to create resources:
+This architecture caters to multiple workloads ranging from the ones requiring compliance with specific data access restrictions to compute-specific proprietary applications with specific licensing and OS requirements. Descriptions of some possible use-cases are as follows:
 
-* `terraform init`
-* `terraform apply -var project_id=my-project-id`
+* __Proprietary OS workloads__: Some applications require specific Operating systems (enterprise grade Linux distributions for example) with specific licensing requirements or low-level access to the kernel. In such cases, since the applications cannot be containerised and horizontal scaling is required, multi-region Managed Instance Group (MIG) with custom instance images are the ideal implementation.
+* __Industry-specific applications__: Other applications may require high compute power alongside a sophisticated layer of networking security. This architecture satisfies both these requirements by promising configurable compute power on the instances backed by various features offered by Cloud Armor such as traffic restriction, DDoS protection etc.
+* __Workloads requiring GDPR compliance__: Most applications require restricting data access and usage from outside a certain region (mostly to comply with data residency requirements). This architecture caters to such workloads as Cloud Armor allows you to lock access to your workloads from various fine-grained identifiers.
+* __Medical Queuing systems__: Another great example usage for this  architecture will be applications requiring high compute power, availability and limited memory access requirements such as a medical queuing system.
+* __DDoS Protection and WAF__:  Applications and workloads exposed to the internet expose themselves to the risk of DDoS attacks. While L3/L4 and protocol based attacks are handled at Google’s edge, L7 attacks can still be effective with botnets. A setup of an external Cloud Load Balancer with Cloud Armor and appropriate WAF rules can mitigate such attacks.
+* __Geofencing__: If you want to restrict content served on your application due to licensing restrictions (similar to OTT content in the US), Geofencing allows you to create a virtual perimeter to stop the service from being accessed outside the region. The architecture of using a Cloud Load Balancer with Cloud Armor enables you to implement geofencing around your applications and services.
 
-The following outputs will be available once everything is deployed:
+## Architecture
 
-* `glb_ip_address`, containing the IPv4 address of the HTTP Load Balancer 
-* `vm_siege_external_ip`, containing the external IPv4 address of the siege VM.
+<p align="center"> <img src="architecture.png" width="700"> </p>
 
-Once done testing, you can clean up resources by running `terraform destroy`.
+The main components that we would be setting up are (to learn more about these products, click on the hyperlinks):
 
-## Testing the example
+* [Cloud Armor](https://cloud.google.com/armor) - Google Cloud Armor is the web-application firewall (WAF) and DDoS mitigation service that helps users defend their web apps and services at Google scale at the edge of Google’s network.
+* [Cloud Load Balancer](https://cloud.google.com/load-balancing) - When your app usage spikes, it is important to scale, optimize and secure the app. Cloud Load Balancing is a fully distributed solution that balances user traffic to multiple backends to avoid congestion, reduce latency and increase security. Some important features it offers that we use here are:
+  * Single global anycast IP and autoscaling - CLB acts as a frontend to all your backend instances across all regions. It provides cross-region load balancing, automatic multi-region failover and scales to support increase in resources.
+  * Global Forwarding Rule - To route traffic to different regions, global load balancers use global forwarding rules, which bind the global IP address and a single target proxy.
+  * Target Proxy - For external HTTP(S) load balancers, proxies route incoming requests to a URL map. This is essentially how you can handle the connections.
+  * URL Map - URL Maps are used to route requests to a backend service based on the rules that you define for the host and path of an incoming URL.
+  * Backend Service - A Backend Service defines CLB distributes traffic. The backend service configuration consists of a set of values - protocols to connect to backends, session settings, health checks and timeouts.
+  * Health Check - Health check is a method provided to determine if the corresponding backends respond to traffic. Health checks connect to backends on a configurable, periodic basis. Each connection attempt is called a probe. Google Cloud records the success or failure of each probe.
+* [Firewall Rules](https://cloud.google.com/vpc/docs/firewalls) - Firewall rules let you allow or deny connections to or from your VM instances based on a configuration you specify.
+* [Managed Instance Groups (MIG)](https://cloud.google.com/compute/docs/instance-groups) - Instance group is a collection of VM instances that you can manage as a single entity. MIGs allow you to operate apps and workloads on multiple identical VMs. You can also leverage the various features like autoscaling, autohealing, regional / multi-zone deployments.
 
-1. Connect to the siege VM and run the following command 
+## Costs
 
-        siege -c 250 -t150s http://$LB_IP`ß
+Pricing Estimates - We have created a sample estimate based on some usage we see from new startups looking to scale. This estimate would give you an idea of how much this deployment would essentially cost per month at this scale and you extend it to the scale you further prefer. Here's the [link](https://cloud.google.com/products/calculator/#id=3105bbf2-4ee0-4289-978e-9ab6855d37ed).
 
-2. In the Cloud Console, on the Navigation menu, click Network Services > Load balancing.
-3. Click Backends.
-4. Click http-backend.
-5. Navigate to http-lb.
-6. Click on the Monitoring tab.
-7. Monitor the Frontend Location (Total inbound traffic) between North America and the two backends for 2 to 3 minutes. At first, traffic should just be directed to us-east1-mig but as the RPS increases, traffic is also directed to europe-west1-mig. This demonstrates that by default traffic is forwarded to the closest backend but if the load is very high, traffic can be distributed across the backends.
-8. Re-run terraform as follows:
+## Setup
+
+This solution assumes you already have a project created and set up where you wish to host these resources. If not, and you would like for the project to create a new project as well,  please refer to the [github repository](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/examples/data-solutions/gcs-to-bq-with-least-privileges) for instructions.
+
+### Prerequisites
+
+* Have an [organization](https://cloud.google.com/resource-manager/docs/creating-managing-organization) set up in Google cloud.
+* Have a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) set up.
+* Have an existing [project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) with [billing enabled](https://cloud.google.com/billing/docs/how-to/modify-project).
+
+### Roles & Permissions
+
+In order to spin up this architecture, you will need to be a user with the “__Project owner__” [IAM](https://cloud.google.com/iam) role on the existing project:
+
+Note: To grant a user a role, take a look at the [Granting and Revoking Access](https://cloud.google.com/iam/docs/granting-changing-revoking-access#grant-single-role) documentation.
+
+### Spinning up the architecture
+
+#### Step 1: Cloning the repository
+
+Click on the button below, sign in if required and when the prompt appears, click on “confirm”.
+
+[<p align="center"> <img alt="Open Cloudshell" width = "300px" src="shell_button.png" /> </p>](https://goo.gle/GoCloudArmor)
+
+This will clone the repository to your cloud shell and a screen like this one will appear:
+
+![cloud_shell](cloud_shell.png)
+
+Before we deploy the architecture, you will need the following information:
+
+* The __project ID__.
+
+#### Step 2: Deploying the resources
+
+1. After cloning the repo, and going through the prerequisites, head back to the cloud shell editor.
+2. Make sure you’re in the following directory. if not, you can change your directory to it via the ‘cd’ command:
+
+       cloudshell_open/cloud-foundation-fabric/examples/cloud-operations/glb_and_armor
+
+3. Run the following command to initialize the terraform working directory:
+
+       terraform init
+
+4. Copy the following command into a console and replace __[my-project-id]__ with your project’s ID. Then run the following command to run the terraform script and create all relevant resources for this architecture:
+
+       terraform apply -var project_id=[my-project-id]
+
+The resource creation will take a few minutes… but when it’s complete, you should see an output stating the command completed successfully with a list of the created resources.
+
+__Congratulations__! You have successfully deployed an HTTP Load Balancer with two Managed Instance Group backends and Cloud Armor security.
+
+## Testing your architecture
+
+1. Connect to the siege VM using SSH (from Cloud Console or CLI) and run the following command:
+
+        siege -c 250 -t150s http://$LB_IP
+
+2. In the Cloud Console, on the Navigation menu, click __Network Services > Load balancing__.
+3. Click __Backends__, then click __http-backend__ and navigate to __http-lb__
+4. Click on the __Monitoring__ tab.
+5. Monitor the Frontend Location (Total inbound traffic) between North America and the two backends for 2 to 3 minutes. At first, traffic should just be directed to __us-east1-mig__ but as the RPS increases, traffic is also directed to __europe-west1-mig__. This demonstrates that by default traffic is forwarded to the closest backend but if the load is very high, traffic can be distributed across the backends.
+6. Now, to test the IP deny-listing, rerun terraform as follows:
 
         terraform apply -var project_id=my-project-id -var enforce_security_policy=true
 
-   Like this we have applied a security policy to denylist the IP address of the siege VM
+This, applies a security policy to denylist the IP address of the siege VM
 
-9. From the siege VM run the following command and verify that you get a 403 Forbidden error code back.
+7. To test this, from the siege VM run the following command and verify that you get a __403 Forbidden__ error code back.
 
         curl http://$LB_IP
+
+## Cleaning up your environment
+
+The easiest way to remove all the deployed resources is to run the following command in Cloud Shell:
+
+        terraform destroy
+
+The above command will delete the associated resources so there will be no billable charges made afterwards.
+
 <!-- BEGIN TFDOC -->
 
 ## Variables

examples/data-solutions/gcs-to-bq-with-least-privileges/README.md

@@ -1,140 +1,192 @@
-# Cloud Storage to Bigquery with Cloud Dataflow with least privileges
+# Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery
 
-This example creates the infrastructure needed to run a [Cloud Dataflow](https://cloud.google.com/dataflow) pipeline to import data from [GCS](https://cloud.google.com/storage) to [Bigquery](https://cloud.google.com/bigquery). The example will create different service accounts with least privileges on resources. To run the pipeline, users listed in `data_eng_principals` can impersonate all those service accounts.
+## Introduction
 
-The solution will use:
-- internal IPs for GCE and Cloud Dataflow instances
-- Cloud NAT to let resources egress to the Internet, to run system updates and install packages
-- rely on [Service Account Impersonation](https://cloud.google.com/iam/docs/impersonating-service-accounts) to avoid the use of service account keys
-- Service Accounts with least privilege on each resource
-- (Optional) CMEK encription for GCS bucket, DataFlow instances and BigQuery tables
+This repository contains the necessary Terraform modules to securely deploy a basic ETL pipeline that will dump data from a Google Cloud Storage (GCS) bucket to tables in BigQuery.
 
-The example is designed to match real-world use cases with a minimum amount of resources and some compromises listed below. It can be used as a starting point for more complex scenarios.
+An ETL pipeline is defined in three steps:
 
-This is the high level diagram:
+* Extraction: retrieving data from sources.
+* Transformation: cleaning the data, putting it into a common format, calculating other fields, taking out duplicates or erroneous records so it can be stored into a target.
+* Loading: inserting the formatted data into the target database, data store, data warehouse or data lake.
+
+You can learn more about cloud-based ETL [here](https://cloud.google.com/learn/what-is-etl).
+
+## Use cases
+
+Whether you’re transferring from another Cloud Service Provider or you’re taking your first steps into the cloud with Google Cloud, building a data pipeline sets a good foundation to begin deriving insights for your business.
+
+* __Anomaly Detection__: building data pipelines to identify cyber security threats or fraudulent transactions using machine learning (ML) models.
+* __Interactive Data Analysis__: carry out interactive data analysis with BigQuery BI Engine that enables you to analyze large and complex datasets interactively with sub-second query response time and high concurrency.
+* __Predictive Forecasting__: building solid pipelines to capture real-time data for ML modeling and using it as a forecasting engine for situations ranging from weather predictions to market forecasting.
+* __Create Machine Learning models__: using BigQueryML you can create and execute machine learning models in BigQuery using standard SQL queries. Create a variety of models pre-built into BigQuery that you train with your data.
+
+## Architecture
 
 ![GCS to Biquery High-level diagram](diagram.png "GCS to Biquery High-level diagram")
 
-## Move to real use case consideration
-In the example we implemented some compromise to keep the example minimal and easy to read. On a real word use case, you may evaluate the option to:
- - Configure a Shared-VPC
- - Use only Identity Groups to assigne roles
- - Use Authorative IAM role assignement
- - Split resources in different project: Data Landing, Data Transformation, Data Lake, ...
- - Use VPC-SC to mitigate data exfiltration
+The main components that we would be setting up are (to learn more about these products, click on the hyperlinks):
 
-## Managed resources and services
+* [Cloud Storage (GCS) bucket](https://cloud.google.com/storage/): data lake solution to store extracted raw data that must undergo some kind of transformation.
+* [Cloud Dataflow pipeline](https://cloud.google.com/dataflow): to build fully managed batch and streaming pipelines to transform data stored in GCS buckets ready for processing in the Data Warehouse using Apache Beam.
+* [BigQuery datasets and tables](https://cloud.google.com/bigquery): to store the transformed data in and query it using SQL, use it to make reports or begin training [machine learning](https://cloud.google.com/bigquery-ml/docs/introduction) models without having to take your data out.
+* [Service accounts](https://cloud.google.com/iam/docs/service-accounts) (__created with least privilege on each resource__): one for uploading data into the GCS bucket, one for Orchestration, one for Dataflow instances and one for the BigQuery tables. You can also configure users or groups of users to assign them a viewer role on the created resources and the ability to impersonate service accounts to test the Dataflow pipelines before automating them with a tool like [Cloud Composer](https://cloud.google.com/composer).
 
-This sample creates several distinct groups of resources:
+For a full list of the resources that will be created, please refer to the [github repository](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/examples/data-solutions/gcs-to-bq-with-least-privileges) for this project. If you're migrating from another Cloud Provider, refer to [this](https://cloud.google.com/free/docs/aws-azure-gcp-service-comparison) documentation to see equivalent services and comparisons in Microsoft Azure and Amazon Web Services
 
-- projects
-  - Service Project configured for GCS buckets, Dataflow instances and BigQuery tables and orchestration
-- networking
-  - VPC network
-  - One subnet
-  - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC
-- IAM
-  - One service account for uploading data into the GCS landing bucket
-  - One service account for Orchestration
-  - One service account for Dataflow instances
-  - One service account for Bigquery tables
-- GCS
-  - One bucket
-- BQ
-  - One dataset
-  - One table. Tables are defined in Terraform for the porpuse of the example. Probably, in real scenario, would handle Tables creation in a separate Terraform State or using a different tool/pipeline (for example: Dataform).
+## Costs
 
-In this example you can also configure users or group of user to assign them viewer role on the resources created and the ability to imprsonate service accounts to test dataflow pipelines before autometing them with Composer or any other orchestration systems.
+Pricing Estimates - We have created a sample estimate based on some usage we see from new startups looking to scale. This estimate would give you an idea of how much this deployment would essentially cost per month at this scale and you extend it to the scale you further prefer. Here's the [link](https://cloud.google.com/products/calculator#id=44710202-c9d4-49d5-a378-99d7dd34f5e2).
 
-## Deploy your enviroment
+## Setup
 
-We assume the identiy running the following steps has the following role:
- - `resourcemanager.projectCreator` in case a new project will be created.
- - `owner` on the project in case you use an existing project. 
+This solution assumes you already have a project created and set up where you wish to host these resources. If not, and you would like for the project to create a new project as well,  please refer to the [github repository](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/examples/data-solutions/gcs-to-bq-with-least-privileges) for instructions.
 
-Run Terraform init:
+### Prerequisites
 
-```
-$ terraform init
-```
+* Have an [organization](https://cloud.google.com/resource-manager/docs/creating-managing-organization) set up in Google cloud.
+* Have a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) set up.
+* Have an existing [project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) with [billing enabled](https://cloud.google.com/billing/docs/how-to/modify-project), we’ll call this the __service project__.
 
-Configure the Terraform variable in your `terraform.tfvars` file. You need to spefify at least the following variables:
+### Roles & Permissions
 
-```
-data_eng_principals = ["user:data-eng@domain.com"]
-project_id      = "datalake-001"
-prefix          = "prefix"
-```
+In order to spin up this architecture, you will need to be a user with the “__Project owner__” [IAM](https://cloud.google.com/iam) role on the existing project:
 
-You can run now:
+__Note__: To grant a user a role, take a look at the [Granting and Revoking Access](https://cloud.google.com/iam/docs/granting-changing-revoking-access#grant-single-role) documentation.
 
-```
-$ terraform apply
-```
+### Spinning up the architecture
 
-You should see the output of the Terraform script with resources created and some command pre-created for you to run the example following steps below.
+#### Step 1: Cloning the repository
 
-### Virtual Private Cloud (VPC) design
+Click on the button below, sign in if required and when the prompt appears, click on “confirm”.
 
-As is often the case in real-world configurations, this example accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable.
+[<p align="center"> <img alt="Open Cloudshell" width = "250" src="shell_button.png" /> </p>](https://goo.gle/GoDataPipe)
 
-If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
+This will clone the repository to your cloud shell and a screen like this one will appear:
 
-When `network_config` variable is configured, the identity running the Terraform script need to have the following roles:
-  - `roles/compute.xpnAdmin` on the host project folder or org
-  - `roles/resourcemanager.projectIamAdmin` on the host project, either with no conditions or with a condition allowing [delegated role grants](https://medium.com/google-cloud/managing-gcp-service-usage-through-delegated-role-grants-a843610f2226#:~:text=Delegated%20role%20grants%20is%20a,setIamPolicy%20permission%20on%20a%20resource.) for `roles/compute.networkUser`, `roles/composer.sharedVpcAgent`, `roles/container.hostServiceAgentUser`
+![cloud_shell](cloud_shell.png)
 
-## Test your environment with Cloud Dataflow
+Before you deploy the architecture, make sure you run the following command to move your cloudshell session into your service project:
 
-We assume all those steps are run using a user listed on `data_eng_principals`. You can authenticate as the user using the following command:
+        gcloud config set project [SERVICE_PROJECT_ID]
 
-```
-$ gcloud init
-$ gcloud auth application-default login 
-```
+Once you can see your service project id in the yellow parenthesis, you’re ready to start.
+
+Before we deploy the architecture, you will need the following information:
+
+* The __service project ID__.
+* A __unique prefix__ that you want all the deployed resources to have (for example: awesomestartup). This must be a string with no spaces or tabs.
+* A __list of Groups or Users__ with Service Account Token creator role on Service Accounts in IAM format, eg 'group:group@domain.com'.
+
+#### Step 2: Deploying the resources
+
+1. Once you have the required information, head back to the cloud shell editor. Make sure you’re in the following directory:
+
+        cloudshell_open/cloud-foundation-fabric/examples/data-solutions/gcs-to-bq-with-least-privileges
+
+2. In the editor, edit the terraform.tfvars.sample file with the variables you gathered in the step above.
+
+![editor](editor.png)
+
+* a. Fill in __data_eng_principals__ with the list of Users or Groups to impersonate service accounts.
+
+* b. Fill in __project_id__ with the service project ID.
+
+* c. Fill in the prefix with your chosen unique prefix for resources.
+
+* d. Save the file with __Ctrl(or ⌘)+S__ or by going to __File → Save__.
+
+3. Then, run the following commands:
+
+        terraform init
+        
+        terraform apply -var-file=terraform.tfvars.sample -auto-approve
+
+The resource creation will take a few minutes, at the end this is the output you should expect for successful completion along with a list of the created resources:
+
+![output](output.png)
+
+__Congratulations!__ You have successfully deployed the foundation for running your first ETL pipeline on Google Cloud.
+
+### Testing your architecture
+
+For the purpose of demonstrating how the ETL pipeline flow works, we’ve set up an example pipeline for you to run.  First of all, we assume all the steps are run using a user listed on the __data_eng_principles__ variable (or a user that belongs to one of the groups you specified). Authenticate the user using the following command and make sure your active cloudshell session is set to the __service project__:
+
+        gcloud auth application-default login
+
+Follow the instructions in the cloudshell to authenticate the user.
+
+To make the next steps easier, create two environment variables with the service project id and the prefix:
+
+        export SERVICE_PROJECT_ID=[SERVICE_PROJECT_ID]
+        export PREFIX=[PREFIX]
+
+Again, make sure you’re in the following directory:
+
+        cloudshell_open/cloud-foundation-fabric/examples/data-solutions/gcs-to-bq-with-least-privileges
 
 For the purpose of the example we will import from GCS to Bigquery a CSV file with the following structure:
 
-```
-name,surname,timestam
-```
+        name,surname,timestamp
 
-We need to create 3 file:
- - A `person.csv` file containing your data in the form `name,surname,timestam`. Here an example line `Lorenzo,Caggioni,1637771951'.
- - A `person_udf.js` containing the UDF javascript file used by the Dataflow template.
- - A `person_schema.json` file containing the table schema used to import the CSV.
- 
-You can find an example of those file in the folder `./data-demo`. You can copy the example files in the GCS bucket using the  command returned in the terraform output as `command_01_gcs`. Below an example:
+We need to create 3 files:
 
-```bash
-gsutil -i gcs-landing@PROJECT.iam.gserviceaccount.com cp data-demo/* gs://LANDING_BUCKET
-```
+* A person.csv file containing your data in the form name,surname,timestamp.  For example: `Eva,Rivarola,1637771951'.
+* A person_udf.js containing the [UDF javascript file](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions) used by the Dataflow template.
+* A person_schema.json file containing the table schema used to import the CSV.
 
-We can now run the Dataflow pipeline using the `gcloud` returned in the terraform output as `command_02_dataflow`. Below an example:
+An example of those files can be found  in the folder ./data-demo. Inside the same repository where you ran the terraform commands.
 
-```bash
-gcloud --impersonate-service-account=orch-test@PROJECT.iam.gserviceaccount.com dataflow jobs run test_batch_01 \
+You can copy the example files into the GCS bucket by running:
+
+        gsutil -i gcs-landing@$SERVICE_PROJECT_ID.iam.gserviceaccount.com cp data-demo/* gs://$PREFIX-data
+
+Once this is done, the 3 files necessary to run the Dataflow Job will have been copied to the GCS bucket that was created along with the resources.
+
+Run the following command to start the dataflow job:
+
+        gcloud --impersonate-service-account=orchestrator@$SERVICE_PROJECT_ID.iam.gserviceaccount.com dataflow jobs run test_batch_01 \
     --gcs-location gs://dataflow-templates/latest/GCS_Text_to_BigQuery \
-    --project PROJECT \
-    --region REGION \
+    --project $SERVICE_PROJECT_ID \
+    --region europe-west1 \
     --disable-public-ips \
-    --subnetwork https://www.googleapis.com/compute/v1/projects/PROJECT/regions/REGION/subnetworks/subnet \
-    --staging-location gs://PREFIX-df-tmp \
-    --service-account-email df-loading@PROJECT.iam.gserviceaccount.com \
+    --subnetwork https://www.googleapis.com/compute/v1/projects/$SERVICE_PROJECT_ID/regions/europe-west1/subnetworks/subnet \
+    --staging-location gs://$PREFIX-df-tmp\
+    --service-account-email df-loading@$SERVICE_PROJECT_ID.iam.gserviceaccount.com \
     --parameters \
-javascriptTextTransformFunctionName=transform,\
-JSONPath=gs://PREFIX-data/person_schema.json,\
-javascriptTextTransformGcsPath=gs://PREFIX-data/person_udf.js,\
-inputFilePattern=gs://PREFIX-data/person.csv,\
-outputTable=PROJECT:datalake.person,\
-bigQueryLoadingTemporaryDirectory=gs://PREFIX-df-tmp 
-```
+    javascriptTextTransformFunctionName=transform,\
+    JSONPath=gs://$PREFIX-data/person_schema.json,\
+    javascriptTextTransformGcsPath=gs://$PREFIX-data/person_udf.js,\
+    inputFilePattern=gs://$PREFIX-data/person.csv,\
+    outputTable=$SERVICE_PROJECT_ID:datalake.person,\
+    bigQueryLoadingTemporaryDirectory=gs://$PREFIX-df-tmp
 
-You can check data imported into Google BigQuery using the  command returned in the terraform output as `command_03_bq`. Below an example:
+This command will start a dataflow job called test_batch_01 that uses a Dataflow transformation script stored in the public GCS bucket:
 
-```
-bq query --use_legacy_sql=false 'SELECT * FROM `PROJECT.datalake.person` LIMIT 1000'
-```
+        gs://dataflow-templates/latest/GCS_Text_to_BigQuery.
+
+The expected output is the following:
+
+![second_output](second_output.png)
+
+Then, if you navigate to Dataflow on the console, you will see the following:
+
+![dataflow_console](dataflow_console.png)
+
+This shows the job you started from the cloudshell is currently running in Dataflow.
+If you click on the job name, you can see the job graph created and how every step of the Dataflow pipeline is moving along:
+
+![dataflow_execution](dataflow_execution.png)
+
+Once the job completes, you can navigate to BigQuery in the console and under __SERVICE_PROJECT_ID__ → datalake → person, you can see the data that was successfully imported into BigQuery through the Dataflow job.
+
+## Cleaning up your environment
+
+The easiest way to remove all the deployed resources is to run the following command in Cloud Shell:
+
+        terraform destroy -var-file=terraform.tfvars.sample -auto-approve
+  
+The above command will delete the associated resources so there will be no billable charges made afterwards.
 <!-- BEGIN TFDOC -->
 
 ## Variables

modules/project/README.md

@@ -404,7 +404,7 @@ output "compute_robot" {
 | [name](outputs.tf#L25) | Project name. |  |
 | [number](outputs.tf#L38) | Project number. |  |
 | [project_id](outputs.tf#L51) | Project id. |  |
-| [service_accounts](outputs.tf#L69) | Product robot service accounts in project. |  |
-| [sink_writer_identities](outputs.tf#L85) | Writer identities created for each sink. |  |
+| [service_accounts](outputs.tf#L70) | Product robot service accounts in project. |  |
+| [sink_writer_identities](outputs.tf#L86) | Writer identities created for each sink. |  |
 
 <!-- END TFDOC -->

modules/project/outputs.tf

@@ -58,6 +58,7 @@ output "project_id" {
     google_project_organization_policy.list,
     google_project_service.project_services,
     google_compute_shared_vpc_host_project.shared_vpc_host,
+    google_compute_shared_vpc_service_project.shared_vpc_service,
     google_compute_shared_vpc_service_project.service_projects,
     google_project_iam_member.shared_vpc_host_robots,
     google_kms_crypto_key_iam_member.service_identity_cmek,

tests/modules/project/fixture/main.tf

@@ -39,3 +39,28 @@ module "test" {
   shared_vpc_host_config     = var.shared_vpc_host_config
 }
 
+module "test-svpc-service" {
+  source              = "../../../../modules/project"
+  count               = var._test_service_project ? 1 : 0
+  name                = "test-svc"
+  billing_account     = var.billing_account
+  auto_create_network = false
+  parent              = var.parent
+  services            = var.services
+  shared_vpc_service_config = {
+    attach       = true
+    host_project = module.test.project_id
+    service_identity_iam = {
+      "roles/compute.networkUser" = [
+        "cloudservices", "container-engine"
+      ]
+      "roles/vpcaccess.user" = [
+        "cloudrun"
+      ]
+      "roles/container.hostServiceAgentUser" = [
+        "container-engine"
+      ]
+    }
+  }
+}
+

tests/modules/project/fixture/variables.tf

@@ -14,6 +14,11 @@
  * limitations under the License.
  */
 
+variable "_test_service_project" {
+  type    = bool
+  default = false
+}
+
 variable "name" {
   type    = string
   default = "my-project"

tests/modules/project/test_plan_svpc.py

@@ -0,0 +1,26 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+def test_svpc(_plan_runner):
+  "Test Shared VPC service project attachment."
+  fixture_path = os.path.join(os.path.dirname(__file__), 'fixture')
+  plan = _plan_runner(fixture_path=fixture_path, _test_service_project='true')
+  modules = [m for m in plan.root_module['child_modules']]
+  resources = [r for r in modules[0]['resources'] if r['address'] == 'module.test.google_compute_shared_vpc_host_project.shared_vpc_host[0]']
+  assert len(resources) == 1
+  print(modules[1]['resources'])
+  resources = [r for r in modules[1]['resources'] if r['address'] == 'module.test-svpc-service[0].google_compute_shared_vpc_service_project.shared_vpc_service[0]']
+  assert len(resources) == 1

tools/changelog.py

@@ -15,92 +15,182 @@
 
 import click
 import collections
-import os
-import pprint
-import re
 
 import ghapi.all
 import iso8601
 
-MARK_BEGIN = '<!-- BEGIN CHANGELOG -->'
-MARK_END = '<!-- END CHANGELOG -->'
 ORG = 'GoogleCloudPlatform'
 REPO = 'cloud-foundation-fabric'
+URL = f'https://github.com/{ORG}/{REPO}'
 
-PullRequest = collections.namedtuple('PullRequest', 'id author title merged_at')
+PullRequest = collections.namedtuple('PullRequest',
+                                     'id author title merged_at labels')
+Release = collections.namedtuple('Release', 'name published since pulls')
 
 
-def format_pull(pr):
-  url = f'https://github.com/{ORG}/{REPO}/pull/'
-  return f'- [[#{pr.id}]({url}{pr.id})] {pr.title} ({pr.author})'
+class Error(Exception):
+  pass
 
 
-def get_api(token, owner=ORG, name=REPO):
-  return ghapi.all.GhApi(owner=owner, repo=name, token=token)
-
-
-def get_pulls(token, api=None):
-  api = api or get_api(token)
-  release = api.repos.get_latest_release()
-  release_published_at = iso8601.parse_date(release.published_at)
-
+def _paginate(method, **kw):
+  'Paginate GitHub API call.'
+  page = 1
   while True:
-    page = 1
-    for item in api.pulls.list(base='master', state='closed', sort='updated',
-                               direction='desc', page=page, per_page=100):
-      try:
-        merged_at = iso8601.parse_date(item['merged_at'])
-      except iso8601.ParseError:
-        continue
-      pr = PullRequest(item['number'], item['user']['login'], item['title'],
-                       merged_at)
-      if pr.merged_at <= release_published_at:
-        page = None
-        break
-      yield pr
-    if page is None:
+    result = method(page=page, per_page=100, **kw)
+    for item in result:
+      yield item
+    if len(result) < 100:
       break
     page += 1
 
 
-def write_doc(path, snippet):
-  'Replace changelog file.'
+def changelog_load(path):
+  'Parse changelog file and return structured data.'
+  releases = []
   try:
-    doc = open(path).read()
+    with open(path) as f:
+      for l in f.readlines():
+        l = l.strip()
+        if l.startswith('## '):
+          name, _, date = l[3:].partition(' - ')
+          releases.append(Release(name[1:-1], date, None, []))
+        elif l.startswith('- '):
+          if not releases:
+            raise Error(f'Pull found with no releases: {l}')
+          releases[-1].pulls.append(l)
+    return releases
   except (IOError, OSError) as e:
-    raise SystemExit(f'Error opening {path}: {e.args[0]}')
-  m = re.search('(?sm)%s\n(.*)\n%s' % (MARK_BEGIN, MARK_END), doc)
-  if not m:
-    raise SystemExit('Mark not found.')
-  start, end = m.start(), m.end()
-  try:
-    open(path, 'w').write('\n'.join([
-        doc[:start].rstrip(),
-        f'\n{MARK_BEGIN}',
-        snippet,
-        f'{MARK_END}\n',
-        doc[end:].lstrip(),
-    ]))
-  except (IOError, OSError) as e:
-    raise SystemExit(f'Error replacing {path}: {e.args[0]}')
+    raise Error(f'Cannot open {path}: {e.args[0]}')
+
+
+def changelog_dumps(releases, overrides=None):
+  'Return formatted changelog from structured data, overriding versions.'
+  overrides = overrides or {}
+  buffer = [
+      ('# Changelog\n\n'
+       'All notable changes to this project will be documented in this file.\n')
+  ]
+  ref_buffer = ['<!-- markdown-link-check-disable -->']
+  for i, release in enumerate(releases):
+    name, published, _, pulls = release
+    prev_name = releases[i + 1].name if i + 1 < len(releases) else '0.1'
+    if name != 'Unreleased':
+      buffer.append(f'## [{name}] - {published}\n')
+      ref_buffer.append(f'[{name}]: {URL}/compare/v{prev_name}...v{name}')
+    else:
+      buffer.append(f'## [{name}]\n')
+      ref_buffer.append(f'[Unreleased]: {URL}/compare/v{prev_name}...HEAD')
+    if name in overrides:
+      buffer.append(
+          f'<!-- {overrides[name].published} < {overrides[name].since} -->\n')
+      pulls = group_pulls(overrides[name].pulls)
+      for k in sorted(pulls.keys(), key=lambda s: s or ''):
+        if k is not None:
+          buffer.append(f'### {k}\n')
+        for pull in pulls[k]:
+          buffer.append(format_pull(pull))
+        buffer.append('')
+    else:
+      for pull in pulls:
+        buffer.append(pull)
+    buffer.append('')
+  return '\n'.join(buffer + [''] + ref_buffer)
+
+
+def format_pull(pull):
+  'Format pull request.'
+  url = 'https://github.com'
+  pull_url = f'{url}/{ORG}/{REPO}/pull'
+  prefix = ''
+  if 'incompatible change' in pull.labels:
+    prefix = '**incompatible change:** '
+  return (f'- [[#{pull.id}]({pull_url}/{pull.id})] '
+          f'{prefix}'
+          f'{pull.title} '
+          f'([{pull.author}]({url}/{pull.author})) <!-- {pull.merged_at} -->')
+
+
+def group_pulls(pulls):
+  pulls.sort(key=lambda p: p.merged_at, reverse=True)
+  groups = {None: []}
+  for pull in pulls:
+    labels = [l[3:] for l in pull.labels if l.startswith('on:')]
+    if not labels:
+      groups[None].append(pull)
+      continue
+    for label in labels:
+      group = groups.setdefault(label.upper(), [])
+      group.append(pull)
+  return groups
+
+
+def get_api(token, owner=ORG, name=REPO):
+  'Get GitHub API object.'
+  return ghapi.all.GhApi(owner=owner, repo=name, token=token)
+
+
+def get_release_pulls(api, releases):
+  'Get and add pull requests for releases.'
+  i = 0
+  for p in _paginate(api.pulls.list, base='master', state='closed',
+                     sort='updated', direction='desc'):
+    try:
+      merged_at = iso8601.parse_date(p['merged_at'])
+    except iso8601.ParseError:
+      continue
+    if releases[i].published and merged_at >= releases[i].published:
+      continue
+    if releases[i].since and merged_at <= releases[i].since:
+      i += 1
+      if i == len(releases):
+        break
+    releases[i].pulls.append(
+        PullRequest(p['number'], p['user']['login'], p['title'], merged_at,
+                    [l['name'] for l in p['labels']]))
+  return releases
+
+
+def get_releases(api, filter_names=None):
+  'Get releases with optional filter on release names.'
+  Buffer = collections.namedtuple('Buffer', 'name published')
+  buffer = Buffer('Unreleased', None)
+  for r in _paginate(api.repos.list_releases):
+    published = iso8601.parse_date(r['published_at'])
+    if not filter_names or buffer.name in filter_names:
+      yield Release(buffer.name, buffer.published, published, [])
+    buffer = Buffer(r['name'], published)
+  if buffer and (not filter_names or buffer.name in filter_names):
+    yield Release(buffer.name, buffer.published, None, [])
 
 
 @click.command
-@click.option('--token', required=True, envvar='GH_TOKEN')
-@click.argument('changelog', required=False, type=click.Path(exists=True))
-def main(token, changelog=None):
-  buffer = []
+@click.option(
+    '--release', required=False, default=['Unreleased'], multiple=True,
+    help='Release to replace, specify multiple times for more than one version.'
+)
+@click.option('--token', required=True, envvar='GH_TOKEN',
+              help='GitHub API token.')
+@click.option('--write/-w', is_flag=True, required=False, default=False,
+              help='Write modified changelog file.')
+@click.argument('changelog', required=False, default='CHANGELOG.md',
+                type=click.Path(exists=True))
+def main(token, changelog='CHANGELOG.md', release=None, write=False):
+  api = get_api(token)
+  releases = [r for r in get_releases(api, release)]
+  releases = {r.name: r for r in get_release_pulls(api, releases)}
   try:
-    for pr in get_pulls(token=token):
-      buffer.append(format_pull(pr))
-  except Exception as e:
-    raise SystemExit(f'API error: {e}')
-  buffer = '\n'.join(buffer)
-  if not changelog:
-    print(buffer)
+    changelog_releases = changelog_load(changelog)
+    result = changelog_dumps(changelog_releases, releases)
+  except Error as e:
+    raise SystemExit(f'Cannot read or generate changelog: {e.args[0]}')
+  if not write:
+    print(result)
   else:
-    write_doc(changelog, buffer)
+    try:
+      open(changelog, 'w').write(result)
+    except (IOError, OSError) as e:
+      raise SystemExit('Cannot write to changelog file.')
 
 
 if __name__ == '__main__':
-  main()
+  main()