Compare commits
269 Commits
d245088fa1
...
aefd35e730
Author | SHA1 | Date |
---|---|---|
Julio Castillo | aefd35e730 | |
Julio Castillo | 959012e93e | |
Julio Castillo | 42fdd08411 | |
Eunan Hardy | 9ec38581d7 | |
Eunan Hardy | dd3a298892 | |
Eunan Hardy | 69ea9dd3be | |
Eunan Hardy | 10e3184451 | |
Ludo | ca2763754a | |
Wiktor Niesiobędzki | 182ce27143 | |
Ludo | 634e4bfbd6 | |
Ludovico Magnocavallo | 9c75aa469c | |
Ludo | cb66dd6991 | |
Ludovico Magnocavallo | 83d2976949 | |
Ludovico Magnocavallo | aecba0bbd0 | |
Julio Castillo | a6c90f2ba7 | |
Julio Castillo | ce510583bf | |
Julio Castillo | bbe778e095 | |
Julio Castillo | 24bd6179a7 | |
Julio Castillo | 702c428da4 | |
Julio Castillo | fe3a86e34b | |
Luca Prete | 311bed8e83 | |
Luca Prete | 47daeaafe1 | |
Daniel Strebel | a0bd0d4414 | |
apichick | fb003af2fa | |
Miren Esnaola | fe0a9ed138 | |
apichick | 6f3a484b9d | |
Daniel Strebel | 0906ddb8df | |
Miren Esnaola | 22b661fe93 | |
apichick | b99fd17887 | |
apichick | ad24b59e39 | |
Thinh Ha | 43e73aba9a | |
Miren Esnaola | fa34499767 | |
apichick | 1b17786634 | |
Miren Esnaola | 3b7e62990c | |
Natalia Strelkova | c8383720e2 | |
Natalia Strelkova | 29845fb1ec | |
Natalia Strelkova | 4bb51a42ce | |
Natalia Strelkova | a676c8e1de | |
Natalia Strelkova | 7acd9f624e | |
Natalia Strelkova | f09d32a703 | |
Natalia Strelkova | bcf191ab5a | |
Natalia Strelkova | 915f09461e | |
Julio Castillo | bae985513f | |
Julio Castillo | bd0fe1f928 | |
Julio Castillo | 3df98c8feb | |
apichick | 337912e1e3 | |
apichick | 75bd7d7c2d | |
Miren Esnaola | 63ac5710ac | |
Ludo | 926a21da00 | |
apichick | 4f5d953c93 | |
Ludovico Magnocavallo | 48e46515ea | |
javiergp | a2eddd0be0 | |
Miren Esnaola | 51bc31ae21 | |
apichick | b702e5794c | |
apichick | 09c9e291a0 | |
Ludovico Magnocavallo | fe3e8a3941 | |
Ludo | c82b229349 | |
Miren Esnaola | f49515495c | |
apichick | 1c3d47266f | |
Miren Esnaola | 47a9947b18 | |
Ludovico Magnocavallo | b524aa137c | |
Wiktor Niesiobędzki | 4998f1d376 | |
Ludo | fdd53624f1 | |
Ludo | a9c0d36e96 | |
Thinh Ha | 0415cf64f1 | |
Ludovico Magnocavallo | 8d7dba49cb | |
Julio Castillo | fc1373b85c | |
Julio Castillo | 32bb69caa5 | |
apichick | 6ff556755b | |
Julio Castillo | 7fb656b8ae | |
Miren Esnaola | a89e0dac31 | |
Julio Castillo | 1800d3e617 | |
Julio Castillo | 4318be6cd8 | |
Julio Castillo | 408e4eedd0 | |
Julio Castillo | e6be368367 | |
Julio Castillo | 1fdec356e2 | |
Julio Castillo | d61e7df562 | |
Julio Castillo | 4f123ccc74 | |
Ludovico Magnocavallo | 5a534edf5c | |
Simone Ruffilli | 960c84b347 | |
Simone Ruffilli | a8d9d80d5b | |
Daniel De Leo | 009c6d0f93 | |
Ludovico Magnocavallo | c918cfc800 | |
apichick | 3f554ec17c | |
Miren Esnaola | cf567b0c43 | |
lcaggio | d46312a7f1 | |
Ludo | dea6b5ef7c | |
Mikhail Filipchuk | ec74f351f5 | |
Ludovico Magnocavallo | ea800fa475 | |
Natalia Strelkova | fde498800c | |
Natalia Strelkova | a8c27b1b6e | |
apichick | cb560064ca | |
Miren Esnaola | bad5748be4 | |
Wiktor Niesiobędzki | 94daeab14b | |
apichick | 84eb5f0d72 | |
Miren Esnaola | cacb0c02e2 | |
Julio Castillo | d16b616821 | |
Julio Castillo | 74c50f95a8 | |
Ludovico Magnocavallo | aa1a79632b | |
Ludovico Magnocavallo | 4ad55923b7 | |
Ludovico Magnocavallo | fbbe668015 | |
dependabot[bot] | 69028a801e | |
Wiktor Niesiobędzki | cc0b278df3 | |
apichick | 93b2f9cba2 | |
Miren Esnaola | b79261dfa5 | |
Natalia Strelkova | bce60bb58a | |
Natalia Strelkova | e00d3bcba4 | |
simonebruzzechesse | dd37d07955 | |
Ludovico Magnocavallo | 5da7dc0796 | |
simonebruzzechesse | 973a8594b6 | |
Wiktor Niesiobędzki | 0b224a7f55 | |
Ludovico Magnocavallo | 551dc581e8 | |
Ludo | 438a3134e2 | |
Ludo | ab8705b049 | |
Ludo | 506cfe4c22 | |
Ludovico Magnocavallo | d9f7cef1e7 | |
Ludo | f7d00d8107 | |
Ludovico Magnocavallo | 154df17951 | |
Aurélien Legrand | 623c886e95 | |
Ludovico Magnocavallo | 0bc6dffce0 | |
Ludovico Magnocavallo | 86cc6eee4c | |
Ludo | aeaf1f66fd | |
Julio Castillo | ef0500ae36 | |
Ludovico Magnocavallo | a3f4908056 | |
Ludovico Magnocavallo | 31f7a9ecca | |
Julio Castillo | d49a5c0fbb | |
Thinh Ha | 10e15356ea | |
Ludovico Magnocavallo | ac9417b82d | |
Roberto Jung Drebes | 772cf813fc | |
Alejandro Leal | 337857cc19 | |
Alejandro Leal | d3739cc0ff | |
Alejandro Leal | 43b3490ef1 | |
Julio Castillo | 0f5c98916a | |
Julio Castillo | c10fdc7c12 | |
Julio Castillo | f0fbc14115 | |
Arvind Ganesh | d3e4864b57 | |
Julio Castillo | 0bd3f5ab0e | |
Ludovico Magnocavallo | 981068e442 | |
Keith Harvey | b203198441 | |
Keith Harvey | 912497a158 | |
Keith Harvey | f15e2f3509 | |
Keith Harvey | e3e05540de | |
lcaggio | 026071209c | |
Arvind Ganesh | 0b19a16593 | |
Julio Castillo | bb1eaf54f1 | |
Arvind Ganesh | f75bc321b9 | |
Julio Castillo | 925a288316 | |
Julio Castillo | 55062fe66f | |
Keith Harvey | 52ad1fee34 | |
Keith Harvey | f9742c5405 | |
Keith Harvey | 9e8323bd7d | |
Keith Harvey | b30efa8bfe | |
Keith Harvey | 7cd9083fd7 | |
lcaggio | 099ad03910 | |
Ludovico Magnocavallo | 6fcb010ff2 | |
Julio Castillo | d6aea3ff5f | |
Ludo | 501051f32d | |
apichick | e03a787b22 | |
Ludovico Magnocavallo | eace97cf4b | |
Ludo | 431fcb7847 | |
dependabot[bot] | e64c043aaa | |
Ludovico Magnocavallo | e11d1a8ea8 | |
Miren Esnaola | 2c7c41b25f | |
Ludovico Magnocavallo | 638841c8d1 | |
Wiktor Niesiobędzki | 4b6552a6f6 | |
Wiktor Niesiobędzki | 173a00d795 | |
Wiktor Niesiobędzki | 8a6555c581 | |
Ludovico Magnocavallo | ce647647cc | |
LudovicEmo | 32b347b104 | |
lcaggio | f459f86477 | |
lcaggio | 45adcf1187 | |
apichick | 6297a578bc | |
Miren Esnaola | ad25b1a31c | |
Alejandro Leal | 2f7f5ff904 | |
Alejandro Leal | 15ae95df90 | |
Jose Luis Bermudez | 9c26760773 | |
Canburak Tümer | adc1ff3fca | |
lcaggio | 261ad646a8 | |
Keith Harvey | dc964411e0 | |
Keith Harvey | a37b99501e | |
apichick | 8bb356b071 | |
apichick | f6e918ffbd | |
Julio Castillo | 00e9d20829 | |
Ludovico Magnocavallo | 5a42c15a2b | |
Albert Lloveras | 97d6e48bde | |
Albert Lloveras | 1f6f0c306d | |
Albert Lloveras | c05bc41b69 | |
Albert Lloveras | 4e18def0c6 | |
Ludovico Magnocavallo | 2830e4b9e2 | |
Albert Lloveras | 7cacc46b4b | |
Miren Esnaola | ddfab0a0e4 | |
Ludovico Magnocavallo | bd3296bc46 | |
apichick | 604ca5afd8 | |
Miren Esnaola | a424635221 | |
Keith Harvey | a68a3b55cb | |
Alejandro Leal | c3d99b8156 | |
bluPhy | 26e5662e84 | |
Ludovico Magnocavallo | 815728aca6 | |
apichick | 782cf09f3b | |
Miren Esnaola | 064d86511f | |
Wiktor Niesiobędzki | 30ae108c89 | |
Taneli Leppä | 0d0b37b599 | |
Taneli Leppä | 81285065ca | |
apichick | 19860333a7 | |
apichick | 9db280bc28 | |
Julio Castillo | 0ae22006e2 | |
Julio Castillo | fca74c2258 | |
Julio Castillo | afd804d8f5 | |
Julio Castillo | bd9aa1514e | |
Julio Castillo | bb47615493 | |
Miren Esnaola | 500179f02f | |
Julio Castillo | 6e4efda159 | |
Julio Castillo | 50d8d4b8b8 | |
Mark Schlagenhauf | ff231bd28c | |
mark1000 | 7bdb69f8f8 | |
mark1000 | ef3d988da6 | |
Julio Castillo | eddef7e5b6 | |
Julio Castillo | e900e9c951 | |
apichick | 6baf8720fa | |
apichick | d248f799d9 | |
Ludovico Magnocavallo | a7b944aa8e | |
Miren Esnaola | ebeace21dd | |
Wiktor Niesiobędzki | 6b4bca10bd | |
Mark Schlagenhauf | 359b30c141 | |
Ludo | ae73274bfb | |
Ludovico Magnocavallo | c024eca320 | |
lcaggio | 39b27ac25e | |
Ludovico Magnocavallo | 7bd6e5d57b | |
Ana Fernandez | 4876161003 | |
Ana Fernandez del Alamo | 0fe3f165ed | |
Julio Castillo | 9af4db2fa0 | |
lcaggio | 7ed197aff4 | |
Ludovico Magnocavallo | fedb894f97 | |
Ludovico Magnocavallo | ea4c00756b | |
Ludo | a93a78f4e7 | |
Ludo | c14ab4de55 | |
Luca Prete | 0d6751a5f4 | |
Ludo | eb76a60208 | |
Ludovico Magnocavallo | c439a66b27 | |
Prabha Arya | f2fe406a62 | |
Ludo | a7d694f9b0 | |
David Asaf | 43ce70e1ed | |
Julio Castillo | d4de7219c5 | |
Taneli Leppä | 87db60de1f | |
Ludovico Magnocavallo | 884cb8b4bf | |
apichick | f069562998 | |
Alejandro Leal | 7f4825feeb | |
Alejandro Leal | 6c11527762 | |
Simone Ruffilli | fda4daecff | |
Simone Ruffilli | 7f561565e7 | |
Wiktor Niesiobędzki | 3ac6ceac1e | |
Ludo | 91daad5570 | |
Julio Castillo | b1ea36b069 | |
Julio Castillo | b6ce4222d1 | |
Julio Castillo | fb121b4d08 | |
Julio Castillo | 0888cce3a5 | |
Julio Castillo | cecbd2072c | |
Julio Castillo | 563b5fa0cb | |
Julio Castillo | 1e8c58c88e | |
Julio Castillo | 7a91a7e41c | |
Julio Castillo | 868507e932 | |
Julio Castillo | 8a3c81b022 | |
Julio Castillo | 7b9e2aeb09 | |
Julio Castillo | 584a2e055b | |
Julio Castillo | e479d9815b | |
Benoît Sauvère | aa80109081 | |
Ludovico Magnocavallo | 4aa99ea829 | |
Julio Castillo | d22bf2ec6b | |
Gustavo Valverde | 00cac9148a |
|
@ -37,7 +37,7 @@ jobs:
|
||||||
- name: Set up Terraform
|
- name: Set up Terraform
|
||||||
uses: hashicorp/setup-terraform@v2
|
uses: hashicorp/setup-terraform@v2
|
||||||
with:
|
with:
|
||||||
terraform_version: 1.4.4
|
terraform_version: 1.5.1
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
|
|
|
@ -28,7 +28,7 @@ env:
|
||||||
PYTEST_ADDOPTS: "--color=yes"
|
PYTEST_ADDOPTS: "--color=yes"
|
||||||
PYTHON_VERSION: "3.10"
|
PYTHON_VERSION: "3.10"
|
||||||
TF_PLUGIN_CACHE_DIR: "/home/runner/.terraform.d/plugin-cache"
|
TF_PLUGIN_CACHE_DIR: "/home/runner/.terraform.d/plugin-cache"
|
||||||
TF_VERSION: 1.4.4
|
TF_VERSION: 1.5.1
|
||||||
TFTEST_COPY: 1
|
TFTEST_COPY: 1
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
308
CHANGELOG.md
308
CHANGELOG.md
|
@ -4,10 +4,233 @@ All notable changes to this project will be documented in this file.
|
||||||
<!-- markdownlint-disable MD024 -->
|
<!-- markdownlint-disable MD024 -->
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
<!-- None < 2023-03-24 12:44:02+00:00 -->
|
<!-- None < 2023-07-07 16:22:14+00:00 -->
|
||||||
|
|
||||||
### BLUEPRINTS
|
### BLUEPRINTS
|
||||||
|
|
||||||
|
- [[#1573](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1573)] Add information about required groups ([wiktorn](https://github.com/wiktorn)) <!-- 2023-08-06 18:27:59+00:00 -->
|
||||||
|
- [[#1572](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1572)] **incompatible change:** More module descriptions ([ludoo](https://github.com/ludoo)) <!-- 2023-08-06 09:25:45+00:00 -->
|
||||||
|
- [[#1560](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1560)] Removed unused attribute in variable of ha-vpn-over-blueprint blueprint ([apichick](https://github.com/apichick)) <!-- 2023-08-02 11:41:08+00:00 -->
|
||||||
|
- [[#1548](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1548)] Minor fixes in Vertex Ai MLOPs blueprint ([javiergp](https://github.com/javiergp)) <!-- 2023-07-31 10:52:37+00:00 -->
|
||||||
|
- [[#1547](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1547)] **incompatible change:** Peering module refactor ([ludoo](https://github.com/ludoo)) <!-- 2023-07-29 19:33:58+00:00 -->
|
||||||
|
- [[#1542](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1542)] Grant IAM rights to service identities in host project ([wiktorn](https://github.com/wiktorn)) <!-- 2023-07-29 18:07:21+00:00 -->
|
||||||
|
- [[#1536](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1536)] **incompatible change:** Update and refactor artifact registry module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-28 09:54:37+00:00 -->
|
||||||
|
- [[#1533](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1533)] Make demo pipeline append into BQ tables ([danieldeleo](https://github.com/danieldeleo)) <!-- 2023-07-27 15:38:01+00:00 -->
|
||||||
|
- [[#1510](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1510)] **incompatible change:** Refactoring of dns module ([apichick](https://github.com/apichick)) <!-- 2023-07-19 11:13:41+00:00 -->
|
||||||
|
- [[#1504](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1504)] Bump semver from 5.7.1 to 5.7.2 in /blueprints/serverless/api-gateway/function ([dependabot[bot]](https://github.com/dependabot[bot])) <!-- 2023-07-13 06:05:52+00:00 -->
|
||||||
|
- [[#1501](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1501)] Fix in nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg blueprint ([apichick](https://github.com/apichick)) <!-- 2023-07-11 10:01:54+00:00 -->
|
||||||
|
- [[#1498](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1498)] Return only bucket name of composer, not full url to dags folder ([wiktorn](https://github.com/wiktorn)) <!-- 2023-07-10 09:20:51+00:00 -->
|
||||||
|
|
||||||
|
### DOCUMENTATION
|
||||||
|
|
||||||
|
- [[#1573](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1573)] Add information about required groups ([wiktorn](https://github.com/wiktorn)) <!-- 2023-08-06 18:27:59+00:00 -->
|
||||||
|
- [[#1545](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1545)] add dataplex autodq base module ([thinhha](https://github.com/thinhha)) <!-- 2023-08-02 11:16:33+00:00 -->
|
||||||
|
- [[#1557](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1557)] renaming net-vpc-swp to net-swp ([skalolazka](https://github.com/skalolazka)) <!-- 2023-08-01 15:48:22+00:00 -->
|
||||||
|
- [[#1553](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1553)] Added module for Regional Internal Proxy Load Balancer ([apichick](https://github.com/apichick)) <!-- 2023-07-31 15:58:09+00:00 -->
|
||||||
|
- [[#1546](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1546)] **incompatible change:** rename cloud-dataplex to dataplex ([thinhha](https://github.com/thinhha)) <!-- 2023-07-29 12:31:18+00:00 -->
|
||||||
|
- [[#1506](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1506)] Document architectural decisions ([ludoo](https://github.com/ludoo)) <!-- 2023-07-13 14:15:32+00:00 -->
|
||||||
|
- [[#1500](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1500)] README: audit logs on org level go to a logging bucket, not bigquery ([skalolazka](https://github.com/skalolazka)) <!-- 2023-07-10 14:59:00+00:00 -->
|
||||||
|
|
||||||
|
### FAST
|
||||||
|
|
||||||
|
- [[#1572](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1572)] **incompatible change:** More module descriptions ([ludoo](https://github.com/ludoo)) <!-- 2023-08-06 09:25:45+00:00 -->
|
||||||
|
- [[#1566](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1566)] Remove unused ASN numbers from CloudNAT to avoid provider errors ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-08-04 08:02:12+00:00 -->
|
||||||
|
- [[#1563](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1563)] Update FAST CI/CD workflows so it can work with ID_TOKEN and Gitlab 15+ ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-08-03 16:09:45+00:00 -->
|
||||||
|
- [[#1547](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1547)] **incompatible change:** Peering module refactor ([ludoo](https://github.com/ludoo)) <!-- 2023-07-29 19:33:58+00:00 -->
|
||||||
|
- [[#1514](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1514)] Fix FAST stage links script for GKE stage ([ludoo](https://github.com/ludoo)) <!-- 2023-07-20 10:48:45+00:00 -->
|
||||||
|
- [[#1510](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1510)] **incompatible change:** Refactoring of dns module ([apichick](https://github.com/apichick)) <!-- 2023-07-19 11:13:41+00:00 -->
|
||||||
|
|
||||||
|
### MODULES
|
||||||
|
|
||||||
|
- [[#1572](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1572)] **incompatible change:** More module descriptions ([ludoo](https://github.com/ludoo)) <!-- 2023-08-06 09:25:45+00:00 -->
|
||||||
|
- [[#1569](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1569)] Add support for cost management to GKE module ([ludoo](https://github.com/ludoo)) <!-- 2023-08-05 11:46:53+00:00 -->
|
||||||
|
- [[#1568](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1568)] Add support for ipv6 to net-vpc module ([ludoo](https://github.com/ludoo)) <!-- 2023-08-05 11:07:27+00:00 -->
|
||||||
|
- [[#1567](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1567)] Allow custom route descriptions in net-vpc module ([juliocc](https://github.com/juliocc)) <!-- 2023-08-04 16:45:15+00:00 -->
|
||||||
|
- [[#1558](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1558)] feat(apigee): add retention variable ([danistrebel](https://github.com/danistrebel)) <!-- 2023-08-04 11:25:36+00:00 -->
|
||||||
|
- [[#1564](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1564)] Fixed error of inconsistent conditional result types when evaluating … ([apichick](https://github.com/apichick)) <!-- 2023-08-03 06:09:38+00:00 -->
|
||||||
|
- [[#1561](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1561)] Removed unused attribute in peer_gateway_config variable ([apichick](https://github.com/apichick)) <!-- 2023-08-02 13:38:45+00:00 -->
|
||||||
|
- [[#1545](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1545)] add dataplex autodq base module ([thinhha](https://github.com/thinhha)) <!-- 2023-08-02 11:16:33+00:00 -->
|
||||||
|
- [[#1559](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1559)] Added IPSEC_INTERCONNECT addresses to net-address module ([apichick](https://github.com/apichick)) <!-- 2023-08-02 10:28:46+00:00 -->
|
||||||
|
- [[#1557](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1557)] renaming net-vpc-swp to net-swp ([skalolazka](https://github.com/skalolazka)) <!-- 2023-08-01 15:48:22+00:00 -->
|
||||||
|
- [[#1513](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1513)] optional description in modules/net-vpc-swp ([skalolazka](https://github.com/skalolazka)) <!-- 2023-08-01 13:50:07+00:00 -->
|
||||||
|
- [[#1555](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1555)] Fix permadiff in artifact-registry ([juliocc](https://github.com/juliocc)) <!-- 2023-07-31 16:20:28+00:00 -->
|
||||||
|
- [[#1553](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1553)] Added module for Regional Internal Proxy Load Balancer ([apichick](https://github.com/apichick)) <!-- 2023-07-31 15:58:09+00:00 -->
|
||||||
|
- [[#1554](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1554)] Fix in IAM bindings of cloud function v2 module ([apichick](https://github.com/apichick)) <!-- 2023-07-31 11:22:07+00:00 -->
|
||||||
|
- [[#1551](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1551)] Fix in validation of healthchecks variable ([apichick](https://github.com/apichick)) <!-- 2023-07-31 10:13:19+00:00 -->
|
||||||
|
- [[#1552](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1552)] Add image path output to ar module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-31 09:34:02+00:00 -->
|
||||||
|
- [[#1550](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1550)] Fix in validation of healthchecks variable ([apichick](https://github.com/apichick)) <!-- 2023-07-31 08:16:52+00:00 -->
|
||||||
|
- [[#1547](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1547)] **incompatible change:** Peering module refactor ([ludoo](https://github.com/ludoo)) <!-- 2023-07-29 19:33:58+00:00 -->
|
||||||
|
- [[#1542](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1542)] Grant IAM rights to service identities in host project ([wiktorn](https://github.com/wiktorn)) <!-- 2023-07-29 18:07:21+00:00 -->
|
||||||
|
- [[#1546](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1546)] **incompatible change:** rename cloud-dataplex to dataplex ([thinhha](https://github.com/thinhha)) <!-- 2023-07-29 12:31:18+00:00 -->
|
||||||
|
- [[#1540](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1540)] Fixes in cloud function v2 module for trigger service account ([apichick](https://github.com/apichick)) <!-- 2023-07-28 15:21:18+00:00 -->
|
||||||
|
- [[#1536](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1536)] **incompatible change:** Update and refactor artifact registry module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-28 09:54:37+00:00 -->
|
||||||
|
- [[#1537](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1537)] Wrong ASN when using partner_interconnect. ([sruffilli](https://github.com/sruffilli)) <!-- 2023-07-28 09:16:04+00:00 -->
|
||||||
|
- [[#1535](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1535)] Renamed output.tf in net-vlan-attachment ([sruffilli](https://github.com/sruffilli)) <!-- 2023-07-28 08:35:48+00:00 -->
|
||||||
|
- [[#1523](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1523)] Fix in event_filters of trigger_config ([apichick](https://github.com/apichick)) <!-- 2023-07-25 14:49:07+00:00 -->
|
||||||
|
- [[#1519](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1519)] Improve Dataplex ([lcaggio](https://github.com/lcaggio)) <!-- 2023-07-24 08:52:08+00:00 -->
|
||||||
|
- [[#1520](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1520)] feat(cloudsql-instance): Add query insights config ([LiuVII](https://github.com/LiuVII)) <!-- 2023-07-21 18:14:35+00:00 -->
|
||||||
|
- [[#1512](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1512)] enable-logging flag can only be true for public zones ([apichick](https://github.com/apichick)) <!-- 2023-07-19 15:09:47+00:00 -->
|
||||||
|
- [[#1510](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1510)] **incompatible change:** Refactoring of dns module ([apichick](https://github.com/apichick)) <!-- 2023-07-19 11:13:41+00:00 -->
|
||||||
|
- [[#1509](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1509)] Add output to org module with custom constraint details and depends_on ([juliocc](https://github.com/juliocc)) <!-- 2023-07-18 08:24:39+00:00 -->
|
||||||
|
- [[#1503](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1503)] Move IAM grant to function level for trigger SA ([wiktorn](https://github.com/wiktorn)) <!-- 2023-07-12 14:19:35+00:00 -->
|
||||||
|
- [[#1479](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1479)] Update ncc-spoke-ra module to explicity request ncc hub id when referencing existing hubs ([simonebruzzechesse](https://github.com/simonebruzzechesse)) <!-- 2023-07-10 14:18:43+00:00 -->
|
||||||
|
- [[#1499](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1499)] Add support for custom description in net-address ([simonebruzzechesse](https://github.com/simonebruzzechesse)) <!-- 2023-07-10 11:04:54+00:00 -->
|
||||||
|
- [[#1497](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1497)] **incompatible change:** Implement proper support for data access logs in resource manager modules ([ludoo](https://github.com/ludoo)) <!-- 2023-07-10 08:08:03+00:00 -->
|
||||||
|
|
||||||
|
### TOOLS
|
||||||
|
|
||||||
|
- [[#1544](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1544)] Minimal tfdoc refactoring for legibility ([ludoo](https://github.com/ludoo)) <!-- 2023-07-29 09:11:31+00:00 -->
|
||||||
|
- [[#1538](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1538)] Extend tfdoc to generate TOCs ([juliocc](https://github.com/juliocc)) <!-- 2023-07-28 15:45:12+00:00 -->
|
||||||
|
- [[#1511](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1511)] Fail if run with Python below 3.10 ([wiktorn](https://github.com/wiktorn)) <!-- 2023-07-19 12:18:55+00:00 -->
|
||||||
|
|
||||||
|
## [24.0.0] - 2023-07-07
|
||||||
|
|
||||||
|
### BLUEPRINTS
|
||||||
|
|
||||||
|
- [[#1496](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1496)] Allow using a separate resource for boot disk in compute-vm module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-07 15:40:14+00:00 -->
|
||||||
|
- [[#1488](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1488)] **incompatible change:** Fix and improve quota monitor blueprint ([ludoo](https://github.com/ludoo)) <!-- 2023-07-03 07:23:49+00:00 -->
|
||||||
|
- [[#1483](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1483)] Updating a few files to fix typos ([bluPhy](https://github.com/bluPhy)) <!-- 2023-06-30 05:55:32+00:00 -->
|
||||||
|
- [[#1474](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1474)] data-platform-minimal - support web_server_network_access_control ([kthhrv](https://github.com/kthhrv)) <!-- 2023-06-29 16:38:19+00:00 -->
|
||||||
|
- [[#1482](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1482)] Add region to quota monitor cloud function ([ludoo](https://github.com/ludoo)) <!-- 2023-06-29 11:02:57+00:00 -->
|
||||||
|
- [[#1475](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1475)] Minimal Data Platform - Shared VPC ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-28 19:58:03+00:00 -->
|
||||||
|
- [[#1473](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1473)] Improve Minimal Data Platform Blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-28 07:05:49+00:00 -->
|
||||||
|
- [[#1468](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1468)] Dependencies update for API Gateway blueprint ([apichick](https://github.com/apichick)) <!-- 2023-06-27 06:30:35+00:00 -->
|
||||||
|
- [[#1469](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1469)] Bump semver and @google-cloud/storage in /blueprints/gke/binauthz/image ([dependabot[bot]](https://github.com/dependabot[bot])) <!-- 2023-06-26 13:03:48+00:00 -->
|
||||||
|
- [[#1466](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1466)] **incompatible change:** Rename network load balancer modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-26 07:50:11+00:00 -->
|
||||||
|
- [[#1459](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1459)] Add preliminary support for partner interconnect ([wiktorn](https://github.com/wiktorn)) <!-- 2023-06-26 07:22:09+00:00 -->
|
||||||
|
- [[#1464](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1464)] Fix Shielded folder README ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-23 16:38:37+00:00 -->
|
||||||
|
- [[#1458](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1458)] Fixing typos ([bluPhy](https://github.com/bluPhy)) <!-- 2023-06-23 05:12:52+00:00 -->
|
||||||
|
- [[#1455](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1455)] Match readme groups with variables file in shielded folder blueprint ([CanburakTumer](https://github.com/CanburakTumer)) <!-- 2023-06-21 09:51:33+00:00 -->
|
||||||
|
- [[#1451](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1451)] Improve Minimal Data Platform blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-20 16:47:16+00:00 -->
|
||||||
|
- [[#1454](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1454)] data-platform-minimal - 02-processing.tf typo ([kthhrv](https://github.com/kthhrv)) <!-- 2023-06-20 13:26:10+00:00 -->
|
||||||
|
- [[#1453](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1453)] data-platform-minimal - correct typo ([kthhrv](https://github.com/kthhrv)) <!-- 2023-06-20 11:12:00+00:00 -->
|
||||||
|
- [[#1450](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1450)] Split Cloud Function module in separate v1 and v2 modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-19 10:50:36+00:00 -->
|
||||||
|
- [[#1447](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1447)] **incompatible change:** Refactored apigee module and adjusted the blueprints accordingly ([apichick](https://github.com/apichick)) <!-- 2023-06-19 07:16:00+00:00 -->
|
||||||
|
- [[#1409](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1409)] Added module for Secure Web Proxy ([rosmo](https://github.com/rosmo)) <!-- 2023-06-13 07:07:18+00:00 -->
|
||||||
|
- [[#1420](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1420)] Move net-dedicated-vlan-attachment module to net-vlan-attachment and … ([apichick](https://github.com/apichick)) <!-- 2023-06-13 06:34:34+00:00 -->
|
||||||
|
- [[#1427](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1427)] Updating hub-and-spoke peering blueprint to use HA VPN. ([mark1000](https://github.com/mark1000)) <!-- 2023-06-12 20:07:53+00:00 -->
|
||||||
|
- [[#1432](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1432)] Make internal/external addresses optional in compute-vm ([juliocc](https://github.com/juliocc)) <!-- 2023-06-08 12:14:26+00:00 -->
|
||||||
|
- [[#1423](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1423)] Add support for Log Analytics on logging-bucket module and bump provider version ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-07 21:23:29+00:00 -->
|
||||||
|
- [[#1416](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1416)] Fix and improve GCS2BQ blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-06 07:06:59+00:00 -->
|
||||||
|
|
||||||
|
### DOCUMENTATION
|
||||||
|
|
||||||
|
- [[#1483](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1483)] Updating a few files to fix typos ([bluPhy](https://github.com/bluPhy)) <!-- 2023-06-30 05:55:32+00:00 -->
|
||||||
|
- [[#1473](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1473)] Improve Minimal Data Platform Blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-28 07:05:49+00:00 -->
|
||||||
|
- [[#1466](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1466)] **incompatible change:** Rename network load balancer modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-26 07:50:11+00:00 -->
|
||||||
|
- [[#1450](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1450)] Split Cloud Function module in separate v1 and v2 modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-19 10:50:36+00:00 -->
|
||||||
|
- [[#1444](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1444)] Fixing typos ([bluPhy](https://github.com/bluPhy)) <!-- 2023-06-16 06:00:57+00:00 -->
|
||||||
|
- [[#1409](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1409)] Added module for Secure Web Proxy ([rosmo](https://github.com/rosmo)) <!-- 2023-06-13 07:07:18+00:00 -->
|
||||||
|
- [[#1420](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1420)] Move net-dedicated-vlan-attachment module to net-vlan-attachment and … ([apichick](https://github.com/apichick)) <!-- 2023-06-13 06:34:34+00:00 -->
|
||||||
|
- [[#1418](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1418)] Network Load Balancer module ([ludoo](https://github.com/ludoo)) <!-- 2023-06-05 11:21:40+00:00 -->
|
||||||
|
|
||||||
|
### FAST
|
||||||
|
|
||||||
|
- [[#1470](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1470)] FAST: initial implementation of lightweight tenants ([ludoo](https://github.com/ludoo)) <!-- 2023-07-07 06:40:38+00:00 -->
|
||||||
|
- [[#1492](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1492)] Peering dashboard ([aurelienlegrand](https://github.com/aurelienlegrand)) <!-- 2023-07-05 16:25:32+00:00 -->
|
||||||
|
- [[#1487](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1487)] Fix primary gke/dp ranges in FAST subnets ([juliocc](https://github.com/juliocc)) <!-- 2023-06-30 18:11:43+00:00 -->
|
||||||
|
- [[#1478](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1478)] FAST: short_name_is_prefix for multi-tenant ([drebes](https://github.com/drebes)) <!-- 2023-06-30 07:49:26+00:00 -->
|
||||||
|
- [[#1483](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1483)] Updating a few files to fix typos ([bluPhy](https://github.com/bluPhy)) <!-- 2023-06-30 05:55:32+00:00 -->
|
||||||
|
- [[#1477](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1477)] Changing the IP range of pods from 100.64.48.0/20 to 100.65.16.0/20 Fixes #1461 ([arvindag07](https://github.com/arvindag07)) <!-- 2023-06-29 16:57:53+00:00 -->
|
||||||
|
- [[#1466](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1466)] **incompatible change:** Rename network load balancer modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-26 07:50:11+00:00 -->
|
||||||
|
- [[#1446](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1446)] fixup(project-factory): Use the correct KMS Service Agents attribute … ([alloveras](https://github.com/alloveras)) <!-- 2023-06-19 23:53:09+00:00 -->
|
||||||
|
- [[#1445](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1445)] Bump TF version in all workflow templates to coincide with module requirements ([kthhrv](https://github.com/kthhrv)) <!-- 2023-06-16 07:39:28+00:00 -->
|
||||||
|
- [[#1443](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1443)] Fix repo names check in extra FAST stage ([ludoo](https://github.com/ludoo)) <!-- 2023-06-15 16:08:57+00:00 -->
|
||||||
|
- [[#1432](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1432)] Make internal/external addresses optional in compute-vm ([juliocc](https://github.com/juliocc)) <!-- 2023-06-08 12:14:26+00:00 -->
|
||||||
|
- [[#1429](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1429)] Use RFC6598 addresses for pods and subnets ([wiktorn](https://github.com/wiktorn)) <!-- 2023-06-08 05:56:31+00:00 -->
|
||||||
|
- [[#1426](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1426)] Add custom tag support to FAST ([ludoo](https://github.com/ludoo)) <!-- 2023-06-07 22:10:27+00:00 -->
|
||||||
|
- [[#1425](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1425)] Small fixes ([ludoo](https://github.com/ludoo)) <!-- 2023-06-07 17:37:47+00:00 -->
|
||||||
|
- [[#1412](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1412)] Add VPN monitoring alerts to 2-networking and VPN usage chart ([afda16](https://github.com/afda16)) <!-- 2023-06-06 13:23:00+00:00 -->
|
||||||
|
|
||||||
|
### MODULES
|
||||||
|
|
||||||
|
- [[#1496](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1496)] Allow using a separate resource for boot disk in compute-vm module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-07 15:40:14+00:00 -->
|
||||||
|
- [[#1489](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1489)] **incompatible change:** Disable googleapi routes creation when vpc is not created in net-vpc module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-03 07:10:12+00:00 -->
|
||||||
|
- [[#1486](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1486)] Allow external editing of group instances in lb modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-30 17:34:10+00:00 -->
|
||||||
|
- [[#1480](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1480)] Add bigquery authorized resources ([thinhha](https://github.com/thinhha)) <!-- 2023-06-30 16:44:58+00:00 -->
|
||||||
|
- [[#1485](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1485)] **incompatible change:** Align group names in lb modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-30 10:18:07+00:00 -->
|
||||||
|
- [[#1456](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1456)] add missing variable image_uri ([jose-bermudez-digitalfemsa](https://github.com/jose-bermudez-digitalfemsa)) <!-- 2023-06-28 18:24:44+00:00 -->
|
||||||
|
- [[#1471](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1471)] Add ToCs to resource manager modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-27 09:36:29+00:00 -->
|
||||||
|
- [[#1466](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1466)] **incompatible change:** Rename network load balancer modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-26 07:50:11+00:00 -->
|
||||||
|
- [[#1467](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1467)] Add support for resource policies to compute vm module ([ludoo](https://github.com/ludoo)) <!-- 2023-06-26 06:49:06+00:00 -->
|
||||||
|
- [[#1439](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1439)] modules/vpc-sc: google_access_context_manager_service_perimeter add support for method_selectors/permission ([LudovicEmo](https://github.com/LudovicEmo)) <!-- 2023-06-25 06:45:37+00:00 -->
|
||||||
|
- [[#1460](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1460)] Added validation for edge_availability_domain value ([apichick](https://github.com/apichick)) <!-- 2023-06-23 10:26:14+00:00 -->
|
||||||
|
- [[#1458](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1458)] Fixing typos ([bluPhy](https://github.com/bluPhy)) <!-- 2023-06-23 05:12:52+00:00 -->
|
||||||
|
- [[#1449](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1449)] Added iam for DNS managed zone to dns module ([apichick](https://github.com/apichick)) <!-- 2023-06-20 10:35:20+00:00 -->
|
||||||
|
- [[#1452](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1452)] feat(artifact-registry): Add support for CMEK ([alloveras](https://github.com/alloveras)) <!-- 2023-06-20 08:15:40+00:00 -->
|
||||||
|
- [[#1450](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1450)] Split Cloud Function module in separate v1 and v2 modules ([ludoo](https://github.com/ludoo)) <!-- 2023-06-19 10:50:36+00:00 -->
|
||||||
|
- [[#1447](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1447)] **incompatible change:** Refactored apigee module and adjusted the blueprints accordingly ([apichick](https://github.com/apichick)) <!-- 2023-06-19 07:16:00+00:00 -->
|
||||||
|
- [[#1440](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1440)] enable_logging variable was not being used ([apichick](https://github.com/apichick)) <!-- 2023-06-15 05:31:14+00:00 -->
|
||||||
|
- [[#1436](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1436)] Ignore Cloud Run system annotations/labels ([wiktorn](https://github.com/wiktorn)) <!-- 2023-06-13 08:07:05+00:00 -->
|
||||||
|
- [[#1409](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1409)] Added module for Secure Web Proxy ([rosmo](https://github.com/rosmo)) <!-- 2023-06-13 07:07:18+00:00 -->
|
||||||
|
- [[#1420](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1420)] Move net-dedicated-vlan-attachment module to net-vlan-attachment and … ([apichick](https://github.com/apichick)) <!-- 2023-06-13 06:34:34+00:00 -->
|
||||||
|
- [[#1434](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1434)] Add subnets id output, expand net-address outputs ([juliocc](https://github.com/juliocc)) <!-- 2023-06-12 09:16:10+00:00 -->
|
||||||
|
- [[#1432](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1432)] Make internal/external addresses optional in compute-vm ([juliocc](https://github.com/juliocc)) <!-- 2023-06-08 12:14:26+00:00 -->
|
||||||
|
- [[#1428](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1428)] Added support for PSC negs in net-ilb-l7 module ([apichick](https://github.com/apichick)) <!-- 2023-06-08 10:50:27+00:00 -->
|
||||||
|
- [[#1430](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1430)] Fix serverless neg example in ILB L7 module ([ludoo](https://github.com/ludoo)) <!-- 2023-06-08 10:05:54+00:00 -->
|
||||||
|
- [[#1426](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1426)] Add custom tag support to FAST ([ludoo](https://github.com/ludoo)) <!-- 2023-06-07 22:10:27+00:00 -->
|
||||||
|
- [[#1423](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1423)] Add support for Log Analytics on logging-bucket module and bump provider version ([lcaggio](https://github.com/lcaggio)) <!-- 2023-06-07 21:23:29+00:00 -->
|
||||||
|
- [[#1425](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1425)] Small fixes ([ludoo](https://github.com/ludoo)) <!-- 2023-06-07 17:37:47+00:00 -->
|
||||||
|
- [[#1419](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1419)] Fix NLB module ([ludoo](https://github.com/ludoo)) <!-- 2023-06-05 17:42:33+00:00 -->
|
||||||
|
- [[#1418](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1418)] Network Load Balancer module ([ludoo](https://github.com/ludoo)) <!-- 2023-06-05 11:21:40+00:00 -->
|
||||||
|
|
||||||
|
### TOOLS
|
||||||
|
|
||||||
|
- [[#1496](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1496)] Allow using a separate resource for boot disk in compute-vm module ([ludoo](https://github.com/ludoo)) <!-- 2023-07-07 15:40:14+00:00 -->
|
||||||
|
|
||||||
|
## [23.0.0] - 2023-06-05
|
||||||
|
|
||||||
|
<!-- None < 2023-05-24 17:31:22+00:00 -->
|
||||||
|
|
||||||
|
### BLUEPRINTS
|
||||||
|
|
||||||
|
- [[#1410](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1410)] **incompatible change:** Ensure all modules have an `id` output ([ludoo](https://github.com/ludoo)) <!-- 2023-06-02 14:07:23+00:00 -->
|
||||||
|
- [[#1390](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1390)] HA VPN over Interconnect modules and blueprint ([sruffilli](https://github.com/sruffilli)) <!-- 2023-05-31 10:53:39+00:00 -->
|
||||||
|
|
||||||
|
### DOCUMENTATION
|
||||||
|
|
||||||
|
- [[#1403](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1403)] add alloydb module ([prabhaarya](https://github.com/prabhaarya)) <!-- 2023-06-04 10:12:32+00:00 -->
|
||||||
|
- [[#1407](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1407)] Multiple Updates in READMEs and wording ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-31 17:53:00+00:00 -->
|
||||||
|
- [[#1390](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1390)] HA VPN over Interconnect modules and blueprint ([sruffilli](https://github.com/sruffilli)) <!-- 2023-05-31 10:53:39+00:00 -->
|
||||||
|
|
||||||
|
### FAST
|
||||||
|
|
||||||
|
- [[#1414](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1414)] Bump GH TF version to coincide with module requirements ([davideasaf](https://github.com/davideasaf)) <!-- 2023-06-03 06:20:12+00:00 -->
|
||||||
|
- [[#1400](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1400)] Add default googleapi route creation to net-vpc ([juliocc](https://github.com/juliocc)) <!-- 2023-05-26 15:50:00+00:00 -->
|
||||||
|
|
||||||
|
### MODULES
|
||||||
|
|
||||||
|
- [[#1417](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1417)] Remove hardcoded description from instance groups created under net-lb-int ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-06-05 09:35:17+00:00 -->
|
||||||
|
- [[#1415](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1415)] Add notice to net-lb-int module on routes ([ludoo](https://github.com/ludoo)) <!-- 2023-06-05 07:40:34+00:00 -->
|
||||||
|
- [[#1403](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1403)] add alloydb module ([prabhaarya](https://github.com/prabhaarya)) <!-- 2023-06-04 10:12:32+00:00 -->
|
||||||
|
- [[#1411](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1411)] Add networksecurity to JIT identity list ([rosmo](https://github.com/rosmo)) <!-- 2023-06-02 16:32:53+00:00 -->
|
||||||
|
- [[#1410](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1410)] **incompatible change:** Ensure all modules have an `id` output ([ludoo](https://github.com/ludoo)) <!-- 2023-06-02 14:07:23+00:00 -->
|
||||||
|
- [[#1405](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1405)] Added comment in the dns module, saying that inbound/outbound server … ([apichick](https://github.com/apichick)) <!-- 2023-06-02 09:35:26+00:00 -->
|
||||||
|
- [[#1407](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1407)] Multiple Updates in READMEs and wording ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-31 17:53:00+00:00 -->
|
||||||
|
- [[#1390](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1390)] HA VPN over Interconnect modules and blueprint ([sruffilli](https://github.com/sruffilli)) <!-- 2023-05-31 10:53:39+00:00 -->
|
||||||
|
- [[#1404](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1404)] Add trigger SA for Cloud Run ([wiktorn](https://github.com/wiktorn)) <!-- 2023-05-30 15:08:37+00:00 -->
|
||||||
|
- [[#1400](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1400)] Add default googleapi route creation to net-vpc ([juliocc](https://github.com/juliocc)) <!-- 2023-05-26 15:50:00+00:00 -->
|
||||||
|
|
||||||
|
### TOOLS
|
||||||
|
|
||||||
|
- [[#1410](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1410)] **incompatible change:** Ensure all modules have an `id` output ([ludoo](https://github.com/ludoo)) <!-- 2023-06-02 14:07:23+00:00 -->
|
||||||
|
|
||||||
|
## [22.0.0] - 2023-05-24
|
||||||
|
<!-- 2023-05-24 17:31:22+00:00 < 2023-03-24 12:44:02+00:00 -->
|
||||||
|
|
||||||
|
### BLUEPRINTS
|
||||||
|
|
||||||
|
- [[#1389](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1389)] Bump requests from 2.28.1 to 2.31.0 in /blueprints/cloud-operations/network-dashboard/src ([dependabot[bot]](<https://github.com/dependabot[bot]>)) <!-- 2023-05-23 05:37:16+00:00 -->
|
||||||
|
- [[#1388](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1388)] Firewall Validator fix target_service_accounts ref ([afda16](https://github.com/afda16)) <!-- 2023-05-22 14:49:38+00:00 -->
|
||||||
|
- [[#1382](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1382)] chore: update mlops blueprint metadata ([bharathkkb](https://github.com/bharathkkb)) <!-- 2023-05-17 07:41:57+00:00 -->
|
||||||
|
- [[#1380](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1380)] Minimal Data Platform - Make components optional ([lcaggio](https://github.com/lcaggio)) <!-- 2023-05-16 12:08:04+00:00 -->
|
||||||
|
- [[#1378](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1378)] Updates to blueprints/data-solutions/shielded-folder ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-16 05:28:34+00:00 -->
|
||||||
|
- [[#1375](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1375)] Several updates ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-15 21:08:19+00:00 -->
|
||||||
|
- [[#1365](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1365)] feat(net-cloudnat): add toggle for independent endpoint mapping and dynamic port allocation ([JSchwerberg](https://github.com/JSchwerberg)) <!-- 2023-05-12 13:38:01+00:00 -->
|
||||||
|
- [[#1362](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1362)] Add Minimal Data Platform blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-05-08 08:25:07+00:00 -->
|
||||||
|
- [[#1364](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1364)] Cloud Run services in service projects ([juliodiez](https://github.com/juliodiez)) <!-- 2023-05-08 05:28:16+00:00 -->
|
||||||
|
- [[#1358](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1358)] update variables files for gke nodepool taints ([jackspyder](https://github.com/jackspyder)) <!-- 2023-05-05 17:42:00+00:00 -->
|
||||||
|
- [[#1359](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1359)] Blueprint metadata validator ([juliocc](https://github.com/juliocc)) <!-- 2023-05-05 15:20:15+00:00 -->
|
||||||
- [[#1355](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1355)] Fix Shielded Folder - VertexML interoperability ([lcaggio](https://github.com/lcaggio)) <!-- 2023-05-05 07:54:57+00:00 -->
|
- [[#1355](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1355)] Fix Shielded Folder - VertexML interoperability ([lcaggio](https://github.com/lcaggio)) <!-- 2023-05-05 07:54:57+00:00 -->
|
||||||
- [[#1353](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1353)] fix in IAM binding of Apigee BigQuery analytics blueprint ([apichick](https://github.com/apichick)) <!-- 2023-05-03 16:31:57+00:00 -->
|
- [[#1353](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1353)] fix in IAM binding of Apigee BigQuery analytics blueprint ([apichick](https://github.com/apichick)) <!-- 2023-05-03 16:31:57+00:00 -->
|
||||||
- [[#1346](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1346)] **incompatible change:** FAST: shorten stage 3 prefixes, enforce prefix length in stage 3s ([ludoo](https://github.com/ludoo)) <!-- 2023-05-03 05:39:41+00:00 -->
|
- [[#1346](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1346)] **incompatible change:** FAST: shorten stage 3 prefixes, enforce prefix length in stage 3s ([ludoo](https://github.com/ludoo)) <!-- 2023-05-03 05:39:41+00:00 -->
|
||||||
|
@ -33,6 +256,11 @@ All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
### DOCUMENTATION
|
### DOCUMENTATION
|
||||||
|
|
||||||
|
- [[#1393](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1393)] Update README.md ([juliocc](https://github.com/juliocc)) <!-- 2023-05-24 10:59:14+00:00 -->
|
||||||
|
- [[#1379](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1379)] Update to multiple README.md ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-16 06:11:34+00:00 -->
|
||||||
|
- [[#1375](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1375)] Several updates ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-15 21:08:19+00:00 -->
|
||||||
|
- [[#1377](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1377)] Fixed home path ([skalolazka](https://github.com/skalolazka)) <!-- 2023-05-15 11:29:02+00:00 -->
|
||||||
|
- [[#1362](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1362)] Add Minimal Data Platform blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-05-08 08:25:07+00:00 -->
|
||||||
- [[#1357](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1357)] Add module link to README ([prabhaarya](https://github.com/prabhaarya)) <!-- 2023-05-05 08:10:09+00:00 -->
|
- [[#1357](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1357)] Add module link to README ([prabhaarya](https://github.com/prabhaarya)) <!-- 2023-05-05 08:10:09+00:00 -->
|
||||||
- [[#1347](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1347)] Fix external documentation links ([bobidle](https://github.com/bobidle)) <!-- 2023-05-02 05:26:58+00:00 -->
|
- [[#1347](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1347)] Fix external documentation links ([bobidle](https://github.com/bobidle)) <!-- 2023-05-02 05:26:58+00:00 -->
|
||||||
- [[#1330](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1330)] Separating GKE Standard and Autopilot Modules ([avinashkumar1289](https://github.com/avinashkumar1289)) <!-- 2023-04-21 12:08:14+00:00 -->
|
- [[#1330](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1330)] Separating GKE Standard and Autopilot Modules ([avinashkumar1289](https://github.com/avinashkumar1289)) <!-- 2023-04-21 12:08:14+00:00 -->
|
||||||
|
@ -44,6 +272,14 @@ All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
### FAST
|
### FAST
|
||||||
|
|
||||||
|
- [[#1394](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1394)] Allow setting identities in VPC SC module egress policies ([ludoo](https://github.com/ludoo)) <!-- 2023-05-24 10:05:16+00:00 -->
|
||||||
|
- [[#1391](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1391)] fix(stages): only add sandbox SA when `sandbox` feature is enabled ([gustavovalverde](https://github.com/gustavovalverde)) <!-- 2023-05-24 05:17:35+00:00 -->
|
||||||
|
- [[#1385](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1385)] Add conditional org admin role to sandbox SA ([ludoo](https://github.com/ludoo)) <!-- 2023-05-21 08:48:41+00:00 -->
|
||||||
|
- [[#1383](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1383)] Allows groups from other orgs/domains ([drebes](https://github.com/drebes)) <!-- 2023-05-17 09:07:48+00:00 -->
|
||||||
|
- [[#1375](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1375)] Several updates ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-15 21:08:19+00:00 -->
|
||||||
|
- [[#1376](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1376)] fixed permissions for security stage SA ([alx13](https://github.com/alx13)) <!-- 2023-05-15 10:20:34+00:00 -->
|
||||||
|
- [[#1367](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1367)] fix routes priority typo ([fawzihmouda](https://github.com/fawzihmouda)) <!-- 2023-05-09 14:26:24+00:00 -->
|
||||||
|
- [[#1358](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1358)] update variables files for gke nodepool taints ([jackspyder](https://github.com/jackspyder)) <!-- 2023-05-05 17:42:00+00:00 -->
|
||||||
- [[#1352](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1352)] **incompatible change:** Switch FAST networking stages to network policies for Google domains ([ludoo](https://github.com/ludoo)) <!-- 2023-05-04 05:38:41+00:00 -->
|
- [[#1352](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1352)] **incompatible change:** Switch FAST networking stages to network policies for Google domains ([ludoo](https://github.com/ludoo)) <!-- 2023-05-04 05:38:41+00:00 -->
|
||||||
- [[#1346](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1346)] **incompatible change:** FAST: shorten stage 3 prefixes, enforce prefix length in stage 3s ([ludoo](https://github.com/ludoo)) <!-- 2023-05-03 05:39:41+00:00 -->
|
- [[#1346](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1346)] **incompatible change:** FAST: shorten stage 3 prefixes, enforce prefix length in stage 3s ([ludoo](https://github.com/ludoo)) <!-- 2023-05-03 05:39:41+00:00 -->
|
||||||
- [[#1344](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1344)] Add logging details to bootstrap outputs ([juliocc](https://github.com/juliocc)) <!-- 2023-04-27 11:27:25+00:00 -->
|
- [[#1344](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1344)] Add logging details to bootstrap outputs ([juliocc](https://github.com/juliocc)) <!-- 2023-04-27 11:27:25+00:00 -->
|
||||||
|
@ -58,7 +294,18 @@ All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
### MODULES
|
### MODULES
|
||||||
|
|
||||||
- [[#1329](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1329)] fix: Change net-glb serve_while_stale type to number ([tobbbles](https://github.com/tobbbles)) <!-- 2023-05-05 07:41:13+00:00 -->
|
- [[#1395](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1395)] allow to configure stack type in GKE autopilot ([NitriKx](https://github.com/NitriKx)) <!-- 2023-05-24 10:19:43+00:00 -->
|
||||||
|
- [[#1394](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1394)] Allow setting identities in VPC SC module egress policies ([ludoo](https://github.com/ludoo)) <!-- 2023-05-24 10:05:16+00:00 -->
|
||||||
|
- [[#1387](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1387)] Add default Cloud Build SA to project module ([juliocc](https://github.com/juliocc)) <!-- 2023-05-22 17:25:18+00:00 -->
|
||||||
|
- [[#1386](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1386)] Support CMEK encryption in logging-bucket module ([afda16](https://github.com/afda16)) <!-- 2023-05-22 14:28:16+00:00 -->
|
||||||
|
- [[#1375](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1375)] Several updates ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-15 21:08:19+00:00 -->
|
||||||
|
- [[#1372](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1372)] Cloud NAT rules support ([juliocc](https://github.com/juliocc)) <!-- 2023-05-14 13:42:34+00:00 -->
|
||||||
|
- [[#1374](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1374)] added the export_public_ip_routes variable in the net-vpc-peering mod… ([itManuel](https://github.com/itManuel)) <!-- 2023-05-14 13:29:24+00:00 -->
|
||||||
|
- [[#1373](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1373)] Made available CPUs configurable in Cloud Functions module ([apichick](https://github.com/apichick)) <!-- 2023-05-13 07:59:35+00:00 -->
|
||||||
|
- [[#1365](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1365)] feat(net-cloudnat): add toggle for independent endpoint mapping and dynamic port allocation ([JSchwerberg](https://github.com/JSchwerberg)) <!-- 2023-05-12 13:38:01+00:00 -->
|
||||||
|
- [[#1367](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1367)] fix routes priority typo ([fawzihmouda](https://github.com/fawzihmouda)) <!-- 2023-05-09 14:26:24+00:00 -->
|
||||||
|
- [[#1360](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1360)] Add support for Shared VPC in Cloud Run ([juliodiez](https://github.com/juliodiez)) <!-- 2023-05-05 18:17:49+00:00 -->
|
||||||
|
- [[#1329](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1329)] fix: Change net-lb-app-ext serve_while_stale type to number ([tobbbles](https://github.com/tobbbles)) <!-- 2023-05-05 07:41:13+00:00 -->
|
||||||
- [[#1308](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1308)] Add cloud dataplex module ([prabhaarya](https://github.com/prabhaarya)) <!-- 2023-05-05 07:26:46+00:00 -->
|
- [[#1308](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1308)] Add cloud dataplex module ([prabhaarya](https://github.com/prabhaarya)) <!-- 2023-05-05 07:26:46+00:00 -->
|
||||||
- [[#1352](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1352)] **incompatible change:** Switch FAST networking stages to network policies for Google domains ([ludoo](https://github.com/ludoo)) <!-- 2023-05-04 05:38:41+00:00 -->
|
- [[#1352](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1352)] **incompatible change:** Switch FAST networking stages to network policies for Google domains ([ludoo](https://github.com/ludoo)) <!-- 2023-05-04 05:38:41+00:00 -->
|
||||||
- [[#1349](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1349)] Enhance GKE Backup Configuration Support ([tacchino](https://github.com/tacchino)) <!-- 2023-05-02 14:59:12+00:00 -->
|
- [[#1349](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1349)] Enhance GKE Backup Configuration Support ([tacchino](https://github.com/tacchino)) <!-- 2023-05-02 14:59:12+00:00 -->
|
||||||
|
@ -90,6 +337,8 @@ All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
### TOOLS
|
### TOOLS
|
||||||
|
|
||||||
|
- [[#1375](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1375)] Several updates ([bluPhy](https://github.com/bluPhy)) <!-- 2023-05-15 21:08:19+00:00 -->
|
||||||
|
- [[#1359](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1359)] Blueprint metadata validator ([juliocc](https://github.com/juliocc)) <!-- 2023-05-05 15:20:15+00:00 -->
|
||||||
- [[#1340](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1340)] Extend tests to use lockfile if available ([juliocc](https://github.com/juliocc)) <!-- 2023-04-26 09:10:13+00:00 -->
|
- [[#1340](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1340)] Extend tests to use lockfile if available ([juliocc](https://github.com/juliocc)) <!-- 2023-04-26 09:10:13+00:00 -->
|
||||||
- [[#1339](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1339)] Deprecate plan runner fixture and all its variants ([juliocc](https://github.com/juliocc)) <!-- 2023-04-22 11:43:51+00:00 -->
|
- [[#1339](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1339)] Deprecate plan runner fixture and all its variants ([juliocc](https://github.com/juliocc)) <!-- 2023-04-22 11:43:51+00:00 -->
|
||||||
- [[#1327](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1327)] Migrate more tests ([juliocc](https://github.com/juliocc)) <!-- 2023-04-17 07:18:07+00:00 -->
|
- [[#1327](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1327)] Migrate more tests ([juliocc](https://github.com/juliocc)) <!-- 2023-04-17 07:18:07+00:00 -->
|
||||||
|
@ -120,8 +369,8 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1189](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1189)] Update healthchecker deps (dependabot alerts) ([averbuks](https://github.com/averbuks)) <!-- 2023-02-27 21:48:49+00:00 -->
|
- [[#1189](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1189)] Update healthchecker deps (dependabot alerts) ([averbuks](https://github.com/averbuks)) <!-- 2023-02-27 21:48:49+00:00 -->
|
||||||
- [[#1184](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1184)] **incompatible change:** Allow multiple peer gateways in VPN HA module ([ludoo](https://github.com/ludoo)) <!-- 2023-02-27 10:19:00+00:00 -->
|
- [[#1184](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1184)] **incompatible change:** Allow multiple peer gateways in VPN HA module ([ludoo](https://github.com/ludoo)) <!-- 2023-02-27 10:19:00+00:00 -->
|
||||||
- [[#1143](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1143)] Test blueprints from README files ([juliocc](https://github.com/juliocc)) <!-- 2023-02-27 08:57:41+00:00 -->
|
- [[#1143](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1143)] Test blueprints from README files ([juliocc](https://github.com/juliocc)) <!-- 2023-02-27 08:57:41+00:00 -->
|
||||||
- [[#1181](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1181)] Bump golang.org/x/sys from 0.0.0-20220310020820-b874c991c1a5 to 0.1.0 in /blueprints/cloud-operations/unmanaged-instances-healthcheck/function/healthchecker ([dependabot[bot]](https://github.com/dependabot[bot])) <!-- 2023-02-25 17:02:08+00:00 -->
|
- [[#1181](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1181)] Bump golang.org/x/sys from 0.0.0-20220310020820-b874c991c1a5 to 0.1.0 in /blueprints/cloud-operations/unmanaged-instances-healthcheck/function/healthchecker ([dependabot[bot]](<https://github.com/dependabot[bot]>)) <!-- 2023-02-25 17:02:08+00:00 -->
|
||||||
- [[#1180](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1180)] Bump golang.org/x/sys from 0.0.0-20220310020820-b874c991c1a5 to 0.1.0 in /blueprints/cloud-operations/unmanaged-instances-healthcheck/function/restarter ([dependabot[bot]](https://github.com/dependabot[bot])) <!-- 2023-02-25 16:47:56+00:00 -->
|
- [[#1180](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1180)] Bump golang.org/x/sys from 0.0.0-20220310020820-b874c991c1a5 to 0.1.0 in /blueprints/cloud-operations/unmanaged-instances-healthcheck/function/restarter ([dependabot[bot]](<https://github.com/dependabot[bot]>)) <!-- 2023-02-25 16:47:56+00:00 -->
|
||||||
- [[#1175](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1175)] Serverless networking program ([juliodiez](https://github.com/juliodiez)) <!-- 2023-02-25 10:15:12+00:00 -->
|
- [[#1175](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1175)] Serverless networking program ([juliodiez](https://github.com/juliodiez)) <!-- 2023-02-25 10:15:12+00:00 -->
|
||||||
- [[#1179](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1179)] Added a PSC GCLB example ([cgrotz](https://github.com/cgrotz)) <!-- 2023-02-24 20:09:31+00:00 -->
|
- [[#1179](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1179)] Added a PSC GCLB example ([cgrotz](https://github.com/cgrotz)) <!-- 2023-02-24 20:09:31+00:00 -->
|
||||||
- [[#1165](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1165)] DataPlatform: Support project creation ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-23 11:10:44+00:00 -->
|
- [[#1165](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1165)] DataPlatform: Support project creation ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-23 11:10:44+00:00 -->
|
||||||
|
@ -194,7 +443,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1269](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1269)] Ignore changes to metadata.0.annotations in Cloud Run module ([juliocc](https://github.com/juliocc)) <!-- 2023-03-21 11:21:59+00:00 -->
|
- [[#1269](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1269)] Ignore changes to metadata.0.annotations in Cloud Run module ([juliocc](https://github.com/juliocc)) <!-- 2023-03-21 11:21:59+00:00 -->
|
||||||
- [[#1267](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1267)] Improvements to NCC-RA spoke module. ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-03-21 07:07:44+00:00 -->
|
- [[#1267](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1267)] Improvements to NCC-RA spoke module. ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-03-21 07:07:44+00:00 -->
|
||||||
- [[#1268](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1268)] simple-nva: add ability to parse BGP configs as strings. ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-03-21 06:41:13+00:00 -->
|
- [[#1268](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1268)] simple-nva: add ability to parse BGP configs as strings. ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-03-21 06:41:13+00:00 -->
|
||||||
- [[#1258](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1258)] Add backend service names to outputs for net-glb and net-ilb-l7 ([rosmo](https://github.com/rosmo)) <!-- 2023-03-17 10:40:11+00:00 -->
|
- [[#1258](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1258)] Add backend service names to outputs for net-lb-app-ext and net-lb-app-int ([rosmo](https://github.com/rosmo)) <!-- 2023-03-17 10:40:11+00:00 -->
|
||||||
- [[#1259](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1259)] Add support for `iam_additive` and simplify factory interface in net VPC module ([ludoo](https://github.com/ludoo)) <!-- 2023-03-17 10:12:35+00:00 -->
|
- [[#1259](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1259)] Add support for `iam_additive` and simplify factory interface in net VPC module ([ludoo](https://github.com/ludoo)) <!-- 2023-03-17 10:12:35+00:00 -->
|
||||||
- [[#1255](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1255)] **incompatible change:** Change `target_vpcs` variable in firewall policy module to support dynamic values ([ludoo](https://github.com/ludoo)) <!-- 2023-03-17 07:14:10+00:00 -->
|
- [[#1255](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1255)] **incompatible change:** Change `target_vpcs` variable in firewall policy module to support dynamic values ([ludoo](https://github.com/ludoo)) <!-- 2023-03-17 07:14:10+00:00 -->
|
||||||
- [[#1256](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1256)] **incompatible change:** Pin local provider ([ludoo](https://github.com/ludoo)) <!-- 2023-03-16 10:59:07+00:00 -->
|
- [[#1256](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1256)] **incompatible change:** Pin local provider ([ludoo](https://github.com/ludoo)) <!-- 2023-03-16 10:59:07+00:00 -->
|
||||||
|
@ -202,7 +451,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1241](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1241)] **incompatible change:** Allow using existing boot disk in compute-vm module ([ludoo](https://github.com/ludoo)) <!-- 2023-03-12 09:54:00+00:00 -->
|
- [[#1241](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1241)] **incompatible change:** Allow using existing boot disk in compute-vm module ([ludoo](https://github.com/ludoo)) <!-- 2023-03-12 09:54:00+00:00 -->
|
||||||
- [[#1239](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1239)] Allow overriding name in net-vpc subnet factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-11 08:30:43+00:00 -->
|
- [[#1239](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1239)] Allow overriding name in net-vpc subnet factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-11 08:30:43+00:00 -->
|
||||||
- [[#1226](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1226)] Fix policy_based_routing.sh script on simple-nva module ([simonebruzzechesse](https://github.com/simonebruzzechesse)) <!-- 2023-03-10 17:36:08+00:00 -->
|
- [[#1226](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1226)] Fix policy_based_routing.sh script on simple-nva module ([simonebruzzechesse](https://github.com/simonebruzzechesse)) <!-- 2023-03-10 17:36:08+00:00 -->
|
||||||
- [[#1234](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1234)] Fixed connection tracking configuration on LB backend in net-ilb module ([simonebruzzechesse](https://github.com/simonebruzzechesse)) <!-- 2023-03-10 14:25:30+00:00 -->
|
- [[#1234](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1234)] Fixed connection tracking configuration on LB backend in net-lb-int module ([simonebruzzechesse](https://github.com/simonebruzzechesse)) <!-- 2023-03-10 14:25:30+00:00 -->
|
||||||
- [[#1232](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1232)] Network firewall policy module ([ludoo](https://github.com/ludoo)) <!-- 2023-03-10 08:21:50+00:00 -->
|
- [[#1232](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1232)] Network firewall policy module ([ludoo](https://github.com/ludoo)) <!-- 2023-03-10 08:21:50+00:00 -->
|
||||||
- [[#1219](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1219)] Network Connectivity Center module ([juliodiez](https://github.com/juliodiez)) <!-- 2023-03-09 15:01:51+00:00 -->
|
- [[#1219](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1219)] Network Connectivity Center module ([juliodiez](https://github.com/juliodiez)) <!-- 2023-03-09 15:01:51+00:00 -->
|
||||||
- [[#1227](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1227)] Add CMEK support on BQML blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-09 09:12:50+00:00 -->
|
- [[#1227](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1227)] Add CMEK support on BQML blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-09 09:12:50+00:00 -->
|
||||||
|
@ -211,12 +460,12 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
|
- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
|
||||||
- [[#1206](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1206)] Dataproc module. Fix output. ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-02 12:59:19+00:00 -->
|
- [[#1206](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1206)] Dataproc module. Fix output. ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-02 12:59:19+00:00 -->
|
||||||
- [[#1205](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1205)] Fix issue with GKE cluster notifications topic & static output for pubsub module ([rosmo](https://github.com/rosmo)) <!-- 2023-03-02 10:43:40+00:00 -->
|
- [[#1205](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1205)] Fix issue with GKE cluster notifications topic & static output for pubsub module ([rosmo](https://github.com/rosmo)) <!-- 2023-03-02 10:43:40+00:00 -->
|
||||||
- [[#1204](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1204)] Fix url_redirect issue on net-glb module ([erabusi](https://github.com/erabusi)) <!-- 2023-03-02 06:51:40+00:00 -->
|
- [[#1204](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1204)] Fix url_redirect issue on net-lb-app-ext module ([erabusi](https://github.com/erabusi)) <!-- 2023-03-02 06:51:40+00:00 -->
|
||||||
- [[#1199](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1199)] [Dataproc module] Fix Variables ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-01 11:16:11+00:00 -->
|
- [[#1199](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1199)] [Dataproc module] Fix Variables ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-01 11:16:11+00:00 -->
|
||||||
- [[#1200](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1200)] Add test for #1197 ([juliocc](https://github.com/juliocc)) <!-- 2023-03-01 09:15:13+00:00 -->
|
- [[#1200](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1200)] Add test for #1197 ([juliocc](https://github.com/juliocc)) <!-- 2023-03-01 09:15:13+00:00 -->
|
||||||
- [[#1198](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1198)] Fix secondary ranges in net-vpc readme ([ludoo](https://github.com/ludoo)) <!-- 2023-03-01 07:08:08+00:00 -->
|
- [[#1198](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1198)] Fix secondary ranges in net-vpc readme ([ludoo](https://github.com/ludoo)) <!-- 2023-03-01 07:08:08+00:00 -->
|
||||||
- [[#1196](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1196)] Fix compute-vm:CloudKMS test for provider>=4.54.0 ([dan-farmer](https://github.com/dan-farmer)) <!-- 2023-02-28 15:53:41+00:00 -->
|
- [[#1196](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1196)] Fix compute-vm:CloudKMS test for provider>=4.54.0 ([dan-farmer](https://github.com/dan-farmer)) <!-- 2023-02-28 15:53:41+00:00 -->
|
||||||
- [[#1194](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1194)] Fix HTTPS health check mismapped to HTTP in compute-mig and net-ilb modules ([jogoldberg](https://github.com/jogoldberg)) <!-- 2023-02-28 14:48:13+00:00 -->
|
- [[#1194](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1194)] Fix HTTPS health check mismapped to HTTP in compute-mig and net-lb-int modules ([jogoldberg](https://github.com/jogoldberg)) <!-- 2023-02-28 14:48:13+00:00 -->
|
||||||
- [[#1192](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1192)] Dataproc module: Fix outputs ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-28 10:47:23+00:00 -->
|
- [[#1192](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1192)] Dataproc module: Fix outputs ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-28 10:47:23+00:00 -->
|
||||||
- [[#1190](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1190)] Dataproc Module ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-28 06:45:41+00:00 -->
|
- [[#1190](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1190)] Dataproc Module ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-28 06:45:41+00:00 -->
|
||||||
- [[#1191](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1191)] Fix external gateway in VPN HA module ([ludoo](https://github.com/ludoo)) <!-- 2023-02-27 23:46:51+00:00 -->
|
- [[#1191](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1191)] Fix external gateway in VPN HA module ([ludoo](https://github.com/ludoo)) <!-- 2023-02-27 23:46:51+00:00 -->
|
||||||
|
@ -235,7 +484,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1160](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1160)] Allow additive IAM grants by robots name ([wiktorn](https://github.com/wiktorn)) <!-- 2023-02-16 13:39:21+00:00 -->
|
- [[#1160](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1160)] Allow additive IAM grants by robots name ([wiktorn](https://github.com/wiktorn)) <!-- 2023-02-16 13:39:21+00:00 -->
|
||||||
- [[#1158](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1158)] changed pod_range reference to include secondary_pod_range issue #1157 ([chemapolo](https://github.com/chemapolo)) <!-- 2023-02-15 05:28:48+00:00 -->
|
- [[#1158](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1158)] changed pod_range reference to include secondary_pod_range issue #1157 ([chemapolo](https://github.com/chemapolo)) <!-- 2023-02-15 05:28:48+00:00 -->
|
||||||
- [[#1156](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1156)] Add 'max_time_travel_hours ' support on BQ module ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-14 08:10:12+00:00 -->
|
- [[#1156](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1156)] Add 'max_time_travel_hours ' support on BQ module ([lcaggio](https://github.com/lcaggio)) <!-- 2023-02-14 08:10:12+00:00 -->
|
||||||
- [[#1151](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1151)] Add example about referencing existing MIGs to net-ilb module readme ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-02-11 16:45:16+00:00 -->
|
- [[#1151](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1151)] Add example about referencing existing MIGs to net-lb-int module readme ([LucaPrete](https://github.com/LucaPrete)) <!-- 2023-02-11 16:45:16+00:00 -->
|
||||||
- [[#1149](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1149)] Add documentation about JIT-ed service accounts ([wiktorn](https://github.com/wiktorn)) <!-- 2023-02-11 14:52:47+00:00 -->
|
- [[#1149](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1149)] Add documentation about JIT-ed service accounts ([wiktorn](https://github.com/wiktorn)) <!-- 2023-02-11 14:52:47+00:00 -->
|
||||||
- [[#1131](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1131)] Add Autopilot Support for cluster_autoscaling Configuration in GKE Module ([tacchino](https://github.com/tacchino)) <!-- 2023-02-10 12:31:57+00:00 -->
|
- [[#1131](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1131)] Add Autopilot Support for cluster_autoscaling Configuration in GKE Module ([tacchino](https://github.com/tacchino)) <!-- 2023-02-10 12:31:57+00:00 -->
|
||||||
- [[#1140](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1140)] CloudSQL Backup Configuration: Support Point In Time Recovery ([tacchino](https://github.com/tacchino)) <!-- 2023-02-10 11:24:50+00:00 -->
|
- [[#1140](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1140)] CloudSQL Backup Configuration: Support Point In Time Recovery ([tacchino](https://github.com/tacchino)) <!-- 2023-02-10 11:24:50+00:00 -->
|
||||||
|
@ -267,7 +516,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1119](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1119)] **incompatible change:** Multi-Cluster Ingress gateway api config ([wiktorn](https://github.com/wiktorn)) <!-- 2023-01-31 13:16:52+00:00 -->
|
- [[#1119](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1119)] **incompatible change:** Multi-Cluster Ingress gateway api config ([wiktorn](https://github.com/wiktorn)) <!-- 2023-01-31 13:16:52+00:00 -->
|
||||||
- [[#1111](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1111)] **incompatible change:** In the apigee module now both the /22 and /28 peering IP ranges are p… ([apichick](https://github.com/apichick)) <!-- 2023-01-31 10:46:38+00:00 -->
|
- [[#1111](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1111)] **incompatible change:** In the apigee module now both the /22 and /28 peering IP ranges are p… ([apichick](https://github.com/apichick)) <!-- 2023-01-31 10:46:38+00:00 -->
|
||||||
- [[#1106](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1106)] Network Dashboard: PSA support for Filestore and Memorystore ([aurelienlegrand](https://github.com/aurelienlegrand)) <!-- 2023-01-25 15:02:31+00:00 -->
|
- [[#1106](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1106)] Network Dashboard: PSA support for Filestore and Memorystore ([aurelienlegrand](https://github.com/aurelienlegrand)) <!-- 2023-01-25 15:02:31+00:00 -->
|
||||||
- [[#1110](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1110)] Bump cookiejar from 2.1.3 to 2.1.4 in /blueprints/apigee/bigquery-analytics/functions/export ([dependabot[bot]](https://github.com/dependabot[bot])) <!-- 2023-01-24 15:07:12+00:00 -->
|
- [[#1110](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1110)] Bump cookiejar from 2.1.3 to 2.1.4 in /blueprints/apigee/bigquery-analytics/functions/export ([dependabot[bot]](<https://github.com/dependabot[bot]>)) <!-- 2023-01-24 15:07:12+00:00 -->
|
||||||
- [[#1097](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1097)] Use terraform resource to activate Anthos Service Mesh ([wiktorn](https://github.com/wiktorn)) <!-- 2023-01-23 08:25:31+00:00 -->
|
- [[#1097](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1097)] Use terraform resource to activate Anthos Service Mesh ([wiktorn](https://github.com/wiktorn)) <!-- 2023-01-23 08:25:31+00:00 -->
|
||||||
- [[#1104](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1104)] Updated apigee hybrid for gke README ([apichick](https://github.com/apichick)) <!-- 2023-01-22 10:34:48+00:00 -->
|
- [[#1104](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1104)] Updated apigee hybrid for gke README ([apichick](https://github.com/apichick)) <!-- 2023-01-22 10:34:48+00:00 -->
|
||||||
- [[#1107](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1107)] Check linting for Python dashboard files ([ludoo](https://github.com/ludoo)) <!-- 2023-01-21 16:17:52+00:00 -->
|
- [[#1107](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1107)] Check linting for Python dashboard files ([ludoo](https://github.com/ludoo)) <!-- 2023-01-21 16:17:52+00:00 -->
|
||||||
|
@ -349,9 +598,9 @@ All notable changes to this project will be documented in this file.
|
||||||
### BLUEPRINTS
|
### BLUEPRINTS
|
||||||
|
|
||||||
- [[#1045](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1045)] Assorted module fixes ([ludoo](https://github.com/ludoo)) <!-- 2022-12-10 14:40:15+00:00 -->
|
- [[#1045](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1045)] Assorted module fixes ([ludoo](https://github.com/ludoo)) <!-- 2022-12-10 14:40:15+00:00 -->
|
||||||
- [[#1044](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1044)] **incompatible change:** Refactor net-glb module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-08 16:35:45+00:00 -->
|
- [[#1044](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1044)] **incompatible change:** Refactor net-lb-app-ext module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-08 16:35:45+00:00 -->
|
||||||
- [[#982](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/982)] Adding Secondary IP Utilization calculation ([brianhmj](https://github.com/brianhmj)) <!-- 2022-12-07 10:45:21+00:00 -->
|
- [[#982](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/982)] Adding Secondary IP Utilization calculation ([brianhmj](https://github.com/brianhmj)) <!-- 2022-12-07 10:45:21+00:00 -->
|
||||||
- [[#1037](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1037)] Bump qs and formidable in /blueprints/cloud-operations/apigee/functions/export ([dependabot[bot]](https://github.com/dependabot[bot])) <!-- 2022-12-06 15:43:35+00:00 -->
|
- [[#1037](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1037)] Bump qs and formidable in /blueprints/cloud-operations/apigee/functions/export ([dependabot[bot]](<https://github.com/dependabot[bot]>)) <!-- 2022-12-06 15:43:35+00:00 -->
|
||||||
- [[#1034](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1034)] feat(blueprints): get audience from tfc environment variable ([Thomgrus](https://github.com/Thomgrus)) <!-- 2022-12-05 20:15:31+00:00 -->
|
- [[#1034](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1034)] feat(blueprints): get audience from tfc environment variable ([Thomgrus](https://github.com/Thomgrus)) <!-- 2022-12-05 20:15:31+00:00 -->
|
||||||
- [[#1024](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1024)] Fix Apigee PAYG environment node config ([g-greatdevaks](https://github.com/g-greatdevaks)) <!-- 2022-11-29 13:08:12+00:00 -->
|
- [[#1024](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1024)] Fix Apigee PAYG environment node config ([g-greatdevaks](https://github.com/g-greatdevaks)) <!-- 2022-11-29 13:08:12+00:00 -->
|
||||||
- [[#1019](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1019)] Added endpoint attachments to Apigee module ([apichick](https://github.com/apichick)) <!-- 2022-11-28 16:53:27+00:00 -->
|
- [[#1019](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1019)] Added endpoint attachments to Apigee module ([apichick](https://github.com/apichick)) <!-- 2022-11-28 16:53:27+00:00 -->
|
||||||
|
@ -413,7 +662,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1048](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1048)] Document new testing approach ([ludoo](https://github.com/ludoo)) <!-- 2022-12-12 19:59:47+00:00 -->
|
- [[#1048](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1048)] Document new testing approach ([ludoo](https://github.com/ludoo)) <!-- 2022-12-12 19:59:47+00:00 -->
|
||||||
- [[#1045](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1045)] Assorted module fixes ([ludoo](https://github.com/ludoo)) <!-- 2022-12-10 14:40:15+00:00 -->
|
- [[#1045](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1045)] Assorted module fixes ([ludoo](https://github.com/ludoo)) <!-- 2022-12-10 14:40:15+00:00 -->
|
||||||
- [[#1014](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1014)] Update typos in `net-vpc-firewall` README.md ([aymanfarhat](https://github.com/aymanfarhat)) <!-- 2022-12-08 16:48:26+00:00 -->
|
- [[#1014](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1014)] Update typos in `net-vpc-firewall` README.md ([aymanfarhat](https://github.com/aymanfarhat)) <!-- 2022-12-08 16:48:26+00:00 -->
|
||||||
- [[#1044](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1044)] **incompatible change:** Refactor net-glb module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-08 16:35:45+00:00 -->
|
- [[#1044](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1044)] **incompatible change:** Refactor net-lb-app-ext module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-08 16:35:45+00:00 -->
|
||||||
- [[#1009](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1009)] Fix encryption in Data Playground blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2022-11-25 15:19:02+00:00 -->
|
- [[#1009](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1009)] Fix encryption in Data Playground blueprint ([lcaggio](https://github.com/lcaggio)) <!-- 2022-11-25 15:19:02+00:00 -->
|
||||||
- [[#1006](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1006)] Add settings for autoscaling to Bigtable module. ([iht](https://github.com/iht)) <!-- 2022-11-24 15:59:32+00:00 -->
|
- [[#1006](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1006)] Add settings for autoscaling to Bigtable module. ([iht](https://github.com/iht)) <!-- 2022-11-24 15:59:32+00:00 -->
|
||||||
- [[#1007](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1007)] fast README, one line fix: 00-cicd stage got moved to extras/ ([skalolazka](https://github.com/skalolazka)) <!-- 2022-11-23 15:31:01+00:00 -->
|
- [[#1007](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1007)] fast README, one line fix: 00-cicd stage got moved to extras/ ([skalolazka](https://github.com/skalolazka)) <!-- 2022-11-23 15:31:01+00:00 -->
|
||||||
|
@ -470,7 +719,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1045](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1045)] Assorted module fixes ([ludoo](https://github.com/ludoo)) <!-- 2022-12-10 14:40:15+00:00 -->
|
- [[#1045](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1045)] Assorted module fixes ([ludoo](https://github.com/ludoo)) <!-- 2022-12-10 14:40:15+00:00 -->
|
||||||
- [[#1040](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1040)] Fix name in google_pubsub_schema resource ([VictorCavalcanteLG](https://github.com/VictorCavalcanteLG)) <!-- 2022-12-08 17:25:36+00:00 -->
|
- [[#1040](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1040)] Fix name in google_pubsub_schema resource ([VictorCavalcanteLG](https://github.com/VictorCavalcanteLG)) <!-- 2022-12-08 17:25:36+00:00 -->
|
||||||
- [[#1043](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1043)] added reverse lookup feature to module dns #1042 ([chemapolo](https://github.com/chemapolo)) <!-- 2022-12-08 17:13:05+00:00 -->
|
- [[#1043](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1043)] added reverse lookup feature to module dns #1042 ([chemapolo](https://github.com/chemapolo)) <!-- 2022-12-08 17:13:05+00:00 -->
|
||||||
- [[#1044](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1044)] **incompatible change:** Refactor net-glb module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-08 16:35:45+00:00 -->
|
- [[#1044](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1044)] **incompatible change:** Refactor net-lb-app-ext module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-08 16:35:45+00:00 -->
|
||||||
- [[#1036](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1036)] **incompatible change:** Fix status ingress/egress policies in vpc-sc module ([ludoo](https://github.com/ludoo)) <!-- 2022-12-05 08:00:01+00:00 -->
|
- [[#1036](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1036)] **incompatible change:** Fix status ingress/egress policies in vpc-sc module ([ludoo](https://github.com/ludoo)) <!-- 2022-12-05 08:00:01+00:00 -->
|
||||||
- [[#1033](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1033)] strongSwan: switch base image to debian-slim ([kunzese](https://github.com/kunzese)) <!-- 2022-12-02 12:11:02+00:00 -->
|
- [[#1033](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1033)] strongSwan: switch base image to debian-slim ([kunzese](https://github.com/kunzese)) <!-- 2022-12-02 12:11:02+00:00 -->
|
||||||
- [[#1026](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1026)] add lifecycle ignore_changes for apigee PAYG env ([g-greatdevaks](https://github.com/g-greatdevaks)) <!-- 2022-12-01 10:38:19+00:00 -->
|
- [[#1026](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1026)] add lifecycle ignore_changes for apigee PAYG env ([g-greatdevaks](https://github.com/g-greatdevaks)) <!-- 2022-12-01 10:38:19+00:00 -->
|
||||||
|
@ -486,7 +735,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#1006](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1006)] Add settings for autoscaling to Bigtable module. ([iht](https://github.com/iht)) <!-- 2022-11-24 15:59:32+00:00 -->
|
- [[#1006](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1006)] Add settings for autoscaling to Bigtable module. ([iht](https://github.com/iht)) <!-- 2022-11-24 15:59:32+00:00 -->
|
||||||
- [[#999](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/999)] Default nodepool creation fix ([astianseb](https://github.com/astianseb)) <!-- 2022-11-22 18:17:58+00:00 -->
|
- [[#999](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/999)] Default nodepool creation fix ([astianseb](https://github.com/astianseb)) <!-- 2022-11-22 18:17:58+00:00 -->
|
||||||
- [[#1005](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1005)] Only set partitioned table when sink type is bigquery ([juliocc](https://github.com/juliocc)) <!-- 2022-11-22 16:13:53+00:00 -->
|
- [[#1005](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1005)] Only set partitioned table when sink type is bigquery ([juliocc](https://github.com/juliocc)) <!-- 2022-11-22 16:13:53+00:00 -->
|
||||||
- [[#997](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/997)] Add BigQuery subcriptions to Pubsub module. ([iht](https://github.com/iht)) <!-- 2022-11-21 17:26:52+00:00 -->
|
- [[#997](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/997)] Add BigQuery subscriptions to Pubsub module. ([iht](https://github.com/iht)) <!-- 2022-11-21 17:26:52+00:00 -->
|
||||||
- [[#995](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/995)] Push container images to GitHub instead of Google Container Registry ([kunzese](https://github.com/kunzese)) <!-- 2022-11-21 14:53:52+00:00 -->
|
- [[#995](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/995)] Push container images to GitHub instead of Google Container Registry ([kunzese](https://github.com/kunzese)) <!-- 2022-11-21 14:53:52+00:00 -->
|
||||||
- [[#994](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/994)] Add schemas to Pubsub topic module. ([iht](https://github.com/iht)) <!-- 2022-11-20 16:56:03+00:00 -->
|
- [[#994](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/994)] Add schemas to Pubsub topic module. ([iht](https://github.com/iht)) <!-- 2022-11-20 16:56:03+00:00 -->
|
||||||
- [[#979](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/979)] Add network tags support to the organization module ([LucaPrete](https://github.com/LucaPrete)) <!-- 2022-11-18 14:56:29+00:00 -->
|
- [[#979](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/979)] Add network tags support to the organization module ([LucaPrete](https://github.com/LucaPrete)) <!-- 2022-11-18 14:56:29+00:00 -->
|
||||||
|
@ -499,7 +748,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#978](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/978)] Worker pool support for `cloud-function` ([maunope](https://github.com/maunope)) <!-- 2022-11-15 16:38:42+00:00 -->
|
- [[#978](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/978)] Worker pool support for `cloud-function` ([maunope](https://github.com/maunope)) <!-- 2022-11-15 16:38:42+00:00 -->
|
||||||
- [[#977](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/977)] Replace Docker's `gcplogs` driver with the GCP COS logging agent ([kunzese](https://github.com/kunzese)) <!-- 2022-11-15 12:19:52+00:00 -->
|
- [[#977](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/977)] Replace Docker's `gcplogs` driver with the GCP COS logging agent ([kunzese](https://github.com/kunzese)) <!-- 2022-11-15 12:19:52+00:00 -->
|
||||||
- [[#975](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/975)] Add validation for health check port specification to ILB L7 module ([ludoo](https://github.com/ludoo)) <!-- 2022-11-14 15:20:01+00:00 -->
|
- [[#975](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/975)] Add validation for health check port specification to ILB L7 module ([ludoo](https://github.com/ludoo)) <!-- 2022-11-14 15:20:01+00:00 -->
|
||||||
- [[#974](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/974)] **incompatible change:** Refactor net-ilb-l7 module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-11-14 13:39:00+00:00 -->
|
- [[#974](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/974)] **incompatible change:** Refactor net-lb-app-int module for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-11-14 13:39:00+00:00 -->
|
||||||
- [[#970](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/970)] Update logging sinks to tf1.3 in resman modules ([juliocc](https://github.com/juliocc)) <!-- 2022-11-12 18:36:59+00:00 -->
|
- [[#970](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/970)] Update logging sinks to tf1.3 in resman modules ([juliocc](https://github.com/juliocc)) <!-- 2022-11-12 18:36:59+00:00 -->
|
||||||
- [[#969](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/969)] Update folder and project org policy tests ([juliocc](https://github.com/juliocc)) <!-- 2022-11-11 17:01:26+00:00 -->
|
- [[#969](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/969)] Update folder and project org policy tests ([juliocc](https://github.com/juliocc)) <!-- 2022-11-11 17:01:26+00:00 -->
|
||||||
- [[#964](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/964)] prefix variable consistency across modules ([skalolazka](https://github.com/skalolazka)) <!-- 2022-11-11 13:38:51+00:00 -->
|
- [[#964](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/964)] prefix variable consistency across modules ([skalolazka](https://github.com/skalolazka)) <!-- 2022-11-11 13:38:51+00:00 -->
|
||||||
|
@ -649,7 +898,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#805](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/805)] Change `modules/project` service_config default ([juliocc](https://github.com/juliocc)) <!-- 2022-09-09 07:54:31+00:00 -->
|
- [[#805](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/805)] Change `modules/project` service_config default ([juliocc](https://github.com/juliocc)) <!-- 2022-09-09 07:54:31+00:00 -->
|
||||||
- [[#787](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/787)] Support manager role in cloud identity group module ([lcaggio](https://github.com/lcaggio)) <!-- 2022-08-31 10:29:05+00:00 -->
|
- [[#787](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/787)] Support manager role in cloud identity group module ([lcaggio](https://github.com/lcaggio)) <!-- 2022-08-31 10:29:05+00:00 -->
|
||||||
- [[#786](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/786)] Secret manager flag sensitive output ([ddaluka](https://github.com/ddaluka)) <!-- 2022-08-29 11:22:52+00:00 -->
|
- [[#786](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/786)] Secret manager flag sensitive output ([ddaluka](https://github.com/ddaluka)) <!-- 2022-08-29 11:22:52+00:00 -->
|
||||||
- [[#775](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/775)] net-glb: Added support for regional external HTTP(s) load balancing ([rosmo](https://github.com/rosmo)) <!-- 2022-08-27 20:58:11+00:00 -->
|
- [[#775](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/775)] net-lb-app-ext: Added support for regional external HTTP(s) load balancing ([rosmo](https://github.com/rosmo)) <!-- 2022-08-27 20:58:11+00:00 -->
|
||||||
- [[#784](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/784)] fix envoy-traffic-director config for xDS v3 ([drebes](https://github.com/drebes)) <!-- 2022-08-24 14:34:33+00:00 -->
|
- [[#784](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/784)] fix envoy-traffic-director config for xDS v3 ([drebes](https://github.com/drebes)) <!-- 2022-08-24 14:34:33+00:00 -->
|
||||||
- [[#785](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/785)] nginx-tls module ([drebes](https://github.com/drebes)) <!-- 2022-08-24 14:20:36+00:00 -->
|
- [[#785](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/785)] nginx-tls module ([drebes](https://github.com/drebes)) <!-- 2022-08-24 14:20:36+00:00 -->
|
||||||
- [[#783](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/783)] fix service unit indent on cloud-config-container module ([drebes](https://github.com/drebes)) <!-- 2022-08-24 07:38:48+00:00 -->
|
- [[#783](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/783)] fix service unit indent on cloud-config-container module ([drebes](https://github.com/drebes)) <!-- 2022-08-24 07:38:48+00:00 -->
|
||||||
|
@ -674,7 +923,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- [[#729](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/729)] Fix connector create logic in cloud run module ([ludoo](https://github.com/ludoo)) <!-- 2022-07-10 09:34:42+00:00 -->
|
- [[#729](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/729)] Fix connector create logic in cloud run module ([ludoo](https://github.com/ludoo)) <!-- 2022-07-10 09:34:42+00:00 -->
|
||||||
- [[#726](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/726)] Fix documentation for organization-policy module ([averbuks](https://github.com/averbuks)) <!-- 2022-07-10 07:12:47+00:00 -->
|
- [[#726](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/726)] Fix documentation for organization-policy module ([averbuks](https://github.com/averbuks)) <!-- 2022-07-10 07:12:47+00:00 -->
|
||||||
- [[#722](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/722)] OrgPolicy module (factory) using new org-policy API, #698 ([averbuks](https://github.com/averbuks)) <!-- 2022-07-08 13:38:42+00:00 -->
|
- [[#722](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/722)] OrgPolicy module (factory) using new org-policy API, #698 ([averbuks](https://github.com/averbuks)) <!-- 2022-07-08 13:38:42+00:00 -->
|
||||||
- [[#695](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/695)] Modified reserved IP address outputs in net-glb module ([apichick](https://github.com/apichick)) <!-- 2022-07-01 17:13:10+00:00 -->
|
- [[#695](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/695)] Modified reserved IP address outputs in net-lb-app-ext module ([apichick](https://github.com/apichick)) <!-- 2022-07-01 17:13:10+00:00 -->
|
||||||
- [[#709](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/709)] Fix incompatibility between logging and monitor config/service arguments in GKE module ([psabhishekgoogle](https://github.com/psabhishekgoogle)) <!-- 2022-06-29 12:34:13+00:00 -->
|
- [[#709](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/709)] Fix incompatibility between logging and monitor config/service arguments in GKE module ([psabhishekgoogle](https://github.com/psabhishekgoogle)) <!-- 2022-06-29 12:34:13+00:00 -->
|
||||||
- [[#708](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/708)] Fix incompatibility between backup and autopilot in GKE module ([ludoo](https://github.com/ludoo)) <!-- 2022-06-28 16:53:55+00:00 -->
|
- [[#708](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/708)] Fix incompatibility between backup and autopilot in GKE module ([ludoo](https://github.com/ludoo)) <!-- 2022-06-28 16:53:55+00:00 -->
|
||||||
- [[#707](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/707)] Fix addons for autopilot clusters and add specific tests in GKE module ([juliocc](https://github.com/juliocc)) <!-- 2022-06-28 10:41:46+00:00 -->
|
- [[#707](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/707)] Fix addons for autopilot clusters and add specific tests in GKE module ([juliocc](https://github.com/juliocc)) <!-- 2022-06-28 10:41:46+00:00 -->
|
||||||
|
@ -732,7 +981,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- optionally turn off gcplogs driver in COS modules
|
- optionally turn off gcplogs driver in COS modules
|
||||||
- fix `tag` output on `data-catalog-policy-tag` module
|
- fix `tag` output on `data-catalog-policy-tag` module
|
||||||
- add shared-vpc support on `gcs-to-bq-with-least-privileges`
|
- add shared-vpc support on `gcs-to-bq-with-least-privileges`
|
||||||
- new `net-ilb-l7` module
|
- new `net-lb-app-int` module
|
||||||
- new `02-networking-peering` networking stage
|
- new `02-networking-peering` networking stage
|
||||||
- **incompatible change** the variable for PSA ranges in networking stages have changed
|
- **incompatible change** the variable for PSA ranges in networking stages have changed
|
||||||
|
|
||||||
|
@ -758,15 +1007,15 @@ All notable changes to this project will be documented in this file.
|
||||||
- added `environment` and `context` resource management tags
|
- added `environment` and `context` resource management tags
|
||||||
- use resource management tags to restrict scope of roles/orgpolicy.policyAdmin
|
- use resource management tags to restrict scope of roles/orgpolicy.policyAdmin
|
||||||
- use `xpnServiceAdmin` (custom role) for stage 3 service accounts that need to attach to a shared VPC
|
- use `xpnServiceAdmin` (custom role) for stage 3 service accounts that need to attach to a shared VPC
|
||||||
- simplify and standarize ourputs from each stage
|
- simplify and standardize ourputs from each stage
|
||||||
- standarize names of projects, service accounts and buckets
|
- standardize names of projects, service accounts and buckets
|
||||||
- swtich to folder-level `xpnAdmin` and `xpnServiceAdmin`
|
- switch to folder-level `xpnAdmin` and `xpnServiceAdmin`
|
||||||
- moved networking projects to folder matching their enviroments
|
- moved networking projects to folder matching their environments
|
||||||
|
|
||||||
## [13.0.0] - 2022-01-27
|
## [13.0.0] - 2022-01-27
|
||||||
|
|
||||||
- **initial Fabric FAST implementation**
|
- **initial Fabric FAST implementation**
|
||||||
- new `net-glb` module for Global External Load balancer
|
- new `net-lb-app-ext` module for Global External Load balancer
|
||||||
- new `project-factory` module in [`blueprints/factories`](./blueprints/factories)
|
- new `project-factory` module in [`blueprints/factories`](./blueprints/factories)
|
||||||
- add missing service identity accounts (artifactregistry, composer) in project module
|
- add missing service identity accounts (artifactregistry, composer) in project module
|
||||||
- new "Cloud Storage to Bigquery with Cloud Dataflow with least privileges" example
|
- new "Cloud Storage to Bigquery with Cloud Dataflow with least privileges" example
|
||||||
|
@ -1097,7 +1346,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- **incompatible change** routes in the `net-vpc` module now interpolate the VPC name to ensure uniqueness, upgrading from a previous version will drop and recreate routes
|
- **incompatible change** routes in the `net-vpc` module now interpolate the VPC name to ensure uniqueness, upgrading from a previous version will drop and recreate routes
|
||||||
- the top-level `docker-images` folder has been moved inside `modules/cloud-config-container/onprem`
|
- the top-level `docker-images` folder has been moved inside `modules/cloud-config-container/onprem`
|
||||||
- `dns_keys` output added to the `dns` module
|
- `dns_keys` output added to the `dns` module
|
||||||
- add `group-config` variable, `groups` and `group_self_links` outputs to `net-ilb` module to allow creating ILBs for externally managed instances
|
- add `group-config` variable, `groups` and `group_self_links` outputs to `net-lb-int` module to allow creating ILBs for externally managed instances
|
||||||
- make the IAM bindings depend on the compute instance in the `compute-vm` module
|
- make the IAM bindings depend on the compute instance in the `compute-vm` module
|
||||||
|
|
||||||
## [2.0.0] - 2020-06-11
|
## [2.0.0] - 2020-06-11
|
||||||
|
@ -1145,7 +1394,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- **incompatible change** subnets in the `net-vpc` modules are now passed as a list instead of map, and all related variables for IAM and flow logs use `region/name` instead of `name` keys; it's now possible to have the same subnet name in different regions
|
- **incompatible change** subnets in the `net-vpc` modules are now passed as a list instead of map, and all related variables for IAM and flow logs use `region/name` instead of `name` keys; it's now possible to have the same subnet name in different regions
|
||||||
- replace all references to the removed `resourceviews.googleapis.com` API with `container.googleapis.com`
|
- replace all references to the removed `resourceviews.googleapis.com` API with `container.googleapis.com`
|
||||||
- fix advanced options in `gke-nodepool` module
|
- fix advanced options in `gke-nodepool` module
|
||||||
- fix health checks in `compute-mig` and `net-ilb` modules
|
- fix health checks in `compute-mig` and `net-lb-int` modules
|
||||||
- new `cos-generic-metadata` module in the `cloud-config-container` suite
|
- new `cos-generic-metadata` module in the `cloud-config-container` suite
|
||||||
- new `envoy-traffic-director` module in the `cloud-config-container` suite
|
- new `envoy-traffic-director` module in the `cloud-config-container` suite
|
||||||
- new `pubsub` module
|
- new `pubsub` module
|
||||||
|
@ -1163,7 +1412,7 @@ All notable changes to this project will be documented in this file.
|
||||||
- add peering route configuration for private clusters to GKE cluster module
|
- add peering route configuration for private clusters to GKE cluster module
|
||||||
- **incompatible changes** in the GKE nodepool module: rename `node_config_workload_metadata_config` variable to `workload_metadata_config`, new default for `workload_metadata_config` is `GKE_METADATA_SERVER`
|
- **incompatible changes** in the GKE nodepool module: rename `node_config_workload_metadata_config` variable to `workload_metadata_config`, new default for `workload_metadata_config` is `GKE_METADATA_SERVER`
|
||||||
- **incompatible change** in the `compute-vm` module: removed support for MIG and the `group_manager` variable
|
- **incompatible change** in the `compute-vm` module: removed support for MIG and the `group_manager` variable
|
||||||
- add `compute-mig` and `net-ilb` modules
|
- add `compute-mig` and `net-lb-int` modules
|
||||||
- **incompatible change** in `net-vpc`: a new `name` attribute has been added to the `subnets` variable, allowing to directly set subnet name, to update to the new module add an extra `name = false` attribute to each subnet
|
- **incompatible change** in `net-vpc`: a new `name` attribute has been added to the `subnets` variable, allowing to directly set subnet name, to update to the new module add an extra `name = false` attribute to each subnet
|
||||||
|
|
||||||
## [1.3.0] - 2020-04-08
|
## [1.3.0] - 2020-04-08
|
||||||
|
@ -1188,7 +1437,10 @@ All notable changes to this project will be documented in this file.
|
||||||
- merge development branch with suite of new modules and end-to-end examples
|
- merge development branch with suite of new modules and end-to-end examples
|
||||||
|
|
||||||
<!-- markdown-link-check-disable -->
|
<!-- markdown-link-check-disable -->
|
||||||
[Unreleased]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v21.0.0...HEAD
|
[Unreleased]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v24.0.0...HEAD
|
||||||
|
[24.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v23.0.0...v24.0.0
|
||||||
|
[23.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v22.0.0...v23.0.0
|
||||||
|
[22.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v21.0.0...v22.0.0
|
||||||
[21.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v20.0.0...v21.0.0
|
[21.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v20.0.0...v21.0.0
|
||||||
[20.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v19.0.0...v20.0.0
|
[20.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v19.0.0...v20.0.0
|
||||||
[19.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v18.0.0...v19.0.0
|
[19.0.0]: https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/compare/v18.0.0...v19.0.0
|
||||||
|
|
|
@ -7,22 +7,22 @@ Contributors are the engine that keeps Fabric alive so if you were or are planni
|
||||||
- [I just found a bug / have a feature request](#i-just-found-a-bug--have-a-feature-request)
|
- [I just found a bug / have a feature request](#i-just-found-a-bug--have-a-feature-request)
|
||||||
- [Quick developer workflow](#quick-developer-workflow)
|
- [Quick developer workflow](#quick-developer-workflow)
|
||||||
- [Developer's handbook](#developers-handbook)
|
- [Developer's handbook](#developers-handbook)
|
||||||
* [The Zen of Fabric](#the-zen-of-fabric)
|
- [The Zen of Fabric](#the-zen-of-fabric)
|
||||||
* [Design principles in action](#design-principles-in-action)
|
- [Design principles in action](#design-principles-in-action)
|
||||||
* [FAST stage design](#fast-stage-design)
|
- [FAST stage design](#fast-stage-design)
|
||||||
* [Style guide reference](#style-guide-reference)
|
- [Style guide reference](#style-guide-reference)
|
||||||
* [Interacting with checks and tools](#interacting-with-checks-and-tools)
|
- [Interacting with checks and tools](#interacting-with-checks-and-tools)
|
||||||
- [Using and writing tests](#using-and-writing-tests)
|
- [Using and writing tests](#using-and-writing-tests)
|
||||||
* [Testing via README.md example blocks.](#testing-via-readmemd-example-blocks)
|
- [Testing via README.md example blocks.](#testing-via-readmemd-example-blocks)
|
||||||
+ [Testing examples against an inventory YAML](#testing-examples-against-an-inventory-yaml)
|
- [Testing examples against an inventory YAML](#testing-examples-against-an-inventory-yaml)
|
||||||
+ [Using external files](#using-external-files)
|
- [Using external files](#using-external-files)
|
||||||
+ [Running tests for specific examples](#running-tests-for-specific-examples)
|
- [Running tests for specific examples](#running-tests-for-specific-examples)
|
||||||
+ [Generating the inventory automatically](#generating-the-inventory-automatically)
|
- [Generating the inventory automatically](#generating-the-inventory-automatically)
|
||||||
+ [Building tests for blueprints](#building-tests-for-blueprints)
|
- [Building tests for blueprints](#building-tests-for-blueprints)
|
||||||
* [Testing via `tfvars` and `yaml` (aka `tftest`-based tests)](#testing-via-tfvars-and-yaml-aka-tftest-based-tests)
|
- [Testing via `tfvars` and `yaml` (aka `tftest`-based tests)](#testing-via-tfvars-and-yaml-aka-tftest-based-tests)
|
||||||
+ [Generating the inventory for `tftest`-based tests](#generating-the-inventory-for-tftest-based-tests)
|
- [Generating the inventory for `tftest`-based tests](#generating-the-inventory-for-tftest-based-tests)
|
||||||
* [Writing tests in Python (legacy approach)](#writing-tests-in-python-legacy-approach)
|
- [Writing tests in Python (legacy approach)](#writing-tests-in-python-legacy-approach)
|
||||||
* [Running tests from a temporary directory](#running-tests-from-a-temporary-directory)
|
- [Running tests from a temporary directory](#running-tests-from-a-temporary-directory)
|
||||||
- [Fabric tools](#fabric-tools)
|
- [Fabric tools](#fabric-tools)
|
||||||
|
|
||||||
## I just found a bug / have a feature request
|
## I just found a bug / have a feature request
|
||||||
|
@ -300,7 +300,6 @@ module "project" {
|
||||||
>
|
>
|
||||||
> — John Ousterhout in "A Philosophy of Software Design"
|
> — John Ousterhout in "A Philosophy of Software Design"
|
||||||
|
|
||||||
|
|
||||||
Designing variable spaces is one of the most complex aspects to get right, as they are the main entry point through which users consume modules, examples and FAST stages. We always strive to **design small variable spaces by leveraging objects and implementing defaults** so that users can quickly produce highly readable code.
|
Designing variable spaces is one of the most complex aspects to get right, as they are the main entry point through which users consume modules, examples and FAST stages. We always strive to **design small variable spaces by leveraging objects and implementing defaults** so that users can quickly produce highly readable code.
|
||||||
|
|
||||||
One of many examples of this approach comes from disk support in the `compute-vm` module, where preset defaults allow quick VM management with very few lines of code, and optional variables allow progressively expanding the code when more control is needed.
|
One of many examples of this approach comes from disk support in the `compute-vm` module, where preset defaults allow quick VM management with very few lines of code, and optional variables allow progressively expanding the code when more control is needed.
|
||||||
|
@ -688,7 +687,7 @@ In the following sections we describe the three testing approaches we currently
|
||||||
- [tfvars-based tests](#testing-via-tfvars-and-yaml): allows you to test a module or blueprint by providing variables via tfvar files and an expected plan result in form of an inventory. This type of test is useful, for example, for FAST stages that don't have any examples within their READMEs.
|
- [tfvars-based tests](#testing-via-tfvars-and-yaml): allows you to test a module or blueprint by providing variables via tfvar files and an expected plan result in form of an inventory. This type of test is useful, for example, for FAST stages that don't have any examples within their READMEs.
|
||||||
- [Python-based (legacy) tests](#writing-tests-in-python--legacy-approach-): in some situations you might still want to interact directly with `tftest` via Python, if that's the case, use this method to write custom Python logic to test your module in any way you see fit.
|
- [Python-based (legacy) tests](#writing-tests-in-python--legacy-approach-): in some situations you might still want to interact directly with `tftest` via Python, if that's the case, use this method to write custom Python logic to test your module in any way you see fit.
|
||||||
|
|
||||||
### Testing via README.md example blocks.
|
### Testing via README.md example blocks
|
||||||
|
|
||||||
This is the preferred method to write tests for modules and blueprints. Example-based tests are triggered from [HCL Markdown fenced code blocks](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting) in any file named README.md, hence there's no need to create any additional files or revert to Python to write a test. Most of our documentation examples are using this method.
|
This is the preferred method to write tests for modules and blueprints. Example-based tests are triggered from [HCL Markdown fenced code blocks](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting) in any file named README.md, hence there's no need to create any additional files or revert to Python to write a test. Most of our documentation examples are using this method.
|
||||||
|
|
||||||
|
@ -795,6 +794,7 @@ module "private-dns" {
|
||||||
}
|
}
|
||||||
# tftest modules=1 resources=2 files=records
|
# tftest modules=1 resources=2 files=records
|
||||||
```
|
```
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# tftest-file id=records path=records/example.yaml
|
# tftest-file id=records path=records/example.yaml
|
||||||
A localhost:
|
A localhost:
|
||||||
|
@ -814,6 +814,7 @@ As mentioned before, we use `pytest` as our test runner, so you can use any of t
|
||||||
Example-based test are named based on the section within the README.md that contains them. You can use this name to select specific tests.
|
Example-based test are named based on the section within the README.md that contains them. You can use this name to select specific tests.
|
||||||
|
|
||||||
Here we show a few commonly used selection commands:
|
Here we show a few commonly used selection commands:
|
||||||
|
|
||||||
- Run all examples:
|
- Run all examples:
|
||||||
- `pytest tests/examples/`
|
- `pytest tests/examples/`
|
||||||
- Run all examples for modules:
|
- Run all examples for modules:
|
||||||
|
@ -919,7 +920,7 @@ The second approach to testing requires you to:
|
||||||
- define `yaml` "inventory" files with the plan and output results you want to test
|
- define `yaml` "inventory" files with the plan and output results you want to test
|
||||||
- declare which of these files need to be run as tests in a `tftest.yaml` file
|
- declare which of these files need to be run as tests in a `tftest.yaml` file
|
||||||
|
|
||||||
Let's go through each step in succession, assuming you are testing the new `net-glb` module.
|
Let's go through each step in succession, assuming you are testing the new `net-lb-app-ext` module.
|
||||||
|
|
||||||
First create a new folder under `tests/modules` replacing any dash in the module name with underscores. Note that if you were testing a blueprint the folder would go in `tests/blueprints`.
|
First create a new folder under `tests/modules` replacing any dash in the module name with underscores. Note that if you were testing a blueprint the folder would go in `tests/blueprints`.
|
||||||
|
|
||||||
|
@ -973,7 +974,7 @@ Create as many pairs of `tfvars`/`yaml` files as you need to test every scenario
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# file: tests/modules/net_glb/tftest.yaml
|
# file: tests/modules/net_glb/tftest.yaml
|
||||||
module: modules/net-glb
|
module: modules/net-lb-app-ext
|
||||||
# if there are variables shared among all tests you can define a common file
|
# if there are variables shared among all tests you can define a common file
|
||||||
# common_tfvars:
|
# common_tfvars:
|
||||||
# - defaults.tfvars
|
# - defaults.tfvars
|
||||||
|
@ -1052,12 +1053,13 @@ You can now use this output to create the inventory file for your test. As menti
|
||||||
Where possible, we recommend using the testing methods described in the previous sections. However, if you need it, you can still write tests using Python directly.
|
Where possible, we recommend using the testing methods described in the previous sections. However, if you need it, you can still write tests using Python directly.
|
||||||
|
|
||||||
In general, you should try to use the `plan_summary` fixture, which runs a a terraform plan and returns a `PlanSummary` object. The most important arguments to `plan_summary` are:
|
In general, you should try to use the `plan_summary` fixture, which runs a a terraform plan and returns a `PlanSummary` object. The most important arguments to `plan_summary` are:
|
||||||
|
|
||||||
- the path of the Terraform module you want to test, relative to the root of the repository
|
- the path of the Terraform module you want to test, relative to the root of the repository
|
||||||
- a list of paths representing the tfvars file to pass in to terraform. These paths are relative to the python file defining the test.
|
- a list of paths representing the tfvars file to pass in to terraform. These paths are relative to the python file defining the test.
|
||||||
|
|
||||||
If successful, `plan_summary` will return a `PlanSummary` object with the `values`, `counts` and `outputs` attributes following the same semantics described in the previous section. You can use this fields to write your custom tests.
|
If successful, `plan_summary` will return a `PlanSummary` object with the `values`, `counts` and `outputs` attributes following the same semantics described in the previous section. You can use this fields to write your custom tests.
|
||||||
|
|
||||||
Like before let's imagine we're writing a (python) test for `net-glb` module. First create a new folder under `tests/modules` replacing any dash in the module name with underscores. You also need to create an empty `__init__.py` file in it, to ensure `pytest` discovers you new tests automatically.
|
Like before let's imagine we're writing a (python) test for `net-lb-app-ext` module. First create a new folder under `tests/modules` replacing any dash in the module name with underscores. You also need to create an empty `__init__.py` file in it, to ensure `pytest` discovers you new tests automatically.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mkdir tests/modules/net_glb
|
mkdir tests/modules/net_glb
|
||||||
|
@ -1065,9 +1067,10 @@ touch tests/modules/net_glb/__init__.py
|
||||||
```
|
```
|
||||||
|
|
||||||
Now create a file containing your tests, e.g. `test_plan.py`:
|
Now create a file containing your tests, e.g. `test_plan.py`:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def test_name(plan_summary, tfvars_to_yaml, tmp_path):
|
def test_name(plan_summary, tfvars_to_yaml, tmp_path):
|
||||||
s = plan_summary('modules/net-glb', tf_var_files=['test-plan.tfvars'])
|
s = plan_summary('modules/net-lb-app-ext', tf_var_files=['test-plan.tfvars'])
|
||||||
address = 'google_compute_url_map.default'
|
address = 'google_compute_url_map.default'
|
||||||
assert s.values[address]['project'] == 'my-project'
|
assert s.values[address]['project'] == 'my-project'
|
||||||
```
|
```
|
||||||
|
@ -1081,16 +1084,19 @@ Most of the time you can run tests using the `pytest` command as described in pr
|
||||||
To enable this option, just define the environment variable `TFTEST_COPY` and any tests using the `plan_summary` fixture will automatically run from a temporary directory.
|
To enable this option, just define the environment variable `TFTEST_COPY` and any tests using the `plan_summary` fixture will automatically run from a temporary directory.
|
||||||
|
|
||||||
Running tests from temporary directories is useful if:
|
Running tests from temporary directories is useful if:
|
||||||
|
|
||||||
- you're running tests in parallel using `pytest-xdist`. In this case, just run you tests as follows:
|
- you're running tests in parallel using `pytest-xdist`. In this case, just run you tests as follows:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
TFTEST_COPY=1 pytest -n 4
|
TFTEST_COPY=1 pytest -n 4
|
||||||
```
|
```
|
||||||
|
|
||||||
- you're running tests for the `fast/` directory which contain tfvars and auto.tfvars files (which are read by terraform automatically) making your tests fail. In this case, you can run
|
- you're running tests for the `fast/` directory which contain tfvars and auto.tfvars files (which are read by terraform automatically) making your tests fail. In this case, you can run
|
||||||
|
|
||||||
```
|
```
|
||||||
TFTEST_COPY=1 pytest fast/
|
TFTEST_COPY=1 pytest fast/
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
## Fabric tools
|
## Fabric tools
|
||||||
|
|
||||||
The main tool you will interact with in development is `tfdoc`, used to generate file, output and variable tables in README documents.
|
The main tool you will interact with in development is `tfdoc`, used to generate file, output and variable tables in README documents.
|
||||||
|
|
|
@ -30,12 +30,12 @@ The current list of modules supports most of the core foundational and networkin
|
||||||
Currently available modules:
|
Currently available modules:
|
||||||
|
|
||||||
- **foundational** - [billing budget](./modules/billing-budget), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source)
|
- **foundational** - [billing budget](./modules/billing-budget), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source)
|
||||||
- **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [Global Load Balancer (classic)](./modules/net-glb/), [L4 ILB](./modules/net-ilb), [L7 ILB](./modules/net-ilb-l7), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC firewall policy](./modules/net-vpc-firewall-policy), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory)
|
- **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [Internal Application LB](./modules/net-lb-app-int), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC firewall policy](./modules/net-vpc-firewall-policy), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp)
|
||||||
- **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool)
|
- **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool)
|
||||||
- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Cloud Dataplex](./modules/cloud-dataplex), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub)
|
- **data** - [AlloyDB instance](./modules/alloydb-instance), [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub)
|
||||||
- **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository)
|
- **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository)
|
||||||
- **security** - [Binauthz](./modules/binauthz/), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc)
|
- **security** - [Binauthz](./modules/binauthz/), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc)
|
||||||
- **serverless** - [Cloud Function](./modules/cloud-function), [Cloud Run](./modules/cloud-run)
|
- **serverless** - [Cloud Function v1](./modules/cloud-function-v1), [Cloud Function v2](./modules/cloud-function-v2), [Cloud Run](./modules/cloud-run)
|
||||||
|
|
||||||
For more information and usage examples see each module's README file.
|
For more information and usage examples see each module's README file.
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,7 @@ Currently available blueprints:
|
||||||
- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Minimal Data Platform](./data-solutions/data-platform-minimal), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml)
|
- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Minimal Data Platform](./data-solutions/data-platform-minimal), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml)
|
||||||
- **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory)
|
- **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory)
|
||||||
- **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [GKE Autopilot](./gke/autopilot)
|
- **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [GKE Autopilot](./gke/autopilot)
|
||||||
- **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke)
|
- **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [HA VPN over Interconnect](./networking/ha-vpn-over-interconnect/), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke)
|
||||||
- **serverless** - [Creating multi-region deployments for API Gateway](./serverless/api-gateway), [Cloud Run series](./serverless/cloud-run-explore)
|
- **serverless** - [Creating multi-region deployments for API Gateway](./serverless/api-gateway), [Cloud Run series](./serverless/cloud-run-explore)
|
||||||
- **third party solutions** - [OpenShift on GCP user-provisioned infrastructure](./third-party-solutions/openshift), [Wordpress deployment on Cloud Run](./third-party-solutions/wordpress/cloudrun)
|
- **third party solutions** - [OpenShift on GCP user-provisioned infrastructure](./third-party-solutions/openshift), [Wordpress deployment on Cloud Run](./third-party-solutions/wordpress/cloudrun)
|
||||||
|
|
||||||
|
|
|
@ -20,5 +20,6 @@ The blueprints in this folder contain a variety of deployment scenarios for Apig
|
||||||
|
|
||||||
The following blueprints demonstrate a set of networking scenarios that can be implemented for Apigee X deployments.
|
The following blueprints demonstrate a set of networking scenarios that can be implemented for Apigee X deployments.
|
||||||
|
|
||||||
#### Apigee X - Northbound: GLB with PSC Neg, Southbouth: PSC with ILB (L7) and Hybrid NEG
|
#### Apigee X - Northbound: External Application LB with PSC Neg, Southbouth: PSC with Internal Application LB and Hybrid NEG
|
||||||
<a href="./network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg" title="Apigee X - Northbound: GLB with PSC Neg, Southbouth: PSC with ILB (L7) and Hybrid NEG"><img src="./network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/diagram.png" align="left" width="280px"></a>This [blueprint](./network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/) shows how to expose an on-prem target backend to clients in the Internet.
|
|
||||||
|
<a href="./network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg" title="Northbound: External Application LB with PSC Neg, Southbouth: PSC with Internal Application LB and Hybrid NEG"><img src="./network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/diagram.png" align="left" width="280px"></a>This [blueprint](./network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/) shows how to expose an on-prem target backend to clients in the Internet.
|
||||||
|
|
|
@ -59,15 +59,15 @@ Do the following to verify that everything works as expected.
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [envgroups](variables.tf#L24) | Environment groups (NAME => [HOSTNAMES]). | <code>map(list(string))</code> | ✓ | |
|
| [envgroups](variables.tf#L24) | Environment groups (NAME => [HOSTNAMES]). | <code>map(list(string))</code> | ✓ | |
|
||||||
| [environments](variables.tf#L30) | Environments. | <code title="map(object({ display_name = optional(string) description = optional(string) node_config = optional(object({ min_node_count = optional(number) max_node_count = optional(number) })) iam = optional(map(list(string))) envgroups = list(string) }))">map(object({…}))</code> | ✓ | |
|
| [environments](variables.tf#L30) | Environments. | <code title="map(object({ display_name = optional(string) description = optional(string) node_config = optional(object({ min_node_count = optional(number) max_node_count = optional(number) })) iam = optional(map(list(string))) envgroups = optional(list(string)) regions = optional(list(string)) }))">map(object({…}))</code> | ✓ | |
|
||||||
| [instances](variables.tf#L45) | Instance. | <code title="map(object({ display_name = optional(string) description = optional(string) region = string environments = list(string) runtime_ip_cidr_range = string troubleshooting_ip_cidr_range = string disk_encryption_key = optional(string) consumer_accept_list = optional(list(string)) }))">map(object({…}))</code> | ✓ | |
|
| [instances](variables.tf#L46) | Instance. | <code title="map(object({ display_name = optional(string) description = optional(string) runtime_ip_cidr_range = string troubleshooting_ip_cidr_range = string disk_encryption_key = optional(string) consumer_accept_list = optional(list(string)) }))">map(object({…}))</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L92) | Project ID. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L91) | Project ID. | <code>string</code> | ✓ | |
|
||||||
| [psc_config](variables.tf#L98) | PSC configuration. | <code>map(string)</code> | ✓ | |
|
| [psc_config](variables.tf#L97) | PSC configuration. | <code>map(string)</code> | ✓ | |
|
||||||
| [datastore_name](variables.tf#L17) | Datastore. | <code>string</code> | | <code>"gcs"</code> |
|
| [datastore_name](variables.tf#L17) | Datastore. | <code>string</code> | | <code>"gcs"</code> |
|
||||||
| [organization](variables.tf#L60) | Apigee organization. | <code title="object({ display_name = optional(string, "Apigee organization created by tf module") description = optional(string, "Apigee organization created by tf module") authorized_network = optional(string, "vpc") runtime_type = optional(string, "CLOUD") billing_type = optional(string) database_encryption_key = optional(string) analytics_region = optional(string, "europe-west1") })">object({…})</code> | | <code title="{ }">{…}</code> |
|
| [organization](variables.tf#L59) | Apigee organization. | <code title="object({ display_name = optional(string, "Apigee organization created by tf module") description = optional(string, "Apigee organization created by tf module") authorized_network = optional(string, "vpc") runtime_type = optional(string, "CLOUD") billing_type = optional(string) database_encryption_key = optional(string) analytics_region = optional(string, "europe-west1") })">object({…})</code> | | <code title="{ }">{…}</code> |
|
||||||
| [path](variables.tf#L76) | Bucket path. | <code>string</code> | | <code>"/analytics"</code> |
|
| [path](variables.tf#L75) | Bucket path. | <code>string</code> | | <code>"/analytics"</code> |
|
||||||
| [project_create](variables.tf#L83) | Parameters for the creation of the new project. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
| [project_create](variables.tf#L82) | Parameters for the creation of the new project. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||||
| [vpc_create](variables.tf#L104) | Boolean flag indicating whether the VPC should be created or not. | <code>bool</code> | | <code>true</code> |
|
| [vpc_create](variables.tf#L103) | Boolean flag indicating whether the VPC should be created or not. | <code>bool</code> | | <code>true</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
@ -92,12 +92,11 @@ module "test" {
|
||||||
environments = {
|
environments = {
|
||||||
apis-test = {
|
apis-test = {
|
||||||
envgroups = ["test"]
|
envgroups = ["test"]
|
||||||
|
regions = ["europe-west1"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
instances = {
|
instances = {
|
||||||
instance-ew1 = {
|
europe-west1 = {
|
||||||
region = "europe-west1"
|
|
||||||
environments = ["apis-test"]
|
|
||||||
runtime_ip_cidr_range = "10.0.4.0/22"
|
runtime_ip_cidr_range = "10.0.4.0/22"
|
||||||
troubleshooting_ip_cidr_range = "10.1.0.0/28"
|
troubleshooting_ip_cidr_range = "10.1.0.0/28"
|
||||||
}
|
}
|
||||||
|
@ -106,5 +105,5 @@ module "test" {
|
||||||
europe-west1 = "10.0.0.0/28"
|
europe-west1 = "10.0.0.0/28"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
# tftest modules=10 resources=62
|
# tftest modules=10 resources=64
|
||||||
```
|
```
|
||||||
|
|
|
@ -88,7 +88,7 @@ module "apigee" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "glb" {
|
module "glb" {
|
||||||
source = "../../../modules/net-glb"
|
source = "../../../modules/net-lb-app-ext"
|
||||||
name = "glb"
|
name = "glb"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
protocol = "HTTPS"
|
protocol = "HTTPS"
|
||||||
|
@ -108,11 +108,11 @@ module "glb" {
|
||||||
neg_configs = {
|
neg_configs = {
|
||||||
for k, v in var.instances : k => {
|
for k, v in var.instances : k => {
|
||||||
psc = {
|
psc = {
|
||||||
region = v.region
|
region = k
|
||||||
target_service = module.apigee.instances[k].service_attachment
|
target_service = module.apigee.instances[k].service_attachment
|
||||||
network = module.vpc.network.self_link
|
network = module.vpc.network.self_link
|
||||||
subnetwork = (
|
subnetwork = (
|
||||||
module.vpc.subnets_psc["${v.region}/subnet-psc-${v.region}"].self_link
|
module.vpc.subnets_psc["${k}/subnet-psc-${k}"].self_link
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -152,7 +152,7 @@ module "bucket_export" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "function_export" {
|
module "function_export" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "export"
|
name = "export"
|
||||||
bucket_name = "${module.project.project_id}-code-export"
|
bucket_name = "${module.project.project_id}-code-export"
|
||||||
|
@ -180,17 +180,15 @@ module "function_export" {
|
||||||
DATASTORE = var.datastore_name
|
DATASTORE = var.datastore_name
|
||||||
}
|
}
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub_export.id
|
resource = module.pubsub_export.id
|
||||||
retry = null
|
retry = null
|
||||||
}
|
}
|
||||||
}
|
|
||||||
service_account_create = true
|
service_account_create = true
|
||||||
}
|
}
|
||||||
|
|
||||||
module "function_gcs2bq" {
|
module "function_gcs2bq" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "gcs2bq"
|
name = "gcs2bq"
|
||||||
bucket_name = "${module.project.project_id}-code-gcs2bq"
|
bucket_name = "${module.project.project_id}-code-gcs2bq"
|
||||||
|
@ -218,12 +216,10 @@ module "function_gcs2bq" {
|
||||||
LOCATION = var.organization.analytics_region
|
LOCATION = var.organization.analytics_region
|
||||||
}
|
}
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.bucket_export.topic
|
resource = module.bucket_export.topic
|
||||||
retry = null
|
retry = null
|
||||||
}
|
}
|
||||||
}
|
|
||||||
service_account_create = true
|
service_account_create = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -12,8 +12,7 @@ environments = {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
instances = {
|
instances = {
|
||||||
instance-ew1 = {
|
europe-west1 = {
|
||||||
region = "europe-west1"
|
|
||||||
environments = ["apis-test"]
|
environments = ["apis-test"]
|
||||||
runtime_ip_cidr_range = "10.0.4.0/22"
|
runtime_ip_cidr_range = "10.0.4.0/22"
|
||||||
troubleshooting_ip_cidr_range = "10.1.1.0/28"
|
troubleshooting_ip_cidr_range = "10.1.1.0/28"
|
||||||
|
|
|
@ -37,7 +37,8 @@ variable "environments" {
|
||||||
max_node_count = optional(number)
|
max_node_count = optional(number)
|
||||||
}))
|
}))
|
||||||
iam = optional(map(list(string)))
|
iam = optional(map(list(string)))
|
||||||
envgroups = list(string)
|
envgroups = optional(list(string))
|
||||||
|
regions = optional(list(string))
|
||||||
}))
|
}))
|
||||||
nullable = false
|
nullable = false
|
||||||
}
|
}
|
||||||
|
@ -47,8 +48,6 @@ variable "instances" {
|
||||||
type = map(object({
|
type = map(object({
|
||||||
display_name = optional(string)
|
display_name = optional(string)
|
||||||
description = optional(string)
|
description = optional(string)
|
||||||
region = string
|
|
||||||
environments = list(string)
|
|
||||||
runtime_ip_cidr_range = string
|
runtime_ip_cidr_range = string
|
||||||
troubleshooting_ip_cidr_range = string
|
troubleshooting_ip_cidr_range = string
|
||||||
disk_encryption_key = optional(string)
|
disk_encryption_key = optional(string)
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -80,5 +80,5 @@ module "test" {
|
||||||
project_id = "my-project"
|
project_id = "my-project"
|
||||||
hostname = "test.myorg.org"
|
hostname = "test.myorg.org"
|
||||||
}
|
}
|
||||||
# tftest modules=18 resources=59
|
# tftest modules=18 resources=61
|
||||||
```
|
```
|
||||||
|
|
|
@ -2,5 +2,6 @@
|
||||||
|
|
||||||
The blueprints in this folder demonstrate a set of networking scenarios that can be implemented for Apigee X deployments.
|
The blueprints in this folder demonstrate a set of networking scenarios that can be implemented for Apigee X deployments.
|
||||||
|
|
||||||
## Apigee X - Northbound: GLB with PSC Neg, Southbouth: PSC with ILB (L7) and Hybrid NEG
|
## Northbound: External Application LB with PSC Neg, Southbouth: PSC with Internal Application LB and Hybrid NEG
|
||||||
<a href="./nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg" title="Apigee X - Northbound: GLB with PSC Neg, Southbouth: PSC with ILB (L7) and Hybrid NEG"><img src="./nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/diagram.png" align="left" width="280px"></a>This [blueprint](./nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/) shows how to expose an on-prem target backend to clients in the Internet.g
|
|
||||||
|
<a href="./nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg" title="Northbound: External Application LB with PSC Neg, Southbouth: PSC with Internal Application LB and Hybrid NEG"><img src="./nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/diagram.png" align="left" width="280px"></a>This [blueprint](./nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/) shows how to expose an on-prem target backend to clients in the Internet.g
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# Apigee X - Northbound GLB with PSC Neg, Southbouth PSC with ILB (L7) and Hybrid NEG
|
# Apigee X - Northbound: External Application LB with PSC Neg, Southbouth: PSC with Internal Application LB and Hybrid NEG
|
||||||
|
|
||||||
The following blueprint shows how to expose an on-prem target backend to clients in the Internet.
|
The following blueprint shows how to expose an on-prem target backend to clients in the Internet.
|
||||||
|
|
||||||
|
@ -79,5 +79,5 @@ module "test" {
|
||||||
onprem_project_id = "my-onprem-project"
|
onprem_project_id = "my-onprem-project"
|
||||||
hostname = "test.myorg.org"
|
hostname = "test.myorg.org"
|
||||||
}
|
}
|
||||||
# tftest modules=14 resources=73
|
# tftest modules=14 resources=77
|
||||||
```
|
```
|
||||||
|
|
|
@ -76,12 +76,11 @@ module "apigee" {
|
||||||
environments = {
|
environments = {
|
||||||
(local.environment) = {
|
(local.environment) = {
|
||||||
envgroups = [local.envgroup]
|
envgroups = [local.envgroup]
|
||||||
|
regions = [var.region]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
instances = {
|
instances = {
|
||||||
instance-1 = {
|
(var.region) = {
|
||||||
region = var.region
|
|
||||||
environments = [local.environment]
|
|
||||||
runtime_ip_cidr_range = var.apigee_runtime_ip_cidr_range
|
runtime_ip_cidr_range = var.apigee_runtime_ip_cidr_range
|
||||||
troubleshooting_ip_cidr_range = var.apigee_troubleshooting_ip_cidr_range
|
troubleshooting_ip_cidr_range = var.apigee_troubleshooting_ip_cidr_range
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
module "glb" {
|
module "glb" {
|
||||||
source = "../../../../modules/net-glb"
|
source = "../../../../modules/net-lb-app-ext"
|
||||||
name = "glb"
|
name = "glb"
|
||||||
project_id = module.apigee_project.project_id
|
project_id = module.apigee_project.project_id
|
||||||
protocol = "HTTPS"
|
protocol = "HTTPS"
|
||||||
|
@ -31,7 +31,7 @@ module "glb" {
|
||||||
neg-0 = {
|
neg-0 = {
|
||||||
psc = {
|
psc = {
|
||||||
region = var.region
|
region = var.region
|
||||||
target_service = module.apigee.instances["instance-1"].service_attachment
|
target_service = module.apigee.instances[var.region].service_attachment
|
||||||
network = module.apigee_vpc.network.self_link
|
network = module.apigee_vpc.network.self_link
|
||||||
subnetwork = (
|
subnetwork = (
|
||||||
module.apigee_vpc.subnets_psc["${var.region}/subnet-psc"].self_link
|
module.apigee_vpc.subnets_psc["${var.region}/subnet-psc"].self_link
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
module "apigee_ilb_l7" {
|
module "apigee_ilb_l7" {
|
||||||
source = "../../../../modules/net-ilb-l7"
|
source = "../../../../modules/net-lb-app-int"
|
||||||
name = "apigee-ilb"
|
name = "apigee-ilb"
|
||||||
project_id = module.apigee_project.project_id
|
project_id = module.apigee_project.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
|
|
|
@ -116,7 +116,7 @@ module "mig" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "onprem_ilb_l7" {
|
module "onprem_ilb_l7" {
|
||||||
source = "../../../../modules/net-ilb-l7"
|
source = "../../../../modules/net-lb-app-int"
|
||||||
name = "ilb"
|
name = "ilb"
|
||||||
project_id = module.onprem_project.project_id
|
project_id = module.onprem_project.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
billing_account_id = "12345-12345-123456"
|
billing_account_id = "12345-12345-123456"
|
||||||
parent = "folders/123456789"
|
parent = "folders/123456789"
|
||||||
apigee_project_id = "my-apigee-project"
|
apigee_project_id = "my-apigee-project"
|
||||||
onprem_project_id = "my-onprem-project"
|
onprem_proje◊ct_id = "my-onprem-project"
|
||||||
hostname = "test.myorg.org"
|
hostname = "test.myorg.org"
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -89,5 +89,5 @@ module "test" {
|
||||||
ad_dns_domain_name = "example.com"
|
ad_dns_domain_name = "example.com"
|
||||||
adfs_dns_domain_name = "adfs.example.com"
|
adfs_dns_domain_name = "adfs.example.com"
|
||||||
}
|
}
|
||||||
# tftest modules=5 resources=18
|
# tftest modules=5 resources=20
|
||||||
```
|
```
|
||||||
|
|
|
@ -96,7 +96,7 @@ module "server" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "glb" {
|
module "glb" {
|
||||||
source = "../../../modules/net-glb"
|
source = "../../../modules/net-lb-app-ext"
|
||||||
name = "${var.prefix}-glb"
|
name = "${var.prefix}-glb"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
protocol = "HTTPS"
|
protocol = "HTTPS"
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -82,5 +82,5 @@ module "test" {
|
||||||
project_id = "project-1"
|
project_id = "project-1"
|
||||||
}
|
}
|
||||||
|
|
||||||
# tftest modules=7 resources=21
|
# tftest modules=7 resources=23
|
||||||
```
|
```
|
||||||
|
|
|
@ -74,7 +74,7 @@ module "service-account" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "cf" {
|
module "cf" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = var.name
|
name = var.name
|
||||||
bucket_name = "${var.name}-${random_pet.random.id}"
|
bucket_name = "${var.name}-${random_pet.random.id}"
|
||||||
|
@ -87,11 +87,9 @@ module "cf" {
|
||||||
}
|
}
|
||||||
service_account = module.service-account.email
|
service_account = module.service-account.email
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub.topic.id
|
resource = module.pubsub.topic.id
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module "simple-vm-example" {
|
module "simple-vm-example" {
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -128,5 +128,5 @@ module "test1" {
|
||||||
project_create = true
|
project_create = true
|
||||||
project_id = "test"
|
project_id = "test"
|
||||||
}
|
}
|
||||||
# tftest modules=9 resources=25
|
# tftest modules=9 resources=28
|
||||||
```
|
```
|
||||||
|
|
|
@ -60,11 +60,14 @@ module "nat-a" {
|
||||||
module "dns-service-zone" {
|
module "dns-service-zone" {
|
||||||
source = "../../../modules/dns"
|
source = "../../../modules/dns"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
type = "service-directory"
|
|
||||||
name = var.name
|
name = var.name
|
||||||
|
zone_config = {
|
||||||
domain = var.zone_domain
|
domain = var.zone_domain
|
||||||
|
private = {
|
||||||
client_networks = [module.vpc.self_link]
|
client_networks = [module.vpc.self_link]
|
||||||
service_directory_namespace = module.service-directory.id
|
service_directory_namespace = module.service-directory.id
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module "service-directory" {
|
module "service-directory" {
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,5 +51,5 @@ module "test" {
|
||||||
shared_vpc_link = "https://www.googleapis.com/compute/v1/projects/test-dns/global/networks/default"
|
shared_vpc_link = "https://www.googleapis.com/compute/v1/projects/test-dns/global/networks/default"
|
||||||
teams = ["team1", "team2"]
|
teams = ["team1", "team2"]
|
||||||
}
|
}
|
||||||
# tftest modules=9 resources=12
|
# tftest modules=9 resources=20
|
||||||
```
|
```
|
||||||
|
|
|
@ -42,11 +42,14 @@ module "dns-private" {
|
||||||
source = "../../../modules/dns"
|
source = "../../../modules/dns"
|
||||||
for_each = local.projects
|
for_each = local.projects
|
||||||
project_id = each.value
|
project_id = each.value
|
||||||
type = "private"
|
|
||||||
name = each.key
|
name = each.key
|
||||||
domain = "${each.key}.${var.dns_domain}."
|
|
||||||
description = "DNS zone for ${each.key}"
|
description = "DNS zone for ${each.key}"
|
||||||
|
zone_config = {
|
||||||
|
domain = "${each.key}.${var.dns_domain}."
|
||||||
|
private = {
|
||||||
client_networks = [module.vpc[each.key].self_link]
|
client_networks = [module.vpc[each.key].self_link]
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module "dns-peering" {
|
module "dns-peering" {
|
||||||
|
@ -54,9 +57,12 @@ module "dns-peering" {
|
||||||
for_each = local.projects
|
for_each = local.projects
|
||||||
project_id = local.svpc_project_id
|
project_id = local.svpc_project_id
|
||||||
name = "peering-${each.key}"
|
name = "peering-${each.key}"
|
||||||
domain = "${each.key}.${var.dns_domain}."
|
|
||||||
description = "DNS peering for ${each.key}"
|
description = "DNS peering for ${each.key}"
|
||||||
type = "peering"
|
zone_config = {
|
||||||
|
domain = "${each.key}.${var.dns_domain}."
|
||||||
|
peering = {
|
||||||
peer_network = module.vpc[each.key].self_link
|
peer_network = module.vpc[each.key].self_link
|
||||||
client_networks = [var.shared_vpc_link]
|
client_networks = [var.shared_vpc_link]
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -72,7 +72,7 @@ Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/)
|
||||||
## Assumptions and limitations
|
## Assumptions and limitations
|
||||||
|
|
||||||
- The tool assumes all VPCs in peering groups are within the same organization, except for PSA peerings.
|
- The tool assumes all VPCs in peering groups are within the same organization, except for PSA peerings.
|
||||||
- The tool will only fetch subnet utilization data from the PSA peerings (not the VMs, ILB or routes usage).
|
- The tool will only fetch subnet utilization data from the PSA peerings (not the VMs, LB or routes usage).
|
||||||
- The tool assumes global routing is ON, this impacts dynamic routes usage calculation.
|
- The tool assumes global routing is ON, this impacts dynamic routes usage calculation.
|
||||||
- The tool assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation.
|
- The tool assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation.
|
||||||
- The tool assumes all networks in peering groups have the same global routing and custom routes sharing configuration.
|
- The tool assumes all networks in peering groups have the same global routing and custom routes sharing configuration.
|
||||||
|
|
|
@ -51,7 +51,7 @@ module "pubsub" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "cloud-function" {
|
module "cloud-function" {
|
||||||
source = "../../../../modules/cloud-function"
|
source = "../../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = var.name
|
name = var.name
|
||||||
bucket_name = coalesce(
|
bucket_name = coalesce(
|
||||||
|
@ -76,11 +76,9 @@ module "cloud-function" {
|
||||||
}
|
}
|
||||||
service_account_create = true
|
service_account_create = true
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub.topic.id
|
resource = module.pubsub.topic.id
|
||||||
}
|
}
|
||||||
}
|
|
||||||
vpc_connector = (
|
vpc_connector = (
|
||||||
var.cloud_function_config.vpc_connector == null
|
var.cloud_function_config.vpc_connector == null
|
||||||
? null
|
? null
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -115,5 +115,5 @@ module "test" {
|
||||||
packer_account_users = ["user:john@example.com"]
|
packer_account_users = ["user:john@example.com"]
|
||||||
create_packer_vars = true
|
create_packer_vars = true
|
||||||
}
|
}
|
||||||
# tftest modules=7 resources=17 files=pkrvars
|
# tftest modules=7 resources=19 files=pkrvars
|
||||||
```
|
```
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,38 +1,55 @@
|
||||||
# Compute Engine quota monitoring
|
# Compute Engine quota monitoring
|
||||||
|
|
||||||
This blueprint improves on the [GCE quota exporter tool](https://github.com/GoogleCloudPlatform/professional-services/tree/master/tools/gce-quota-sync) (by the same author of this blueprint), and shows a practical way of collecting and monitoring [Compute Engine resource quotas](https://cloud.google.com/compute/quotas) via Cloud Monitoring metrics as an alternative to the recently released [built-in quota metrics](https://cloud.google.com/monitoring/alerts/using-quota-metrics).
|
This blueprint improves on the [GCE quota exporter tool](https://github.com/GoogleCloudPlatform/professional-services/tree/master/tools/gce-quota-sync) (by the same author of this blueprint), and shows a practical way of collecting and monitoring [Compute Engine resource quotas](https://cloud.google.com/compute/quotas) via Cloud Monitoring metrics as an alternative to the [built-in quota metrics](https://cloud.google.com/monitoring/alerts/using-quota-metrics).
|
||||||
|
|
||||||
Compared to the built-in metrics, it offers a simpler representation of quotas and quota ratios which is especially useful in charts, it allows filtering or combining quotas between different projects regardless of their monitoring workspace, and it creates a default alerting policy without the need to interact directly with the monitoring API.
|
Compared to the built-in metrics, it offers a simpler representation of quotas and quota ratios which is especially useful in charts, allows filtering or combining quotas between different projects regardless of their monitoring workspace, and optionally creates alerting policies without the need to interact directly with the monitoring API.
|
||||||
|
|
||||||
Regardless of its specific purpose, this blueprint is also useful in showing how to manipulate and write time series to cloud monitoring. The resources it creates are shown in the high level diagram below:
|
Regardless of its specific purpose, this blueprint is also useful in showing how to manipulate and write time series to cloud monitoring. The resources it creates are shown in the high level diagram below:
|
||||||
|
|
||||||
<img src="diagram.png" width="640px" alt="GCP resource diagram">
|
<img src="diagram.png" width="640px" alt="GCP resource diagram">
|
||||||
|
|
||||||
The solution is designed so that the Cloud Function arguments that control function execution (eg to set which project quotas to monitor) are defined in the Cloud Scheduler payload set in the PubSub message, so that a single function can be used for different configurations by creating more schedules.
|
The Cloud Function arguments that control function execution (for example to set which project quotas to monitor) are defined in the Cloud Scheduler payload sent in the PubSub message, so that a single function can be used for different configurations by creating more schedules.
|
||||||
|
|
||||||
Quota time series are stored using [custom metrics](https://cloud.google.com/monitoring/custom-metrics) with metric type for usage, limit and utilization; metric types are named using a common prefix and two tokens joined by a `-` character:
|
Quota time series are stored using [custom metrics](https://cloud.google.com/monitoring/custom-metrics) with different metric types for usage, limit and utilization; metric types are based on a common prefix defaulting to `quota` and two tokens representing the quota name and type of data. This is an example:
|
||||||
|
|
||||||
- `prefix` (custom.googleapis.com/quota/)
|
- `custom.googleapis.com/quota/firewalls/usage`
|
||||||
- `quota name`
|
- `custom.googleapis.com/quota/firewalls/limit`
|
||||||
- `{usage,limit,utilization}`
|
- `custom.googleapis.com/quota/firewalls/ratio`
|
||||||
|
|
||||||
e.g:
|
|
||||||
|
|
||||||
- `custom.googleapis.com/quota/firewalls_usage`
|
|
||||||
- `custom.googleapis.com/quota/firewalls_limit`
|
|
||||||
- `custom.googleapis.com/quota/firewalls_utilization`
|
|
||||||
|
|
||||||
All custom metrics are associated to the `global` resource type and use [gauge kind](https://cloud.google.com/monitoring/api/v3/kinds-and-types#metric-kinds)
|
All custom metrics are associated to the `global` resource type and use [gauge kind](https://cloud.google.com/monitoring/api/v3/kinds-and-types#metric-kinds)
|
||||||
|
|
||||||
Labels are set with project id (which may differ from the monitoring workspace projects) and region (quotas that are not region specific are labelled `global`), this is how a usage/limit/utilization triplet looks in in Metrics Explorer
|
Metric labels contain
|
||||||
|
|
||||||
|
- `project` set to the project of the quota
|
||||||
|
- `location` set to the region of the quota (or `global` for project-level quotas)
|
||||||
|
- `quota` containing the string representation of `usage / limit` for the quota, to provide an immediate reference when checking ratios; this can be easily turned off in code if reducing cardinality is needed
|
||||||
|
|
||||||
|
Labels are set with project id (which may differ from the monitoring workspace projects) and region (quotas that are not region specific are labelled `global`), this is how the `ratio` metric for a quota looks in in Metrics Explorer
|
||||||
|
|
||||||
<img src="explorer.png" width="640px" alt="GCP Metrics Explorer, usage, limit and utilization view sample">
|
<img src="explorer.png" width="640px" alt="GCP Metrics Explorer, usage, limit and utilization view sample">
|
||||||
|
|
||||||
The solution can also create a basic monitoring alert policy, to demonstrate how to raise alerts when quotas utilization goes over a predefined threshold, to enable it, set variable `alert_create` to true and reapply main.tf after main.py has run at least one and quota monitoring metrics have been created.
|
## Configuring resources
|
||||||
|
|
||||||
|
The projects where resources are created is also the one where metrics will be written, and is configured via the `project_id` variable. The project can optionally be created by configuring the `project_create_config` variable.
|
||||||
|
|
||||||
|
The region, location of the bundle used to deploy the function, and scheduling frequency can also be configured via the relevant variables.
|
||||||
|
|
||||||
|
## Configuring Cloud Function parameters
|
||||||
|
|
||||||
|
The `quota_config` variable mirrors the arguments accepted by the Python program, and allows configuring several different aspects of its behaviour:
|
||||||
|
|
||||||
|
- `quota_config.exclude` do not generate metrics for quotas matching prefixes listed here
|
||||||
|
- `quota_config.include` only generate metrics for quotas matching prefixes listed here
|
||||||
|
- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored
|
||||||
|
- `quota_config.regions` regions to track quotas for, defaults to the `global` region for project-level quotas
|
||||||
|
- `dry_run` do not write actual metrics
|
||||||
|
- `verbose` increase logging verbosity
|
||||||
|
|
||||||
|
The solution can also create a basic monitoring alert policies, to demonstrate how to raise alerts when quotas utilization goes over a predefined threshold, to enable it, set variable `alert_create` to true and reapply main.tf after main.py has run at least one and quota monitoring metrics have been created.
|
||||||
|
|
||||||
## Running the blueprint
|
## Running the blueprint
|
||||||
|
|
||||||
Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fcloud-foundation-fabric&cloudshell_print=cloud-shell-readme.txt&cloudshell_working_dir=blueprints%2Fcloud-operations%2Fquota-monitoring), then go through the following steps to create resources:
|
Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fcloud-foundation-fabric&cloudshell_print=cloud-shell-readme.txt&cloudshell_working_dir=blueprints%2Fcloud-operations%2Fquota-monitoring), then go through the following steps to create resources:
|
||||||
|
|
||||||
- `terraform init`
|
- `terraform init`
|
||||||
- `terraform apply -var project_id=my-project-id`
|
- `terraform apply -var project_id=my-project-id`
|
||||||
|
@ -42,25 +59,26 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [project_id](variables.tf#L41) | Project id that references existing project. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L54) | Project id that references existing project. | <code>string</code> | ✓ | |
|
||||||
| [alert_create](variables.tf#L17) | Enables the creation of a sample monitoring alert, false by default. | <code>bool</code> | | <code>false</code> |
|
| [alert_configs](variables.tf#L17) | Configure creation of monitoring alerts for specific quotas. Keys match quota names. | <code title="map(object({ documentation = optional(string) enabled = optional(bool) labels = optional(map(string)) threshold = optional(number, 0.75) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [bundle_path](variables.tf#L23) | Path used to write the intermediate Cloud Function code bundle. | <code>string</code> | | <code>"./bundle.zip"</code> |
|
| [bundle_path](variables.tf#L33) | Path used to write the intermediate Cloud Function code bundle. | <code>string</code> | | <code>"./bundle.zip"</code> |
|
||||||
| [name](variables.tf#L29) | Arbitrary string used to name created resources. | <code>string</code> | | <code>"quota-monitor"</code> |
|
| [name](variables.tf#L39) | Arbitrary string used to name created resources. | <code>string</code> | | <code>"quota-monitor"</code> |
|
||||||
| [project_create](variables.tf#L35) | Create project instead of using an existing one. | <code>bool</code> | | <code>false</code> |
|
| [project_create_config](variables.tf#L45) | Create project instead of using an existing one. | <code title="object({ billing_account = string parent = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||||
| [quota_config](variables.tf#L46) | Cloud function configuration. | <code title="object({ filters = list(string) projects = list(string) regions = list(string) })">object({…})</code> | | <code title="{ filters = null projects = null regions = null }">{…}</code> |
|
| [quota_config](variables.tf#L59) | Cloud function configuration. | <code title="object({ exclude = optional(list(string), [ "a2", "c2", "c2d", "committed", "g2", "interconnect", "m1", "m2", "m3", "nvidia", "preemptible" ]) include = optional(list(string)) projects = optional(list(string)) regions = optional(list(string)) dry_run = optional(bool, false) verbose = optional(bool, false) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [region](variables.tf#L60) | Compute region used in the example. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [region](variables.tf#L76) | Compute region used in the example. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
| [schedule_config](variables.tf#L66) | Schedule timer configuration in crontab format. | <code>string</code> | | <code>"0 * * * *"</code> |
|
| [schedule_config](variables.tf#L82) | Schedule timer configuration in crontab format. | <code>string</code> | | <code>"0 * * * *"</code> |
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
||||||
## Test
|
## Test
|
||||||
|
|
||||||
```hcl
|
```hcl
|
||||||
module "test" {
|
module "test" {
|
||||||
source = "./fabric/blueprints/cloud-operations/quota-monitoring"
|
source = "./fabric/blueprints/cloud-operations/quota-monitoring"
|
||||||
name = "name"
|
name = "name"
|
||||||
project_create = true
|
|
||||||
project_id = "test"
|
project_id = "test"
|
||||||
|
project_create_config = {
|
||||||
|
billing_account = "12345-ABCDE-12345"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
# tftest modules=4 resources=14
|
# tftest modules=4 resources=14
|
||||||
```
|
```
|
||||||
|
|
|
@ -1,226 +0,0 @@
|
||||||
#! /usr/bin/env python3
|
|
||||||
# Copyright 2022 Google LLC
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
"""Sync GCE quota usage to Stackdriver for multiple projects.
|
|
||||||
|
|
||||||
This tool fetches global and/or regional quotas from the GCE API for
|
|
||||||
multiple projects, and sends them to Stackdriver as custom metrics, where they
|
|
||||||
can be used to set alert policies or create charts.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
import warnings
|
|
||||||
|
|
||||||
import click
|
|
||||||
|
|
||||||
from google.api_core.exceptions import GoogleAPIError
|
|
||||||
from google.api import label_pb2 as ga_label
|
|
||||||
from google.api import metric_pb2 as ga_metric
|
|
||||||
from google.cloud import monitoring_v3
|
|
||||||
|
|
||||||
import googleapiclient.discovery
|
|
||||||
import googleapiclient.errors
|
|
||||||
|
|
||||||
_BATCH_SIZE = 5
|
|
||||||
_METRIC_KIND = ga_metric.MetricDescriptor.MetricKind.GAUGE
|
|
||||||
_METRIC_TYPE_STEM = 'custom.googleapis.com/quota/'
|
|
||||||
|
|
||||||
_USAGE = "usage"
|
|
||||||
_LIMIT = "limit"
|
|
||||||
_UTILIZATION = "utilization"
|
|
||||||
|
|
||||||
|
|
||||||
def _add_series(project_id, series, client=None):
|
|
||||||
"""Write metrics series to Stackdriver.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
project_id: series will be written to this project id's account
|
|
||||||
series: the time series to be written, as a list of
|
|
||||||
monitoring_v3.types.TimeSeries instances
|
|
||||||
client: optional monitoring_v3.MetricServiceClient will be used
|
|
||||||
instead of obtaining a new one
|
|
||||||
"""
|
|
||||||
client = client or monitoring_v3.MetricServiceClient()
|
|
||||||
project_name = client.common_project_path(project_id)
|
|
||||||
if isinstance(series, monitoring_v3.types.TimeSeries):
|
|
||||||
series = [series]
|
|
||||||
try:
|
|
||||||
client.create_time_series(name=project_name, time_series=series)
|
|
||||||
except GoogleAPIError as e:
|
|
||||||
raise RuntimeError('Error from monitoring API: %s' % e)
|
|
||||||
|
|
||||||
|
|
||||||
def _configure_logging(verbose=True):
|
|
||||||
"""Basic logging configuration.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
verbose: enable verbose logging
|
|
||||||
"""
|
|
||||||
level = logging.DEBUG if verbose else logging.INFO
|
|
||||||
logging.basicConfig(level=level)
|
|
||||||
warnings.filterwarnings('ignore', r'.*end user credentials.*', UserWarning)
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_quotas(project, region='global', compute=None):
|
|
||||||
"""Fetch GCE per - project or per - region quotas from the API.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
project: fetch global or regional quotas for this project id
|
|
||||||
region: which quotas to fetch, 'global' or region name
|
|
||||||
compute: optional instance of googleapiclient.discovery.build will be used
|
|
||||||
instead of obtaining a new one
|
|
||||||
"""
|
|
||||||
compute = compute or googleapiclient.discovery.build('compute', 'v1')
|
|
||||||
try:
|
|
||||||
if region != 'global':
|
|
||||||
req = compute.regions().get(project=project, region=region)
|
|
||||||
else:
|
|
||||||
req = compute.projects().get(project=project)
|
|
||||||
resp = req.execute()
|
|
||||||
return resp['quotas']
|
|
||||||
except (GoogleAPIError, googleapiclient.errors.HttpError) as e:
|
|
||||||
logging.debug('API Error: %s', e, exc_info=True)
|
|
||||||
raise RuntimeError('Error fetching quota (project: %s, region: %s)' %
|
|
||||||
(project, region))
|
|
||||||
|
|
||||||
|
|
||||||
def _get_series(metric_labels, value, metric_type, timestamp, dt=None):
|
|
||||||
"""Create a Stackdriver monitoring time series from value and labels.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
metric_labels: dict with labels that will be used in the time series
|
|
||||||
value: time series value
|
|
||||||
metric_type: which metric is this series for
|
|
||||||
dt: datetime.datetime instance used for the series end time
|
|
||||||
"""
|
|
||||||
series = monitoring_v3.types.TimeSeries()
|
|
||||||
series.metric.type = metric_type
|
|
||||||
series.resource.type = 'global'
|
|
||||||
for label in metric_labels:
|
|
||||||
series.metric.labels[label] = metric_labels[label]
|
|
||||||
point = monitoring_v3.types.Point()
|
|
||||||
point.value.double_value = value
|
|
||||||
|
|
||||||
seconds = int(timestamp)
|
|
||||||
nanos = int((timestamp - seconds) * 10**9)
|
|
||||||
interval = monitoring_v3.TimeInterval(
|
|
||||||
{"end_time": {
|
|
||||||
"seconds": seconds,
|
|
||||||
"nanos": nanos
|
|
||||||
}})
|
|
||||||
point.interval = interval
|
|
||||||
|
|
||||||
series.points.append(point)
|
|
||||||
return series
|
|
||||||
|
|
||||||
|
|
||||||
def _quota_to_series_triplet(project, region, quota):
|
|
||||||
"""Convert API quota objects to three Stackdriver monitoring time series: usage, limit and utilization
|
|
||||||
|
|
||||||
Args:
|
|
||||||
project: set in converted time series labels
|
|
||||||
region: set in converted time series labels
|
|
||||||
quota: quota object received from the GCE API
|
|
||||||
"""
|
|
||||||
labels = dict()
|
|
||||||
labels['project'] = project
|
|
||||||
labels['region'] = region
|
|
||||||
|
|
||||||
try:
|
|
||||||
utilization = quota['usage'] / float(quota['limit'])
|
|
||||||
except ZeroDivisionError:
|
|
||||||
utilization = 0
|
|
||||||
now = time.time()
|
|
||||||
metric_type_prefix = _METRIC_TYPE_STEM + quota['metric'].lower() + '_'
|
|
||||||
return [
|
|
||||||
_get_series(labels, quota['usage'], metric_type_prefix + _USAGE, now),
|
|
||||||
_get_series(labels, quota['limit'], metric_type_prefix + _LIMIT, now),
|
|
||||||
_get_series(labels, utilization, metric_type_prefix + _UTILIZATION, now),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
|
||||||
@click.option('--monitoring-project', required=True,
|
|
||||||
help='monitoring project id')
|
|
||||||
@click.option('--gce-project', multiple=True,
|
|
||||||
help='project ids (multiple), defaults to monitoring project')
|
|
||||||
@click.option('--gce-region', multiple=True,
|
|
||||||
help='regions (multiple), defaults to "global"')
|
|
||||||
@click.option('--verbose', is_flag=True, help='Verbose output')
|
|
||||||
@click.argument('keywords', nargs=-1)
|
|
||||||
def main_cli(monitoring_project=None, gce_project=None, gce_region=None,
|
|
||||||
verbose=False, keywords=None):
|
|
||||||
"""Fetch GCE quotas and writes them as custom metrics to Stackdriver.
|
|
||||||
|
|
||||||
If KEYWORDS are specified as arguments, only quotas matching one of the
|
|
||||||
keywords will be stored in Stackdriver.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
_main(monitoring_project, gce_project, gce_region, verbose, keywords)
|
|
||||||
except RuntimeError:
|
|
||||||
logging.exception('exception raised')
|
|
||||||
|
|
||||||
|
|
||||||
def main(event, context):
|
|
||||||
"""Cloud Function entry point."""
|
|
||||||
try:
|
|
||||||
data = json.loads(base64.b64decode(event['data']).decode('utf-8'))
|
|
||||||
_main(os.environ.get('GCP_PROJECT'), **data)
|
|
||||||
# uncomment once https://issuetracker.google.com/issues/155215191 is fixed
|
|
||||||
# except RuntimeError:
|
|
||||||
# raise
|
|
||||||
except Exception:
|
|
||||||
logging.exception('exception in cloud function entry point')
|
|
||||||
|
|
||||||
|
|
||||||
def _main(monitoring_project, gce_project=None, gce_region=None, verbose=False,
|
|
||||||
keywords=None):
|
|
||||||
"""Module entry point used by cli and cloud function wrappers."""
|
|
||||||
_configure_logging(verbose=verbose)
|
|
||||||
gce_projects = gce_project or [monitoring_project]
|
|
||||||
gce_regions = gce_region or ['global']
|
|
||||||
keywords = set(keywords or [])
|
|
||||||
logging.debug('monitoring project %s', monitoring_project)
|
|
||||||
logging.debug('projects %s regions %s', gce_projects, gce_regions)
|
|
||||||
logging.debug('keywords %s', keywords)
|
|
||||||
quotas = []
|
|
||||||
compute = googleapiclient.discovery.build('compute', 'v1',
|
|
||||||
cache_discovery=False)
|
|
||||||
for project in gce_projects:
|
|
||||||
logging.debug('project %s', project)
|
|
||||||
for region in gce_regions:
|
|
||||||
logging.debug('region %s', region)
|
|
||||||
for quota in _fetch_quotas(project, region, compute=compute):
|
|
||||||
if keywords and not any(k in quota['metric'] for k in keywords):
|
|
||||||
# logging.debug('skipping %s', quota)
|
|
||||||
continue
|
|
||||||
logging.debug('quota %s', quota)
|
|
||||||
quotas.append((project, region, quota))
|
|
||||||
client, i = monitoring_v3.MetricServiceClient(), 0
|
|
||||||
|
|
||||||
x = len(quotas)
|
|
||||||
while i < len(quotas):
|
|
||||||
series = sum(
|
|
||||||
[_quota_to_series_triplet(*q) for q in quotas[i:i + _BATCH_SIZE]], [])
|
|
||||||
_add_series(monitoring_project, series, client)
|
|
||||||
i += _BATCH_SIZE
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main_cli()
|
|
|
@ -1,3 +0,0 @@
|
||||||
Click>=7.0
|
|
||||||
google-api-python-client>=1.10.1
|
|
||||||
google-cloud-monitoring>=1.1.0
|
|
Binary file not shown.
Before Width: | Height: | Size: 149 KiB After Width: | Height: | Size: 98 KiB |
|
@ -25,14 +25,13 @@ locals {
|
||||||
module "project" {
|
module "project" {
|
||||||
source = "../../../modules/project"
|
source = "../../../modules/project"
|
||||||
name = var.project_id
|
name = var.project_id
|
||||||
project_create = var.project_create
|
billing_account = try(var.project_create_config.billing_account, null)
|
||||||
|
parent = try(var.project_create_config.parent, null)
|
||||||
|
project_create = var.project_create_config != null
|
||||||
services = [
|
services = [
|
||||||
"compute.googleapis.com",
|
"compute.googleapis.com",
|
||||||
"cloudfunctions.googleapis.com"
|
"cloudfunctions.googleapis.com"
|
||||||
]
|
]
|
||||||
iam = {
|
|
||||||
"roles/monitoring.metricWriter" = [module.cf.service_account_iam_email]
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module "pubsub" {
|
module "pubsub" {
|
||||||
|
@ -47,50 +46,47 @@ module "pubsub" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "cf" {
|
module "cf" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
|
region = var.region
|
||||||
name = var.name
|
name = var.name
|
||||||
bucket_name = "${var.name}-${random_pet.random.id}"
|
bucket_name = "${var.name}-${random_pet.random.id}"
|
||||||
bucket_config = {
|
bucket_config = {
|
||||||
location = var.region
|
location = var.region
|
||||||
}
|
}
|
||||||
bundle_config = {
|
bundle_config = {
|
||||||
source_dir = "${path.module}/cf"
|
source_dir = "${path.module}/src"
|
||||||
output_path = var.bundle_path
|
output_path = var.bundle_path
|
||||||
}
|
}
|
||||||
# https://github.com/hashicorp/terraform-provider-archive/issues/40
|
|
||||||
# https://issuetracker.google.com/issues/155215191
|
|
||||||
environment_variables = {
|
|
||||||
USE_WORKER_V2 = "true"
|
|
||||||
PYTHON37_DRAIN_LOGS_ON_CRASH_WAIT_SEC = "5"
|
|
||||||
}
|
|
||||||
service_account_create = true
|
service_account_create = true
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub.topic.id
|
resource = module.pubsub.topic.id
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_cloud_scheduler_job" "job" {
|
resource "google_cloud_scheduler_job" "default" {
|
||||||
project = var.project_id
|
project = module.project.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
name = var.name
|
name = var.name
|
||||||
schedule = var.schedule_config
|
schedule = var.schedule_config
|
||||||
time_zone = "UTC"
|
time_zone = "UTC"
|
||||||
|
|
||||||
pubsub_target {
|
pubsub_target {
|
||||||
attributes = {}
|
attributes = {}
|
||||||
topic_name = module.pubsub.topic.id
|
topic_name = module.pubsub.topic.id
|
||||||
data = base64encode(jsonencode({
|
data = base64encode(jsonencode(merge(
|
||||||
gce_project = var.quota_config.projects
|
{ monitoring_project = var.project_id },
|
||||||
gce_region = var.quota_config.regions
|
var.quota_config
|
||||||
keywords = var.quota_config.filters
|
)))
|
||||||
}))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "google_project_iam_member" "metric_writer" {
|
||||||
|
project = module.project.project_id
|
||||||
|
role = "roles/monitoring.metricWriter"
|
||||||
|
member = module.cf.service_account_iam_email
|
||||||
|
}
|
||||||
|
|
||||||
resource "google_project_iam_member" "network_viewer" {
|
resource "google_project_iam_member" "network_viewer" {
|
||||||
for_each = toset(local.projects)
|
for_each = toset(local.projects)
|
||||||
project = each.key
|
project = each.key
|
||||||
|
@ -105,17 +101,16 @@ resource "google_project_iam_member" "quota_viewer" {
|
||||||
member = module.cf.service_account_iam_email
|
member = module.cf.service_account_iam_email
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "google_monitoring_alert_policy" "default" {
|
||||||
resource "google_monitoring_alert_policy" "alert_policy" {
|
for_each = var.alert_configs
|
||||||
count = var.alert_create ? 1 : 0
|
|
||||||
project = module.project.project_id
|
project = module.project.project_id
|
||||||
display_name = "Quota monitor"
|
display_name = "Monitor quota ${each.key}"
|
||||||
combiner = "OR"
|
combiner = "OR"
|
||||||
conditions {
|
conditions {
|
||||||
display_name = "simple quota threshold for cpus utilization"
|
display_name = "Threshold ${each.value.threshold} for ${each.key}."
|
||||||
condition_threshold {
|
condition_threshold {
|
||||||
filter = "metric.type=\"custom.googleapis.com/quota/cpus_utilization\" resource.type=\"global\""
|
filter = "metric.type=\"custom.googleapis.com/quota/${each.key}\" resource.type=\"global\""
|
||||||
threshold_value = 0.75
|
threshold_value = each.value.threshold
|
||||||
comparison = "COMPARISON_GT"
|
comparison = "COMPARISON_GT"
|
||||||
duration = "0s"
|
duration = "0s"
|
||||||
aggregations {
|
aggregations {
|
||||||
|
@ -129,16 +124,17 @@ resource "google_monitoring_alert_policy" "alert_policy" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
enabled = false
|
enabled = each.value.enabled
|
||||||
user_labels = {
|
user_labels = each.value.labels
|
||||||
name = var.name
|
|
||||||
}
|
|
||||||
documentation {
|
documentation {
|
||||||
content = "GCE cpus quota over threshold."
|
content = (
|
||||||
|
each.value.documentation != null
|
||||||
|
? each.value.documentation
|
||||||
|
: "Quota over threshold of ${each.value.threshold} for ${each.key}."
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
resource "random_pet" "random" {
|
resource "random_pet" "random" {
|
||||||
length = 1
|
length = 1
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,233 @@
|
||||||
|
#! /usr/bin/env python3
|
||||||
|
# Copyright 2023 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""Sync GCE quota usage to Stackdriver for multiple projects.
|
||||||
|
|
||||||
|
This tool fetches global and/or regional quotas from the GCE API for
|
||||||
|
multiple projects, and sends them to Stackdriver as custom metrics, where they
|
||||||
|
can be used to set alert policies or create charts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import collections
|
||||||
|
import datetime
|
||||||
|
import itertools
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
import click
|
||||||
|
import google.auth
|
||||||
|
|
||||||
|
from google.auth.transport.requests import AuthorizedSession
|
||||||
|
|
||||||
|
BASE = 'custom.googleapis.com/quota'
|
||||||
|
HTTP = AuthorizedSession(google.auth.default()[0])
|
||||||
|
HTTP_HEADERS = {'content-type': 'application/json; charset=UTF-8'}
|
||||||
|
URL_PROJECT = 'https://compute.googleapis.com/compute/v1/projects/{}'
|
||||||
|
URL_REGION = 'https://compute.googleapis.com/compute/v1/projects/{}/regions/{}'
|
||||||
|
URL_TS = 'https://monitoring.googleapis.com/v3/projects/{}/timeSeries'
|
||||||
|
|
||||||
|
_Quota = collections.namedtuple('_Quota',
|
||||||
|
'project region tstamp metric limit usage')
|
||||||
|
HTTPRequest = collections.namedtuple(
|
||||||
|
'HTTPRequest', 'url data headers', defaults=[{}, {
|
||||||
|
'content-type': 'application/json; charset=UTF-8'
|
||||||
|
}])
|
||||||
|
|
||||||
|
|
||||||
|
class Quota(_Quota):
|
||||||
|
'Compute quota.'
|
||||||
|
|
||||||
|
def _api_format(self, name, value):
|
||||||
|
'Return a specific timeseries for this quota in API format.'
|
||||||
|
d = {
|
||||||
|
'metric': {
|
||||||
|
'type': f'{BASE}/{self.metric.lower()}/{name}',
|
||||||
|
'labels': {
|
||||||
|
'location': self.region,
|
||||||
|
'project': self.project
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'resource': {
|
||||||
|
'type': 'global',
|
||||||
|
'labels': {}
|
||||||
|
},
|
||||||
|
'metricKind':
|
||||||
|
'GAUGE',
|
||||||
|
'points': [{
|
||||||
|
'interval': {
|
||||||
|
'endTime': f'{self.tstamp.isoformat("T")}Z'
|
||||||
|
},
|
||||||
|
'value': {}
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
if name == 'ratio':
|
||||||
|
d['valueType'] = 'DOUBLE'
|
||||||
|
d['points'][0]['value'] = {'doubleValue': value}
|
||||||
|
else:
|
||||||
|
d['valueType'] = 'INT64'
|
||||||
|
d['points'][0]['value'] = {'int64Value': value}
|
||||||
|
# remove this label if cardinality gets too high
|
||||||
|
d['metric']['labels']['quota'] = f'{self.usage}/{self.limit}'
|
||||||
|
return d
|
||||||
|
|
||||||
|
@property
|
||||||
|
def timeseries(self):
|
||||||
|
try:
|
||||||
|
ratio = self.usage / float(self.limit)
|
||||||
|
except ZeroDivisionError:
|
||||||
|
ratio = 0
|
||||||
|
yield self._api_format('ratio', ratio)
|
||||||
|
yield self._api_format('usage', self.usage)
|
||||||
|
# yield self._api_format('limit', self.limit)
|
||||||
|
|
||||||
|
|
||||||
|
def batched(iterable, n):
|
||||||
|
'Batches data into lists of length n. The last batch may be shorter.'
|
||||||
|
# batched('ABCDEFG', 3) --> ABC DEF G
|
||||||
|
if n < 1:
|
||||||
|
raise ValueError('n must be at least one')
|
||||||
|
it = iter(iterable)
|
||||||
|
while (batch := list(itertools.islice(it, n))):
|
||||||
|
yield batch
|
||||||
|
|
||||||
|
|
||||||
|
def configure_logging(verbose=True):
|
||||||
|
'Basic logging configuration.'
|
||||||
|
level = logging.DEBUG if verbose else logging.INFO
|
||||||
|
logging.basicConfig(level=level)
|
||||||
|
warnings.filterwarnings('ignore', r'.*end user credentials.*', UserWarning)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(request, delete=False):
|
||||||
|
'Minimal HTTP client interface for API calls.'
|
||||||
|
logging.debug(f'fetch {"POST" if request.data else "GET"} {request.url}')
|
||||||
|
logging.debug(request.data)
|
||||||
|
try:
|
||||||
|
if delete:
|
||||||
|
response = HTTP.delete(request.url, headers=request.headers)
|
||||||
|
elif not request.data:
|
||||||
|
response = HTTP.get(request.url, headers=request.headers)
|
||||||
|
else:
|
||||||
|
response = HTTP.post(request.url, headers=request.headers,
|
||||||
|
data=json.dumps(request.data))
|
||||||
|
except google.auth.exceptions.RefreshError as e:
|
||||||
|
raise SystemExit(e.args[0])
|
||||||
|
try:
|
||||||
|
rdata = json.loads(response.content)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
logging.critical(e)
|
||||||
|
raise SystemExit(f'Error decoding response: {response.content}')
|
||||||
|
if response.status_code != 200:
|
||||||
|
logging.critical(rdata)
|
||||||
|
error = rdata.get('error', {})
|
||||||
|
raise SystemExit('API error: {} (HTTP {})'.format(
|
||||||
|
error.get('message', 'error message cannot be decoded'),
|
||||||
|
error.get('code', 'no code found')))
|
||||||
|
return json.loads(response.content)
|
||||||
|
|
||||||
|
|
||||||
|
def write_timeseries(project, data):
|
||||||
|
'Sends timeseries to the API.'
|
||||||
|
# try
|
||||||
|
logging.debug(f'write {len(data["timeSeries"])} timeseries')
|
||||||
|
request = HTTPRequest(URL_TS.format(project), data)
|
||||||
|
return fetch(request)
|
||||||
|
|
||||||
|
|
||||||
|
def get_quotas(project, region='global'):
|
||||||
|
'Fetch GCE per-project or per-region quotas from the API.'
|
||||||
|
if region == 'global':
|
||||||
|
request = HTTPRequest(URL_PROJECT.format(project))
|
||||||
|
else:
|
||||||
|
request = HTTPRequest(URL_REGION.format(project, region))
|
||||||
|
resp = fetch(request)
|
||||||
|
ts = datetime.datetime.utcnow()
|
||||||
|
for quota in resp.get('quotas'):
|
||||||
|
yield Quota(project, region, ts, **quota)
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.argument('project-id', required=True)
|
||||||
|
@click.option(
|
||||||
|
'--project-ids', multiple=True, help=
|
||||||
|
'Project ids to monitor (multiple). Defaults to monitoring project if not set.'
|
||||||
|
)
|
||||||
|
@click.option('--regions', multiple=True,
|
||||||
|
help='Regions (multiple). Defaults to "global" if not set.')
|
||||||
|
@click.option('--include', multiple=True,
|
||||||
|
help='Only include quotas starting with keyword (multiple).')
|
||||||
|
@click.option('--exclude', multiple=True,
|
||||||
|
help='Exclude quotas starting with keyword (multiple).')
|
||||||
|
@click.option('--dry-run', is_flag=True, help='Do not write metrics.')
|
||||||
|
@click.option('--verbose', is_flag=True, help='Verbose output.')
|
||||||
|
def main_cli(project_id=None, project_ids=None, regions=None, include=None,
|
||||||
|
exclude=None, dry_run=False, verbose=False):
|
||||||
|
'Fetch GCE quotas and writes them as custom metrics to Stackdriver.'
|
||||||
|
try:
|
||||||
|
_main(project_id, project_ids, regions, include, exclude, dry_run, verbose)
|
||||||
|
except RuntimeError as e:
|
||||||
|
logging.exception(f'exception raised: {e.args[0]}')
|
||||||
|
|
||||||
|
|
||||||
|
def main(event, context):
|
||||||
|
"""Cloud Function entry point."""
|
||||||
|
try:
|
||||||
|
data = json.loads(base64.b64decode(event['data']).decode('utf-8'))
|
||||||
|
_main(**data)
|
||||||
|
except RuntimeError:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _main(monitoring_project, projects=None, regions=None, include=None,
|
||||||
|
exclude=None, dry_run=False, verbose=False):
|
||||||
|
"""Module entry point used by cli and cloud function wrappers."""
|
||||||
|
configure_logging(verbose=verbose)
|
||||||
|
projects = projects or [monitoring_project]
|
||||||
|
regions = regions or ['global']
|
||||||
|
include = set(include or [])
|
||||||
|
exclude = set(exclude or [])
|
||||||
|
for k in ('monitoring_project', 'projects', 'regions', 'include', 'exclude'):
|
||||||
|
logging.debug(f'{k} {locals().get(k)}')
|
||||||
|
timeseries = []
|
||||||
|
logging.info(f'get quotas ({len(projects)} projects {len(regions)} regions)')
|
||||||
|
for project in projects:
|
||||||
|
for region in regions:
|
||||||
|
logging.info(f'get quota for {project} in {region}')
|
||||||
|
for quota in get_quotas(project, region):
|
||||||
|
metric = quota.metric.lower()
|
||||||
|
if include and not any(metric.startswith(k) for k in include):
|
||||||
|
logging.debug(f'skipping {project}:{region}:{metric} not included')
|
||||||
|
continue
|
||||||
|
if exclude and any(metric.startswith(k) for k in exclude):
|
||||||
|
logging.debug(f'skipping {project}:{region}:{metric} excluded')
|
||||||
|
continue
|
||||||
|
logging.debug(f'quota {project}:{region}:{metric}')
|
||||||
|
timeseries += list(quota.timeseries)
|
||||||
|
logging.info(f'{len(timeseries)} timeseries')
|
||||||
|
i, l = 0, len(timeseries)
|
||||||
|
for batch in batched(timeseries, 30):
|
||||||
|
data = list(batch)
|
||||||
|
logging.info(f'sending {len(batch)} timeseries out of {l - i}/{l} left')
|
||||||
|
i += len(batch)
|
||||||
|
if not dry_run:
|
||||||
|
write_timeseries(monitoring_project, {'timeSeries': list(data)})
|
||||||
|
elif verbose:
|
||||||
|
print(data)
|
||||||
|
logging.info(f'{l} timeseries done (dry run {dry_run})')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main_cli()
|
|
@ -0,0 +1,4 @@
|
||||||
|
click
|
||||||
|
functions-framework
|
||||||
|
google-api-core
|
||||||
|
google-cloud-monitoring
|
|
@ -0,0 +1,104 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# Copyright 2023 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
'Manages metric descriptors for the DR metrics.'
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
import click
|
||||||
|
import google.auth
|
||||||
|
|
||||||
|
from google.auth.transport.requests import AuthorizedSession
|
||||||
|
|
||||||
|
Descriptor = collections.namedtuple('Descriptor', 'name labels is_bool r_type',
|
||||||
|
defaults=[True, 'global'])
|
||||||
|
HTTPRequest = collections.namedtuple(
|
||||||
|
'HTTPRequest', 'url data headers', defaults=[{}, {
|
||||||
|
'content-type': 'application/json; charset=UTF-8'
|
||||||
|
}])
|
||||||
|
|
||||||
|
|
||||||
|
class Error(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
BASE = 'custom.googleapis.com/quota'
|
||||||
|
HTTP = AuthorizedSession(google.auth.default()[0])
|
||||||
|
|
||||||
|
|
||||||
|
def descriptors_get(project):
|
||||||
|
base = urllib.parse.quote_plus(BASE)
|
||||||
|
url = (f'https://content-monitoring.googleapis.com/v3/projects/{project}/'
|
||||||
|
'metricDescriptors?filter=metric.type%20%3D%20starts_with'
|
||||||
|
f'(%22{base}%22)')
|
||||||
|
return HTTPRequest(url)
|
||||||
|
|
||||||
|
|
||||||
|
def descriptor_delete(project, type):
|
||||||
|
url = (f'https://monitoring.googleapis.com/v3/projects/{project}/'
|
||||||
|
f'metricDescriptors/{type}')
|
||||||
|
return HTTPRequest(url)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(request, delete=False):
|
||||||
|
'Minimal HTTP client interface for API calls.'
|
||||||
|
# try
|
||||||
|
logging.debug(f'fetch {"POST" if request.data else "GET"} {request.url}')
|
||||||
|
try:
|
||||||
|
if delete:
|
||||||
|
response = HTTP.delete(request.url, headers=request.headers)
|
||||||
|
elif not request.data:
|
||||||
|
response = HTTP.get(request.url, headers=request.headers)
|
||||||
|
else:
|
||||||
|
response = HTTP.post(request.url, headers=request.headers,
|
||||||
|
data=json.dumps(request.data))
|
||||||
|
except google.auth.exceptions.RefreshError as e:
|
||||||
|
raise SystemExit(e.args[0])
|
||||||
|
if response.status_code != 200:
|
||||||
|
logging.critical(
|
||||||
|
f'response code {response.status_code} for URL {request.url}')
|
||||||
|
logging.critical(response.content)
|
||||||
|
logging.debug(request.data)
|
||||||
|
raise Error('API error')
|
||||||
|
return json.loads(response.content)
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.argument('project')
|
||||||
|
@click.option('--delete', default=False, is_flag=True,
|
||||||
|
help='Delete descriptors.')
|
||||||
|
@click.option('--dry-run', default=False, is_flag=True,
|
||||||
|
help='Show but to not perform actions.')
|
||||||
|
def main(project, delete=False, dry_run=False):
|
||||||
|
'Program entry point.'
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logging.info(f'getting descriptors for "{BASE}"')
|
||||||
|
response = fetch(descriptors_get(project))
|
||||||
|
existing = [d['type'] for d in response.get('metricDescriptors', [])]
|
||||||
|
logging.info(f'{len(existing)} descriptors')
|
||||||
|
if delete:
|
||||||
|
for name in existing:
|
||||||
|
logging.info(f'deleting descriptor {name}')
|
||||||
|
if not dry_run:
|
||||||
|
try:
|
||||||
|
fetch(descriptor_delete(project, name), delete=True)
|
||||||
|
except Error:
|
||||||
|
logging.critical(f'error deleting descriptor {name}')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -14,10 +14,20 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
variable "alert_create" {
|
variable "alert_configs" {
|
||||||
description = "Enables the creation of a sample monitoring alert, false by default."
|
description = "Configure creation of monitoring alerts for specific quotas. Keys match quota names."
|
||||||
type = bool
|
type = map(object({
|
||||||
default = false
|
documentation = optional(string)
|
||||||
|
enabled = optional(bool)
|
||||||
|
labels = optional(map(string))
|
||||||
|
threshold = optional(number, 0.75)
|
||||||
|
}))
|
||||||
|
nullable = false
|
||||||
|
default = {}
|
||||||
|
validation {
|
||||||
|
condition = alltrue([for k, v in var.alert_configs : v != null])
|
||||||
|
error_message = "Set values as {} instead of null."
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "bundle_path" {
|
variable "bundle_path" {
|
||||||
|
@ -32,10 +42,13 @@ variable "name" {
|
||||||
default = "quota-monitor"
|
default = "quota-monitor"
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "project_create" {
|
variable "project_create_config" {
|
||||||
description = "Create project instead of using an existing one."
|
description = "Create project instead of using an existing one."
|
||||||
type = bool
|
type = object({
|
||||||
default = false
|
billing_account = string
|
||||||
|
parent = optional(string)
|
||||||
|
})
|
||||||
|
default = null
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "project_id" {
|
variable "project_id" {
|
||||||
|
@ -46,15 +59,18 @@ variable "project_id" {
|
||||||
variable "quota_config" {
|
variable "quota_config" {
|
||||||
description = "Cloud function configuration."
|
description = "Cloud function configuration."
|
||||||
type = object({
|
type = object({
|
||||||
filters = list(string)
|
exclude = optional(list(string), [
|
||||||
projects = list(string)
|
"a2", "c2", "c2d", "committed", "g2", "interconnect", "m1", "m2", "m3",
|
||||||
regions = list(string)
|
"nvidia", "preemptible"
|
||||||
|
])
|
||||||
|
include = optional(list(string))
|
||||||
|
projects = optional(list(string))
|
||||||
|
regions = optional(list(string))
|
||||||
|
dry_run = optional(bool, false)
|
||||||
|
verbose = optional(bool, false)
|
||||||
})
|
})
|
||||||
default = {
|
nullable = false
|
||||||
filters = null
|
default = {}
|
||||||
projects = null
|
|
||||||
regions = null
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "region" {
|
variable "region" {
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -85,7 +85,7 @@ module "pubsub_file" {
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
|
||||||
module "cf" {
|
module "cf" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
name = var.name
|
name = var.name
|
||||||
|
@ -99,16 +99,14 @@ module "cf" {
|
||||||
}
|
}
|
||||||
service_account = module.service-account.email
|
service_account = module.service-account.email
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub.topic.id
|
resource = module.pubsub.topic.id
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module "cffile" {
|
module "cffile" {
|
||||||
count = var.cai_gcs_export ? 1 : 0
|
count = var.cai_gcs_export ? 1 : 0
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
name = var.name_cffile
|
name = var.name_cffile
|
||||||
|
@ -124,12 +122,10 @@ module "cffile" {
|
||||||
}
|
}
|
||||||
service_account = module.service-account.email
|
service_account = module.service-account.email
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub_file.topic.id
|
resource = module.pubsub_file.topic.id
|
||||||
retry = null
|
retry = null
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "random_pet" "random" {
|
resource "random_pet" "random" {
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -128,5 +128,5 @@ module "test" {
|
||||||
billing_account = "123456-123456-123456"
|
billing_account = "123456-123456-123456"
|
||||||
project_create = true
|
project_create = true
|
||||||
}
|
}
|
||||||
# tftest modules=11 resources=35
|
# tftest modules=11 resources=37
|
||||||
```
|
```
|
||||||
|
|
|
@ -108,7 +108,7 @@ module "pubsub" {
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
|
||||||
module "cf-restarter" {
|
module "cf-restarter" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "cf-restarter"
|
name = "cf-restarter"
|
||||||
region = var.region
|
region = var.region
|
||||||
|
@ -132,16 +132,14 @@ module "cf-restarter" {
|
||||||
}
|
}
|
||||||
|
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
v1 = {
|
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub.topic.id
|
resource = module.pubsub.topic.id
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module "cf-healthchecker" {
|
module "cf-healthchecker" {
|
||||||
source = "../../../modules/cloud-function"
|
source = "../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "cf-healthchecker"
|
name = "cf-healthchecker"
|
||||||
region = var.region
|
region = var.region
|
||||||
|
@ -172,18 +170,14 @@ module "cf-healthchecker" {
|
||||||
create = true
|
create = true
|
||||||
name = "hc-connector"
|
name = "hc-connector"
|
||||||
egress_settings = "PRIVATE_RANGES_ONLY"
|
egress_settings = "PRIVATE_RANGES_ONLY"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
vpc_connector_config = {
|
vpc_connector_config = {
|
||||||
ip_cidr_range = "10.132.0.0/28"
|
ip_cidr_range = "10.132.0.0/28"
|
||||||
network = "vpc"
|
network = "vpc"
|
||||||
}
|
}
|
||||||
|
|
||||||
iam = {
|
iam = {
|
||||||
"roles/cloudfunctions.invoker" = [module.service-account-scheduler.iam_email]
|
"roles/cloudfunctions.invoker" = [module.service-account-scheduler.iam_email]
|
||||||
}
|
}
|
||||||
|
|
||||||
depends_on = [
|
depends_on = [
|
||||||
module.vpc
|
module.vpc
|
||||||
]
|
]
|
||||||
|
|
|
@ -52,5 +52,5 @@ module "test" {
|
||||||
migration_admin_users = ["user:admin@example.com"]
|
migration_admin_users = ["user:admin@example.com"]
|
||||||
migration_viewer_users = ["user:viewer@example.com"]
|
migration_viewer_users = ["user:viewer@example.com"]
|
||||||
}
|
}
|
||||||
# tftest modules=5 resources=20
|
# tftest modules=5 resources=22
|
||||||
```
|
```
|
||||||
|
|
|
@ -98,5 +98,5 @@ module "test" {
|
||||||
prefix = "prefix"
|
prefix = "prefix"
|
||||||
}
|
}
|
||||||
|
|
||||||
# tftest modules=9 resources=48
|
# tftest modules=9 resources=50
|
||||||
```
|
```
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -180,5 +180,5 @@ module "test" {
|
||||||
}
|
}
|
||||||
prefix = "prefix"
|
prefix = "prefix"
|
||||||
}
|
}
|
||||||
# tftest modules=10 resources=50
|
# tftest modules=10 resources=52
|
||||||
```
|
```
|
||||||
|
|
|
@ -66,5 +66,5 @@ module "test" {
|
||||||
}
|
}
|
||||||
prefix = "prefix"
|
prefix = "prefix"
|
||||||
}
|
}
|
||||||
# tftest modules=8 resources=27
|
# tftest modules=8 resources=29
|
||||||
```
|
```
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -125,5 +125,5 @@ module "test" {
|
||||||
}
|
}
|
||||||
prefix = "prefix"
|
prefix = "prefix"
|
||||||
}
|
}
|
||||||
# tftest modules=5 resources=26
|
# tftest modules=5 resources=28
|
||||||
```
|
```
|
||||||
|
|
|
@ -166,9 +166,8 @@ module "orch-nat" {
|
||||||
module "orch-artifact-reg" {
|
module "orch-artifact-reg" {
|
||||||
source = "../../../modules/artifact-registry"
|
source = "../../../modules/artifact-registry"
|
||||||
project_id = module.orch-project.project_id
|
project_id = module.orch-project.project_id
|
||||||
id = "${var.prefix}-app-images"
|
name = "${var.prefix}-app-images"
|
||||||
location = var.region
|
location = var.region
|
||||||
format = "DOCKER"
|
|
||||||
description = "Docker repository storing application images e.g. Dataflow, Cloud Run etc..."
|
description = "Docker repository storing application images e.g. Dataflow, Cloud Run etc..."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -64,7 +64,7 @@ module "common-datacatalog" {
|
||||||
tags = var.data_catalog_tags
|
tags = var.data_catalog_tags
|
||||||
}
|
}
|
||||||
|
|
||||||
# To create KMS keys in the common projet: uncomment this section and assigne key links accondingly in local.service_encryption_keys variable
|
# To create KMS keys in the common project: uncomment this section and assigne key links accondingly in local.service_encryption_keys variable
|
||||||
|
|
||||||
# module "cmn-kms-0" {
|
# module "cmn-kms-0" {
|
||||||
# source = "../../../modules/kms"
|
# source = "../../../modules/kms"
|
||||||
|
|
|
@ -174,6 +174,10 @@ To deploy this blueprint on your GCP organization, you will need
|
||||||
|
|
||||||
- a folder or organization where new projects will be created
|
- a folder or organization where new projects will be created
|
||||||
- a billing account that will be associated with the new projects
|
- a billing account that will be associated with the new projects
|
||||||
|
- user groups defined within the organization (provided as `organization_domain` variable):
|
||||||
|
- gcp-data-analysts
|
||||||
|
- gcp-data-engineers
|
||||||
|
- gcp-data-security
|
||||||
|
|
||||||
The Data Platform is meant to be executed by a Service Account (or a regular user) having this minimal set of permission:
|
The Data Platform is meant to be executed by a Service Account (or a regular user) having this minimal set of permission:
|
||||||
|
|
||||||
|
@ -226,7 +230,7 @@ module "data-platform" {
|
||||||
prefix = "myprefix"
|
prefix = "myprefix"
|
||||||
}
|
}
|
||||||
|
|
||||||
# tftest modules=43 resources=279
|
# tftest modules=43 resources=285
|
||||||
```
|
```
|
||||||
|
|
||||||
## Customizations
|
## Customizations
|
||||||
|
@ -254,24 +258,23 @@ The application layer is out of scope of this script. As a demo purpuse only, se
|
||||||
|
|
||||||
You can find examples in the `[demo](./demo)` folder.
|
You can find examples in the `[demo](./demo)` folder.
|
||||||
<!-- BEGIN TFDOC -->
|
<!-- BEGIN TFDOC -->
|
||||||
|
|
||||||
## Variables
|
## Variables
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [organization_domain](variables.tf#L156) | Organization domain. | <code>string</code> | ✓ | |
|
| [organization_domain](variables.tf#L159) | Organization domain. | <code>string</code> | ✓ | |
|
||||||
| [prefix](variables.tf#L161) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
| [prefix](variables.tf#L164) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||||
| [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ drop = string load = string orc = string trf = string dwh-lnd = string dwh-cur = string dwh-conf = string common = string exp = string }), { drop = "drp" load = "lod" orc = "orc" trf = "trf" dwh-lnd = "dwh-lnd" dwh-cur = "dwh-cur" dwh-conf = "dwh-conf" common = "cmn" exp = "exp" } ) })">object({…})</code> | ✓ | |
|
| [project_config](variables.tf#L173) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ drop = string load = string orc = string trf = string dwh-lnd = string dwh-cur = string dwh-conf = string common = string exp = string }), { drop = "drp" load = "lod" orc = "orc" trf = "trf" dwh-lnd = "dwh-lnd" dwh-cur = "dwh-cur" dwh-conf = "dwh-conf" common = "cmn" exp = "exp" } ) })">object({…})</code> | ✓ | |
|
||||||
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string }), { image_version = "composer-2-airflow-2" }) workloads_config = optional(object({ scheduler = optional(object( { cpu = number memory_gb = number storage_gb = number count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 }) web_server = optional(object( { cpu = number memory_gb = number storage_gb = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 }) worker = optional(object( { cpu = number memory_gb = number storage_gb = number min_count = number max_count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 }) })) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } workloads_config = { scheduler = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } web_server = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } worker = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } } }">{…}</code> |
|
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string }), { image_version = "composer-2-airflow-2" }) workloads_config = optional(object({ scheduler = optional(object( { cpu = number memory_gb = number storage_gb = number count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 }) web_server = optional(object( { cpu = number memory_gb = number storage_gb = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 }) worker = optional(object( { cpu = number memory_gb = number storage_gb = number min_count = number max_count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 }) })) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } workloads_config = { scheduler = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } web_server = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } worker = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } } }">{…}</code> |
|
||||||
| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code title="{ "3_Confidential" = null "2_Private" = null "1_Sensitive" = null }">{…}</code> |
|
| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map(object({ description = optional(string) iam = optional(map(list(string)), {}) }))">map(object({…}))</code> | | <code title="{ "3_Confidential" = {} "2_Private" = {} "1_Sensitive" = {} }">{…}</code> |
|
||||||
| [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
| [data_force_destroy](variables.tf#L114) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||||
| [groups](variables.tf#L117) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
| [groups](variables.tf#L120) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||||
| [location](variables.tf#L127) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
| [location](variables.tf#L130) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
||||||
| [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string }) composer_secondary_ranges = object({ pods = string services = string }) })">object({…})</code> | | <code>null</code> |
|
| [network_config](variables.tf#L136) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string }) composer_secondary_ranges = object({ pods = string services = string }) })">object({…})</code> | | <code>null</code> |
|
||||||
| [project_services](variables.tf#L204) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
| [project_services](variables.tf#L207) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||||
| [project_suffix](variables.tf#L215) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
| [project_suffix](variables.tf#L218) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||||
| [region](variables.tf#L221) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [region](variables.tf#L224) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
| [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> |
|
| [service_encryption_keys](variables.tf#L230) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
@ -282,10 +285,9 @@ You can find examples in the `[demo](./demo)` folder.
|
||||||
| [df_template](outputs.tf#L49) | Dataflow template image and template details. | |
|
| [df_template](outputs.tf#L49) | Dataflow template image and template details. | |
|
||||||
| [gcs-buckets](outputs.tf#L58) | GCS buckets. | |
|
| [gcs-buckets](outputs.tf#L58) | GCS buckets. | |
|
||||||
| [kms_keys](outputs.tf#L71) | Cloud MKS keys. | |
|
| [kms_keys](outputs.tf#L71) | Cloud MKS keys. | |
|
||||||
| [projects](outputs.tf#L76) | GCP Projects informations. | |
|
| [projects](outputs.tf#L76) | GCP Projects information. | |
|
||||||
| [vpc_network](outputs.tf#L102) | VPC network. | |
|
| [vpc_network](outputs.tf#L102) | VPC network. | |
|
||||||
| [vpc_subnet](outputs.tf#L111) | VPC subnetworks. | |
|
| [vpc_subnet](outputs.tf#L111) | VPC subnetworks. | |
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
## TODOs
|
## TODOs
|
||||||
|
|
||||||
|
@ -307,5 +309,5 @@ module "test" {
|
||||||
}
|
}
|
||||||
prefix = "prefix"
|
prefix = "prefix"
|
||||||
}
|
}
|
||||||
# tftest modules=43 resources=279
|
# tftest modules=43 resources=285
|
||||||
```
|
```
|
||||||
|
|
|
@ -283,7 +283,7 @@ with models.DAG(
|
||||||
'datasetId': DWH_CURATED_BQ_DATASET,
|
'datasetId': DWH_CURATED_BQ_DATASET,
|
||||||
'tableId': 'customer_purchase'
|
'tableId': 'customer_purchase'
|
||||||
},
|
},
|
||||||
'writeDisposition':'WRITE_TRUNCATE',
|
'writeDisposition':'WRITE_APPEND',
|
||||||
"useLegacySql": False
|
"useLegacySql": False
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -313,7 +313,7 @@ with models.DAG(
|
||||||
'datasetId': DWH_CONFIDENTIAL_BQ_DATASET,
|
'datasetId': DWH_CONFIDENTIAL_BQ_DATASET,
|
||||||
'tableId': 'customer_purchase'
|
'tableId': 'customer_purchase'
|
||||||
},
|
},
|
||||||
'writeDisposition':'WRITE_TRUNCATE',
|
'writeDisposition':'WRITE_APPEND',
|
||||||
"useLegacySql": False
|
"useLegacySql": False
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -414,7 +414,7 @@ with models.DAG('data_pipeline_dc_tags_dag_flex',
|
||||||
'tableId': 'customer_purchase'
|
'tableId': 'customer_purchase'
|
||||||
},
|
},
|
||||||
'writeDisposition':
|
'writeDisposition':
|
||||||
'WRITE_TRUNCATE',
|
'WRITE_APPEND',
|
||||||
"useLegacySql":
|
"useLegacySql":
|
||||||
False
|
False
|
||||||
}
|
}
|
||||||
|
@ -449,7 +449,7 @@ with models.DAG('data_pipeline_dc_tags_dag_flex',
|
||||||
'tableId': 'customer_purchase'
|
'tableId': 'customer_purchase'
|
||||||
},
|
},
|
||||||
'writeDisposition':
|
'writeDisposition':
|
||||||
'WRITE_TRUNCATE',
|
'WRITE_APPEND',
|
||||||
"useLegacySql":
|
"useLegacySql":
|
||||||
False
|
False
|
||||||
}
|
}
|
||||||
|
|
|
@ -74,7 +74,7 @@ output "kms_keys" {
|
||||||
}
|
}
|
||||||
|
|
||||||
output "projects" {
|
output "projects" {
|
||||||
description = "GCP Projects informations."
|
description = "GCP Projects information."
|
||||||
value = {
|
value = {
|
||||||
project_number = {
|
project_number = {
|
||||||
dwh-landing = module.dwh-lnd-project.number,
|
dwh-landing = module.dwh-lnd-project.number,
|
||||||
|
|
|
@ -99,12 +99,15 @@ variable "composer_config" {
|
||||||
|
|
||||||
variable "data_catalog_tags" {
|
variable "data_catalog_tags" {
|
||||||
description = "List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format."
|
description = "List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format."
|
||||||
type = map(map(list(string)))
|
type = map(object({
|
||||||
|
description = optional(string)
|
||||||
|
iam = optional(map(list(string)), {})
|
||||||
|
}))
|
||||||
nullable = false
|
nullable = false
|
||||||
default = {
|
default = {
|
||||||
"3_Confidential" = null
|
"3_Confidential" = {}
|
||||||
"2_Private" = null
|
"2_Private" = {}
|
||||||
"1_Sensitive" = null
|
"1_Sensitive" = {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,7 @@
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
iam_lnd = {
|
iam_lnd = {
|
||||||
"roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
|
"roles/storage.objectCreator" = [module.land-sa-0.iam_email]
|
||||||
"roles/storage.objectViewer" = [module.processing-sa-cmp-0.iam_email]
|
"roles/storage.objectViewer" = [module.processing-sa-cmp-0.iam_email]
|
||||||
"roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
|
"roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
|
||||||
}
|
}
|
||||||
|
@ -36,6 +36,9 @@ module "land-project" {
|
||||||
iam = var.project_config.billing_account_id != null ? local.iam_lnd : null
|
iam = var.project_config.billing_account_id != null ? local.iam_lnd : null
|
||||||
iam_additive = var.project_config.billing_account_id == null ? local.iam_lnd : null
|
iam_additive = var.project_config.billing_account_id == null ? local.iam_lnd : null
|
||||||
services = [
|
services = [
|
||||||
|
"bigquery.googleapis.com",
|
||||||
|
"bigqueryreservation.googleapis.com",
|
||||||
|
"bigquerystorage.googleapis.com",
|
||||||
"cloudkms.googleapis.com",
|
"cloudkms.googleapis.com",
|
||||||
"cloudresourcemanager.googleapis.com",
|
"cloudresourcemanager.googleapis.com",
|
||||||
"iam.googleapis.com",
|
"iam.googleapis.com",
|
||||||
|
@ -52,12 +55,12 @@ module "land-project" {
|
||||||
|
|
||||||
# Cloud Storage
|
# Cloud Storage
|
||||||
|
|
||||||
module "land-sa-cs-0" {
|
module "land-sa-0" {
|
||||||
source = "../../../modules/iam-service-account"
|
source = "../../../modules/iam-service-account"
|
||||||
project_id = module.land-project.project_id
|
project_id = module.land-project.project_id
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
name = "lnd-cs-0"
|
name = "lnd-sa-0"
|
||||||
display_name = "Data platform GCS landing service account."
|
display_name = "Data platform landing zone service account."
|
||||||
iam = {
|
iam = {
|
||||||
"roles/iam.serviceAccountTokenCreator" = [
|
"roles/iam.serviceAccountTokenCreator" = [
|
||||||
local.groups_iam.data-engineers
|
local.groups_iam.data-engineers
|
||||||
|
@ -75,3 +78,11 @@ module "land-cs-0" {
|
||||||
encryption_key = var.service_encryption_keys.storage
|
encryption_key = var.service_encryption_keys.storage
|
||||||
force_destroy = var.data_force_destroy
|
force_destroy = var.data_force_destroy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
module "land-bq-0" {
|
||||||
|
source = "../../../modules/bigquery-dataset"
|
||||||
|
project_id = module.land-project.project_id
|
||||||
|
id = "${replace(var.prefix, "-", "_")}_lnd_bq_0"
|
||||||
|
location = var.location
|
||||||
|
encryption_key = var.service_encryption_keys.bq
|
||||||
|
}
|
||||||
|
|
|
@ -22,11 +22,10 @@ locals {
|
||||||
CURATED_PRJ = module.cur-project.project_id
|
CURATED_PRJ = module.cur-project.project_id
|
||||||
DP_KMS_KEY = var.service_encryption_keys.compute
|
DP_KMS_KEY = var.service_encryption_keys.compute
|
||||||
DP_REGION = var.region
|
DP_REGION = var.region
|
||||||
GCP_REGION = var.region
|
|
||||||
LAND_PRJ = module.land-project.project_id
|
LAND_PRJ = module.land-project.project_id
|
||||||
LAND_GCS = module.land-cs-0.name
|
LAND_GCS = module.land-cs-0.url
|
||||||
PHS_CLUSTER_NAME = try(module.processing-dp-historyserver[0].name, null)
|
PHS_CLUSTER_NAME = try(module.processing-dp-historyserver[0].name, "")
|
||||||
PROCESSING_GCS = module.processing-cs-0.name
|
PROCESSING_GCS = module.processing-cs-0.url
|
||||||
PROCESSING_PRJ = module.processing-project.project_id
|
PROCESSING_PRJ = module.processing-project.project_id
|
||||||
PROCESSING_SA = module.processing-sa-0.email
|
PROCESSING_SA = module.processing-sa-0.email
|
||||||
PROCESSING_SUBNET = local.processing_subnet
|
PROCESSING_SUBNET = local.processing_subnet
|
||||||
|
@ -110,6 +109,15 @@ resource "google_composer_environment" "processing-cmp-0" {
|
||||||
kms_key_name = var.service_encryption_keys.composer
|
kms_key_name = var.service_encryption_keys.composer
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
web_server_network_access_control {
|
||||||
|
dynamic "allowed_ip_range" {
|
||||||
|
for_each = var.composer_config.web_server_access_control
|
||||||
|
content {
|
||||||
|
value = allowed_ip_range.key
|
||||||
|
description = allowed_ip_range.value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
depends_on = [
|
depends_on = [
|
||||||
module.processing-project
|
module.processing-project
|
||||||
|
|
|
@ -84,7 +84,7 @@ module "processing-dp-historyserver" {
|
||||||
staging_bucket = module.processing-staging-0.name
|
staging_bucket = module.processing-staging-0.name
|
||||||
temp_bucket = module.processing-temp-0.name
|
temp_bucket = module.processing-temp-0.name
|
||||||
gce_cluster_config = {
|
gce_cluster_config = {
|
||||||
subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link
|
subnetwork = local.processing_subnet
|
||||||
zone = "${var.region}-b"
|
zone = "${var.region}-b"
|
||||||
service_account = module.processing-sa-0.email
|
service_account = module.processing-sa-0.email
|
||||||
service_account_scopes = ["cloud-platform"]
|
service_account_scopes = ["cloud-platform"]
|
||||||
|
|
|
@ -16,7 +16,13 @@
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
iam_processing = {
|
iam_processing = {
|
||||||
|
"roles/bigquery.jobUser" = [
|
||||||
|
module.processing-sa-cmp-0.iam_email,
|
||||||
|
module.processing-sa-0.iam_email
|
||||||
|
]
|
||||||
"roles/composer.admin" = [local.groups_iam.data-engineers]
|
"roles/composer.admin" = [local.groups_iam.data-engineers]
|
||||||
|
"roles/dataflow.admin" = [module.processing-sa-cmp-0.iam_email]
|
||||||
|
"roles/dataflow.worker" = [module.processing-sa-0.iam_email]
|
||||||
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
|
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
|
||||||
"roles/composer.ServiceAgentV2Ext" = [
|
"roles/composer.ServiceAgentV2Ext" = [
|
||||||
"serviceAccount:${module.processing-project.service_accounts.robots.composer}"
|
"serviceAccount:${module.processing-project.service_accounts.robots.composer}"
|
||||||
|
@ -43,16 +49,14 @@ locals {
|
||||||
}
|
}
|
||||||
processing_subnet = (
|
processing_subnet = (
|
||||||
local.use_shared_vpc
|
local.use_shared_vpc
|
||||||
? var.network_config.subnet_self_links.processingestration
|
? var.network_config.subnet_self_link
|
||||||
: module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"]
|
: try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null)
|
||||||
)
|
)
|
||||||
processing_vpc = (
|
processing_vpc = (
|
||||||
local.use_shared_vpc
|
local.use_shared_vpc
|
||||||
? var.network_config.network_self_link
|
? var.network_config.network_self_link
|
||||||
: module.processing-vpc.0.self_link
|
: try(module.processing-vpc.0.self_link, null)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module "processing-project" {
|
module "processing-project" {
|
||||||
|
@ -78,6 +82,7 @@ module "processing-project" {
|
||||||
"composer.googleapis.com",
|
"composer.googleapis.com",
|
||||||
"compute.googleapis.com",
|
"compute.googleapis.com",
|
||||||
"container.googleapis.com",
|
"container.googleapis.com",
|
||||||
|
"dataflow.googleapis.com",
|
||||||
"dataproc.googleapis.com",
|
"dataproc.googleapis.com",
|
||||||
"iam.googleapis.com",
|
"iam.googleapis.com",
|
||||||
"servicenetworking.googleapis.com",
|
"servicenetworking.googleapis.com",
|
||||||
|
@ -96,13 +101,13 @@ module "processing-project" {
|
||||||
host_project = var.network_config.host_project
|
host_project = var.network_config.host_project
|
||||||
service_identity_iam = {
|
service_identity_iam = {
|
||||||
"roles/compute.networkUser" = [
|
"roles/compute.networkUser" = [
|
||||||
"cloudservices", "compute", "container-engine"
|
"cloudservices", "compute", "container-engine", "dataflow", "dataproc"
|
||||||
]
|
]
|
||||||
"roles/composer.sharedVpcAgent" = [
|
"roles/composer.sharedVpcAgent" = [
|
||||||
"composer"
|
"composer"
|
||||||
]
|
]
|
||||||
"roles/container.hostServiceAgentUser" = [
|
"roles/container.hostServiceAgentUser" = [
|
||||||
"container-egine"
|
"container-engine"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,33 +18,41 @@ locals {
|
||||||
cur_iam = {
|
cur_iam = {
|
||||||
"roles/bigquery.dataOwner" = [module.processing-sa-0.iam_email]
|
"roles/bigquery.dataOwner" = [module.processing-sa-0.iam_email]
|
||||||
"roles/bigquery.dataViewer" = [
|
"roles/bigquery.dataViewer" = [
|
||||||
|
module.cur-sa-0.iam_email,
|
||||||
local.groups_iam.data-analysts,
|
local.groups_iam.data-analysts,
|
||||||
local.groups_iam.data-engineers
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/bigquery.jobUser" = [
|
"roles/bigquery.jobUser" = [
|
||||||
module.processing-sa-0.iam_email,
|
module.processing-sa-0.iam_email, # Remove once bug is fixed. https://github.com/apache/airflow/issues/32106
|
||||||
|
module.cur-sa-0.iam_email,
|
||||||
local.groups_iam.data-analysts,
|
local.groups_iam.data-analysts,
|
||||||
local.groups_iam.data-engineers
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/datacatalog.tagTemplateViewer" = [
|
"roles/datacatalog.tagTemplateViewer" = [
|
||||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
module.cur-sa-0.iam_email,
|
||||||
|
local.groups_iam.data-analysts,
|
||||||
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/datacatalog.viewer" = [
|
"roles/datacatalog.viewer" = [
|
||||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
module.cur-sa-0.iam_email,
|
||||||
|
local.groups_iam.data-analysts,
|
||||||
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/storage.objectViewer" = [
|
"roles/storage.objectViewer" = [
|
||||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
module.cur-sa-0.iam_email,
|
||||||
|
local.groups_iam.data-analysts,
|
||||||
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
|
"roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
|
||||||
}
|
}
|
||||||
cur_services = [
|
cur_services = [
|
||||||
"iam.googleapis.com",
|
|
||||||
"bigquery.googleapis.com",
|
"bigquery.googleapis.com",
|
||||||
"bigqueryreservation.googleapis.com",
|
"bigqueryreservation.googleapis.com",
|
||||||
"bigquerystorage.googleapis.com",
|
"bigquerystorage.googleapis.com",
|
||||||
"cloudkms.googleapis.com",
|
"cloudkms.googleapis.com",
|
||||||
"cloudresourcemanager.googleapis.com",
|
"cloudresourcemanager.googleapis.com",
|
||||||
"compute.googleapis.com",
|
"compute.googleapis.com",
|
||||||
|
"iam.googleapis.com",
|
||||||
"servicenetworking.googleapis.com",
|
"servicenetworking.googleapis.com",
|
||||||
"serviceusage.googleapis.com",
|
"serviceusage.googleapis.com",
|
||||||
"stackdriver.googleapis.com",
|
"stackdriver.googleapis.com",
|
||||||
|
@ -75,6 +83,19 @@ module "cur-project" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
module "cur-sa-0" {
|
||||||
|
source = "../../../modules/iam-service-account"
|
||||||
|
project_id = module.cur-project.project_id
|
||||||
|
prefix = var.prefix
|
||||||
|
name = "cur-sa-0"
|
||||||
|
display_name = "Data platform curated zone service account."
|
||||||
|
iam = {
|
||||||
|
"roles/iam.serviceAccountTokenCreator" = [
|
||||||
|
local.groups_iam.data-engineers
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# Bigquery
|
# Bigquery
|
||||||
|
|
||||||
module "cur-bq-0" {
|
module "cur-bq-0" {
|
||||||
|
|
|
@ -9,7 +9,7 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
|
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
|
||||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/dlp.estimatesAdmin](https://cloud.google.com/iam/docs/understanding-roles#dlp.estimatesAdmin) <br>[roles/dlp.reader](https://cloud.google.com/iam/docs/understanding-roles#dlp.reader) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/dlp.estimatesAdmin](https://cloud.google.com/iam/docs/understanding-roles#dlp.estimatesAdmin) <br>[roles/dlp.reader](https://cloud.google.com/iam/docs/understanding-roles#dlp.reader) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
|
||||||
|<b>gcp-data-security</b><br><small><i>group</i></small>|[roles/datacatalog.admin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.admin) <br>[roles/dlp.admin](https://cloud.google.com/iam/docs/understanding-roles#dlp.admin) |
|
|<b>gcp-data-security</b><br><small><i>group</i></small>|[roles/datacatalog.admin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.admin) <br>[roles/dlp.admin](https://cloud.google.com/iam/docs/understanding-roles#dlp.admin) |
|
||||||
|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
|
|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
|
||||||
|
|
||||||
## Project <i>cur</i>
|
## Project <i>cur</i>
|
||||||
|
|
||||||
|
@ -18,15 +18,16 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||||
|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
|<b>cur-sa-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||||
|
|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||||
|
|
||||||
## Project <i>lnd</i>
|
## Project <i>lnd</i>
|
||||||
|
|
||||||
| members | roles |
|
| members | roles |
|
||||||
|---|---|
|
|---|---|
|
||||||
|<b>lnd-cs-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|
|<b>lnd-sa-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|
||||||
|
|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||||
|<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
|<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||||
|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
|
||||||
|
|
||||||
## Project <i>prc</i>
|
## Project <i>prc</i>
|
||||||
|
|
||||||
|
@ -35,5 +36,5 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
|
||||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||||
|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||||
|<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker) <br>[roles/dataproc.editor](https://cloud.google.com/iam/docs/understanding-roles#dataproc.editor) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker) <br>[roles/dataproc.worker](https://cloud.google.com/iam/docs/understanding-roles#dataproc.worker) |
|
||||||
|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/dataproc.worker](https://cloud.google.com/iam/docs/understanding-roles#dataproc.worker) |
|
|<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataproc.editor](https://cloud.google.com/iam/docs/understanding-roles#dataproc.editor) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||||
|
|
|
@ -10,7 +10,7 @@ The following diagram is a high-level reference of the resources created and man
|
||||||
|
|
||||||
![Data Platform architecture overview](./images/diagram.png "Data Platform architecture overview")
|
![Data Platform architecture overview](./images/diagram.png "Data Platform architecture overview")
|
||||||
|
|
||||||
A demo [Airflow pipeline](demo/orchestrate_pyspark.py) is also part of this blueprint: it can be built and run on top of the foundational infrastructure to verify or test the setup quickly.
|
A set of demo [Airflow pipelines](./demo/) are also part of this blueprint: they can be run on top of the foundational infrastructure to verify and test the setup.
|
||||||
|
|
||||||
## Design overview and choices
|
## Design overview and choices
|
||||||
|
|
||||||
|
@ -39,7 +39,7 @@ This separation into projects allows adhering to the least-privilege principle b
|
||||||
The script will create the following projects:
|
The script will create the following projects:
|
||||||
|
|
||||||
- **Landing** Data, stored in relevant formats. Structured data can be stored in BigQuery or in GCS using an appropriate file format such as AVRO or Parquet. Unstructured data stored on Cloud Storage.
|
- **Landing** Data, stored in relevant formats. Structured data can be stored in BigQuery or in GCS using an appropriate file format such as AVRO or Parquet. Unstructured data stored on Cloud Storage.
|
||||||
- **Processing** Used to host all resources needed to process and orchestrate data movement. Cloud Composer orchestrates all tasks that move data across layers. Cloud Dataproc Serveless process and move data between layers. Anonymization or tokenization of Personally Identifiable Information (PII) can be implemented here using Cloud DLP or a custom solution, depending on your requirements.
|
- **Processing** Used to host all resources needed to process and orchestrate data movement. Cloud Composer orchestrates all tasks that move data across layers. Cloud Dataproc Serverless process and move data between layers. Anonymization or tokenization of Personally Identifiable Information (PII) can be implemented here using Cloud DLP or a custom solution, depending on your requirements.
|
||||||
- **Curated** Cleansed, aggregated and curated data.
|
- **Curated** Cleansed, aggregated and curated data.
|
||||||
- **Common** Common services such as [Cloud DLP](https://cloud.google.com/dlp) or [Data Catalog](https://cloud.google.com/data-catalog/docs/concepts/overview).
|
- **Common** Common services such as [Cloud DLP](https://cloud.google.com/dlp) or [Data Catalog](https://cloud.google.com/data-catalog/docs/concepts/overview).
|
||||||
|
|
||||||
|
@ -69,7 +69,7 @@ We use three groups to control access to resources:
|
||||||
|
|
||||||
### Virtual Private Cloud (VPC) design
|
### Virtual Private Cloud (VPC) design
|
||||||
|
|
||||||
As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project.
|
As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc.
|
||||||
|
|
||||||
If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
|
If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
|
||||||
|
|
||||||
|
@ -203,7 +203,7 @@ module "data-platform" {
|
||||||
prefix = "myprefix"
|
prefix = "myprefix"
|
||||||
}
|
}
|
||||||
|
|
||||||
# tftest modules=21 resources=110
|
# tftest modules=23 resources=123
|
||||||
```
|
```
|
||||||
|
|
||||||
## Customizations
|
## Customizations
|
||||||
|
@ -229,10 +229,7 @@ To configure the use of a shared VPC, configure the `network_config`, example:
|
||||||
network_config = {
|
network_config = {
|
||||||
host_project = "PROJECT_ID"
|
host_project = "PROJECT_ID"
|
||||||
network_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/global/networks/NAME"
|
network_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/global/networks/NAME"
|
||||||
subnet_self_links = {
|
subnet_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
||||||
processing_transformation = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
|
||||||
processing_composer = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
|
||||||
}
|
|
||||||
composer_ip_ranges = {
|
composer_ip_ranges = {
|
||||||
cloudsql = "192.168.XXX.XXX/24"
|
cloudsql = "192.168.XXX.XXX/24"
|
||||||
gke_master = "192.168.XXX.XXX/28"
|
gke_master = "192.168.XXX.XXX/28"
|
||||||
|
@ -275,7 +272,6 @@ The application layer is out of scope of this script. As a demo purpuse only, on
|
||||||
| [outputs.tf](./outputs.tf) | Output variables. | | |
|
| [outputs.tf](./outputs.tf) | Output variables. | | |
|
||||||
| [variables.tf](./variables.tf) | Terraform Variables. | | |
|
| [variables.tf](./variables.tf) | Terraform Variables. | | |
|
||||||
<!-- BEGIN TFDOC -->
|
<!-- BEGIN TFDOC -->
|
||||||
|
|
||||||
## Variables
|
## Variables
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|
@ -283,13 +279,13 @@ The application layer is out of scope of this script. As a demo purpuse only, on
|
||||||
| [organization_domain](variables.tf#L122) | Organization domain. | <code>string</code> | ✓ | |
|
| [organization_domain](variables.tf#L122) | Organization domain. | <code>string</code> | ✓ | |
|
||||||
| [prefix](variables.tf#L127) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
| [prefix](variables.tf#L127) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||||
| [project_config](variables.tf#L136) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ landing = string processing = string curated = string common = string }), { landing = "lnd" processing = "prc" curated = "cur" common = "cmn" } ) })">object({…})</code> | ✓ | |
|
| [project_config](variables.tf#L136) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ landing = string processing = string curated = string common = string }), { landing = "lnd" processing = "prc" curated = "cur" common = "cmn" } ) })">object({…})</code> | ✓ | |
|
||||||
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(map(string), {}) pypi_packages = optional(map(string), {}) env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) count = optional(number, 1) } ), {}) web_server = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) }), {}) worker = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) min_count = optional(number, 1) max_count = optional(number, 3) } ), {}) }), {}) })">object({…})</code> | | <code>{}</code> |
|
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(map(string), {}) pypi_packages = optional(map(string), {}) env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) web_server_access_control = optional(map(string), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) count = optional(number, 1) } ), {}) web_server = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) }), {}) worker = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) min_count = optional(number, 1) max_count = optional(number, 3) } ), {}) }), {}) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [data_catalog_tags](variables.tf#L54) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code title="{ "3_Confidential" = null "2_Private" = null "1_Sensitive" = null }">{…}</code> |
|
| [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map(object({ description = optional(string) iam = optional(map(list(string)), {}) }))">map(object({…}))</code> | | <code title="{ "3_Confidential" = {} "2_Private" = {} "1_Sensitive" = {} }">{…}</code> |
|
||||||
| [data_force_destroy](variables.tf#L65) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
| [data_force_destroy](variables.tf#L69) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||||
| [enable_services](variables.tf#L71) | Flag to enable or disable services in the Data Platform. | <code title="object({ composer = optional(bool, true) dataproc_history_server = optional(bool, true) })">object({…})</code> | | <code>{}</code> |
|
| [enable_services](variables.tf#L75) | Flag to enable or disable services in the Data Platform. | <code title="object({ composer = optional(bool, true) dataproc_history_server = optional(bool, true) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [groups](variables.tf#L80) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
| [groups](variables.tf#L84) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||||
| [location](variables.tf#L90) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
| [location](variables.tf#L94) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
||||||
| [network_config](variables.tf#L96) | Shared VPC network configurations to use. If null networks will be created in projects. | <code title="object({ host_project = optional(string) network_self_link = optional(string) subnet_self_links = optional(object({ processing_transformation = string processing_composer = string }), null) composer_ip_ranges = optional(object({ connection_subnetwork = optional(string) cloud_sql = optional(string, "10.20.10.0/24") gke_master = optional(string, "10.20.11.0/28") pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) })">object({…})</code> | | <code>{}</code> |
|
| [network_config](variables.tf#L100) | Shared VPC network configurations to use. If null networks will be created in projects. | <code title="object({ host_project = optional(string) network_self_link = optional(string) subnet_self_link = optional(string) composer_ip_ranges = optional(object({ connection_subnetwork = optional(string) cloud_sql = optional(string, "10.20.10.0/24") gke_master = optional(string, "10.20.11.0/28") pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [project_suffix](variables.tf#L160) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
| [project_suffix](variables.tf#L160) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||||
| [region](variables.tf#L166) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [region](variables.tf#L166) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
| [service_encryption_keys](variables.tf#L172) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = optional(string) composer = optional(string) compute = optional(string) storage = optional(string) })">object({…})</code> | | <code>{}</code> |
|
| [service_encryption_keys](variables.tf#L172) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = optional(string) composer = optional(string) compute = optional(string) storage = optional(string) })">object({…})</code> | | <code>{}</code> |
|
||||||
|
@ -299,11 +295,11 @@ The application layer is out of scope of this script. As a demo purpuse only, on
|
||||||
| name | description | sensitive |
|
| name | description | sensitive |
|
||||||
|---|---|:---:|
|
|---|---|:---:|
|
||||||
| [bigquery-datasets](outputs.tf#L17) | BigQuery datasets. | |
|
| [bigquery-datasets](outputs.tf#L17) | BigQuery datasets. | |
|
||||||
| [dataproc-history-server](outputs.tf#L24) | List of bucket names which have been assigned to the cluster. | |
|
| [composer](outputs.tf#L25) | Composer variables. | |
|
||||||
| [gcs-buckets](outputs.tf#L29) | GCS buckets. | ✓ |
|
| [dataproc-history-server](outputs.tf#L33) | List of bucket names which have been assigned to the cluster. | |
|
||||||
| [kms_keys](outputs.tf#L39) | Cloud MKS keys. | |
|
| [gcs_buckets](outputs.tf#L38) | GCS buckets. | |
|
||||||
| [projects](outputs.tf#L44) | GCP Projects informations. | |
|
| [kms_keys](outputs.tf#L47) | Cloud MKS keys. | |
|
||||||
| [vpc_network](outputs.tf#L62) | VPC network. | |
|
| [network](outputs.tf#L52) | VPC network. | |
|
||||||
| [vpc_subnet](outputs.tf#L70) | VPC subnetworks. | |
|
| [projects](outputs.tf#L60) | GCP Projects information. | |
|
||||||
|
| [service_accounts](outputs.tf#L78) | Service account created. | |
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -0,0 +1,58 @@
|
||||||
|
# Data ingestion Demo
|
||||||
|
|
||||||
|
In this folder, you can find Airflow DAG examples to process data on the `minimal data platform` instantiated [here](../). Examples are focused on importing data from `landing` to `curated` resources.
|
||||||
|
|
||||||
|
Examples are not intended to be a production-ready code, but a bollerplate to verify and test the setup.
|
||||||
|
|
||||||
|
## Demo use case
|
||||||
|
|
||||||
|
The demo imports CSV customer data from the `landing` GCS bucket to the `curated` BigQuery dataset.
|
||||||
|
|
||||||
|
## Input files
|
||||||
|
|
||||||
|
Data are uploaded to the `landing` GCS bucket. File structure:
|
||||||
|
|
||||||
|
- [`customers.csv`](./data/customers.csv): Comma separate value with customer information in the following format: Customer ID, Name, Surname, Registration Timestamp
|
||||||
|
|
||||||
|
## Configuration files
|
||||||
|
|
||||||
|
Data relies on the following configuration files:
|
||||||
|
|
||||||
|
- [`customers_schema.json`](./data/customers_schema.json): customer BigQuery table schema definition.
|
||||||
|
- [`customers_udf.js`](./data/customers_udf.js): dataflow user defined function to transform CSV files into BigQuery schema
|
||||||
|
- [`customers.json`](./data/customers.json): customer CSV file schema definition
|
||||||
|
|
||||||
|
## Data processing pipelines
|
||||||
|
|
||||||
|
Different data pipelines are provided to highlight different ways import data.
|
||||||
|
|
||||||
|
Below you can find a description of each example:
|
||||||
|
|
||||||
|
- `bq_import.py`: Importing data using BigQuery import capability.
|
||||||
|
- `dataflow_import.py`: Importing data using Cloud Dataflow.
|
||||||
|
- `dataproc_import.py`: Importing data using Cloud Dataproc.
|
||||||
|
|
||||||
|
## Running the demo
|
||||||
|
|
||||||
|
To run demo examples, please follow the following steps:
|
||||||
|
|
||||||
|
1. Copy sample data to the `landing` Cloud Storage bucket impersonating the `landing` service account.
|
||||||
|
1. Copy sample data structure definition in the `processing` Cloud Storage bucket impersonating the `orchestration` service account.
|
||||||
|
1. Copy the Cloud Composer DAG to the Cloud Composer Storage bucket impersonating the `orchestration` service account.
|
||||||
|
1. Open the Cloud Composer Airflow UI and run the imported DAG.
|
||||||
|
1. Run the BigQuery query to see results.
|
||||||
|
|
||||||
|
Below you can find computed commands to perform steps.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
terraform output -json | jq -r '@sh "export LND_SA=\(.service_accounts.value.landing)\nexport PRC_SA=\(.service_accounts.value.processing)\nexport CMP_SA=\(.service_accounts.value.composer)"' > env.sh
|
||||||
|
|
||||||
|
terraform output -json | jq -r '@sh "export LND_GCS=\(.gcs_buckets.value.landing)\nexport PRC_GCS=\(.gcs_buckets.value.processing)\nexport CUR_GCS=\(.gcs_buckets.value.curated)\nexport CMP_GCS=\(.composer.value.dag_bucket)"' >> env.sh
|
||||||
|
|
||||||
|
source ./env.sh
|
||||||
|
|
||||||
|
gsutil -i $LND_SA cp demo/data/*.csv gs://$LND_GCS
|
||||||
|
gsutil -i $CMP_SA cp demo/data/*.j* gs://$PRC_GCS
|
||||||
|
gsutil -i $CMP_SA cp demo/pyspark_* gs://$PRC_GCS
|
||||||
|
gsutil -i $CMP_SA cp demo/dag_*.py $CMP_GCS
|
||||||
|
```
|
|
@ -0,0 +1,104 @@
|
||||||
|
# Copyright 2022 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Load The Dependencies
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import datetime
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from airflow import models
|
||||||
|
from airflow.operators import dummy
|
||||||
|
from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Set variables - Needed for the DEMO
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
BQ_LOCATION = os.environ.get("BQ_LOCATION")
|
||||||
|
CURATED_PRJ = os.environ.get("CURATED_PRJ")
|
||||||
|
CURATED_BQ_DATASET = os.environ.get("CURATED_BQ_DATASET")
|
||||||
|
CURATED_GCS = os.environ.get("CURATED_GCS")
|
||||||
|
LAND_PRJ = os.environ.get("LAND_PRJ")
|
||||||
|
LAND_GCS = os.environ.get("LAND_GCS")
|
||||||
|
PROCESSING_GCS = os.environ.get("PROCESSING_GCS")
|
||||||
|
PROCESSING_SA = os.environ.get("PROCESSING_SA")
|
||||||
|
PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ")
|
||||||
|
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
||||||
|
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
||||||
|
DP_KMS_KEY = os.environ.get("DP_KMS_KEY", "")
|
||||||
|
DP_REGION = os.environ.get("DP_REGION")
|
||||||
|
DP_ZONE = os.environ.get("DP_REGION") + "-b"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Set default arguments
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# If you are running Airflow in more than one time zone
|
||||||
|
# see https://airflow.apache.org/docs/apache-airflow/stable/timezone.html
|
||||||
|
# for best practices
|
||||||
|
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
|
||||||
|
|
||||||
|
default_args = {
|
||||||
|
'owner': 'airflow',
|
||||||
|
'start_date': yesterday,
|
||||||
|
'depends_on_past': False,
|
||||||
|
'email': [''],
|
||||||
|
'email_on_failure': False,
|
||||||
|
'email_on_retry': False,
|
||||||
|
'retries': 1,
|
||||||
|
'retry_delay': datetime.timedelta(minutes=5),
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Main DAG
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
with models.DAG(
|
||||||
|
'bq_gcs2bq',
|
||||||
|
default_args=default_args,
|
||||||
|
schedule_interval=None) as dag:
|
||||||
|
start = dummy.DummyOperator(
|
||||||
|
task_id='start',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
end = dummy.DummyOperator(
|
||||||
|
task_id='end',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Bigquery Tables automatically created for demo porpuse.
|
||||||
|
# Consider a dedicated pipeline or tool for a real life scenario.
|
||||||
|
|
||||||
|
customers_import = GCSToBigQueryOperator(
|
||||||
|
task_id='csv_to_bigquery',
|
||||||
|
bucket=LAND_GCS[5:],
|
||||||
|
source_objects=['customers.csv'],
|
||||||
|
destination_project_dataset_table='{}:{}.{}'.format(CURATED_PRJ, CURATED_BQ_DATASET, 'customers'),
|
||||||
|
create_disposition='CREATE_IF_NEEDED',
|
||||||
|
write_disposition='WRITE_APPEND',
|
||||||
|
schema_update_options=['ALLOW_FIELD_RELAXATION', 'ALLOW_FIELD_ADDITION'],
|
||||||
|
schema_object="customers.json",
|
||||||
|
schema_object_bucket=PROCESSING_GCS[5:],
|
||||||
|
project_id=PROCESSING_PRJ, # The process will continue to run on the dataset project until the Apache Airflow bug is fixed. https://github.com/apache/airflow/issues/32106
|
||||||
|
impersonation_chain=[PROCESSING_SA]
|
||||||
|
)
|
||||||
|
|
||||||
|
start >> customers_import >> end
|
||||||
|
|
|
@ -0,0 +1,116 @@
|
||||||
|
# Copyright 2022 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Load The Dependencies
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import datetime
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from airflow import models
|
||||||
|
from airflow.providers.google.cloud.operators.dataflow import DataflowTemplatedJobStartOperator
|
||||||
|
from airflow.operators import dummy
|
||||||
|
from airflow.providers.google.cloud.operators.bigquery import BigQueryInsertJobOperator, BigQueryUpsertTableOperator, BigQueryUpdateTableSchemaOperator
|
||||||
|
from airflow.utils.task_group import TaskGroup
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Set variables - Needed for the DEMO
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
BQ_LOCATION = os.environ.get("BQ_LOCATION")
|
||||||
|
CURATED_PRJ = os.environ.get("CURATED_PRJ")
|
||||||
|
CURATED_BQ_DATASET = os.environ.get("CURATED_BQ_DATASET")
|
||||||
|
CURATED_GCS = os.environ.get("CURATED_GCS")
|
||||||
|
LAND_PRJ = os.environ.get("LAND_PRJ")
|
||||||
|
LAND_GCS = os.environ.get("LAND_GCS")
|
||||||
|
PROCESSING_GCS = os.environ.get("PROCESSING_GCS")
|
||||||
|
PROCESSING_SA = os.environ.get("PROCESSING_SA")
|
||||||
|
PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ")
|
||||||
|
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
||||||
|
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
||||||
|
DP_KMS_KEY = os.environ.get("DP_KMS_KEY", "")
|
||||||
|
DP_REGION = os.environ.get("DP_REGION")
|
||||||
|
DP_ZONE = os.environ.get("DP_REGION") + "-b"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Set default arguments
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# If you are running Airflow in more than one time zone
|
||||||
|
# see https://airflow.apache.org/docs/apache-airflow/stable/timezone.html
|
||||||
|
# for best practices
|
||||||
|
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
|
||||||
|
|
||||||
|
default_args = {
|
||||||
|
'owner': 'airflow',
|
||||||
|
'start_date': yesterday,
|
||||||
|
'depends_on_past': False,
|
||||||
|
'email': [''],
|
||||||
|
'email_on_failure': False,
|
||||||
|
'email_on_retry': False,
|
||||||
|
'retries': 1,
|
||||||
|
'retry_delay': datetime.timedelta(minutes=5),
|
||||||
|
'dataflow_default_options': {
|
||||||
|
'location': DP_REGION,
|
||||||
|
'zone': DP_ZONE,
|
||||||
|
'stagingLocation': PROCESSING_GCS + "/staging",
|
||||||
|
'tempLocation': PROCESSING_GCS + "/tmp",
|
||||||
|
'serviceAccountEmail': PROCESSING_SA,
|
||||||
|
'subnetwork': PROCESSING_SUBNET,
|
||||||
|
'ipConfiguration': "WORKER_IP_PRIVATE",
|
||||||
|
'kmsKeyName' : DP_KMS_KEY
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Main DAG
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
with models.DAG(
|
||||||
|
'dataflow_gcs2bq',
|
||||||
|
default_args=default_args,
|
||||||
|
schedule_interval=None) as dag:
|
||||||
|
start = dummy.DummyOperator(
|
||||||
|
task_id='start',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
end = dummy.DummyOperator(
|
||||||
|
task_id='end',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Bigquery Tables automatically created for demo porpuse.
|
||||||
|
# Consider a dedicated pipeline or tool for a real life scenario.
|
||||||
|
customers_import = DataflowTemplatedJobStartOperator(
|
||||||
|
task_id="dataflow_customers_import",
|
||||||
|
template="gs://dataflow-templates/latest/GCS_Text_to_BigQuery",
|
||||||
|
project_id=PROCESSING_PRJ,
|
||||||
|
location=DP_REGION,
|
||||||
|
parameters={
|
||||||
|
"javascriptTextTransformFunctionName": "transform",
|
||||||
|
"JSONPath": PROCESSING_GCS + "/customers_schema.json",
|
||||||
|
"javascriptTextTransformGcsPath": PROCESSING_GCS + "/customers_udf.js",
|
||||||
|
"inputFilePattern": LAND_GCS + "/customers.csv",
|
||||||
|
"outputTable": CURATED_PRJ + ":" + CURATED_BQ_DATASET + ".customers",
|
||||||
|
"bigQueryLoadingTemporaryDirectory": PROCESSING_GCS + "/tmp/bq/",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
start >> customers_import >> end
|
||||||
|
|
|
@ -0,0 +1,102 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
# Copyright 2019 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import time
|
||||||
|
import os
|
||||||
|
|
||||||
|
from airflow import models
|
||||||
|
from airflow.operators import dummy
|
||||||
|
from airflow.providers.google.cloud.operators.dataproc import (
|
||||||
|
DataprocCreateBatchOperator, DataprocDeleteBatchOperator, DataprocGetBatchOperator, DataprocListBatchesOperator
|
||||||
|
|
||||||
|
)
|
||||||
|
from airflow.utils.dates import days_ago
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Get variables
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
BQ_LOCATION = os.environ.get("BQ_LOCATION")
|
||||||
|
CURATED_BQ_DATASET = os.environ.get("CURATED_BQ_DATASET")
|
||||||
|
CURATED_GCS = os.environ.get("CURATED_GCS")
|
||||||
|
CURATED_PRJ = os.environ.get("CURATED_PRJ")
|
||||||
|
DP_KMS_KEY = os.environ.get("DP_KMS_KEY", "")
|
||||||
|
DP_REGION = os.environ.get("DP_REGION")
|
||||||
|
GCP_REGION = os.environ.get("GCP_REGION")
|
||||||
|
LAND_PRJ = os.environ.get("LAND_PRJ")
|
||||||
|
LAND_BQ_DATASET = os.environ.get("LAND_BQ_DATASET")
|
||||||
|
LAND_GCS = os.environ.get("LAND_GCS")
|
||||||
|
PHS_CLUSTER_NAME = os.environ.get("PHS_CLUSTER_NAME")
|
||||||
|
PROCESSING_GCS = os.environ.get("PROCESSING_GCS")
|
||||||
|
PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ")
|
||||||
|
PROCESSING_SA = os.environ.get("PROCESSING_SA")
|
||||||
|
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
||||||
|
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
||||||
|
|
||||||
|
PYTHON_FILE_LOCATION = PROCESSING_GCS+"/pyspark_gcs2bq.py"
|
||||||
|
PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_CLUSTER_NAME
|
||||||
|
SPARK_BIGQUERY_JAR_FILE = "gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.13-0.29.0.jar"
|
||||||
|
BATCH_ID = "batch-create-phs-"+str(int(time.time()))
|
||||||
|
|
||||||
|
default_args = {
|
||||||
|
# Tell airflow to start one day ago, so that it runs as soon as you upload it
|
||||||
|
"start_date": days_ago(1),
|
||||||
|
"region": DP_REGION,
|
||||||
|
}
|
||||||
|
with models.DAG(
|
||||||
|
"dataproc_batch_gcs2bq", # The id you will see in the DAG airflow page
|
||||||
|
default_args=default_args, # The interval with which to schedule the DAG
|
||||||
|
schedule_interval=None, # Override to match your needs
|
||||||
|
) as dag:
|
||||||
|
start = dummy.DummyOperator(
|
||||||
|
task_id='start',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
end = dummy.DummyOperator(
|
||||||
|
task_id='end',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
create_batch = DataprocCreateBatchOperator(
|
||||||
|
task_id="batch_create",
|
||||||
|
project_id=PROCESSING_PRJ,
|
||||||
|
batch_id=BATCH_ID,
|
||||||
|
batch={
|
||||||
|
"environment_config": {
|
||||||
|
"execution_config": {
|
||||||
|
"service_account": PROCESSING_SA,
|
||||||
|
"subnetwork_uri": PROCESSING_SUBNET
|
||||||
|
},
|
||||||
|
"peripherals_config": {
|
||||||
|
"spark_history_server_config":{
|
||||||
|
"dataproc_cluster": PHS_CLUSTER_PATH
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pyspark_batch": {
|
||||||
|
"args": [
|
||||||
|
LAND_GCS + "/customers.csv",
|
||||||
|
CURATED_PRJ + ":" + CURATED_BQ_DATASET + ".customers",
|
||||||
|
PROCESSING_GCS[5:]
|
||||||
|
],
|
||||||
|
"main_python_file_uri": PYTHON_FILE_LOCATION,
|
||||||
|
"jar_file_uris": [SPARK_BIGQUERY_JAR_FILE]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
start >> create_batch >> end
|
|
@ -0,0 +1,97 @@
|
||||||
|
# Copyright 2022 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Load The Dependencies
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import datetime
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from airflow import models
|
||||||
|
from airflow.providers.google.cloud.operators.dataflow import DataflowTemplatedJobStartOperator
|
||||||
|
from airflow.operators import dummy
|
||||||
|
from airflow.providers.google.cloud.operators.bigquery import BigQueryDeleteTableOperator
|
||||||
|
from airflow.utils.task_group import TaskGroup
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Set variables - Needed for the DEMO
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
BQ_LOCATION = os.environ.get("BQ_LOCATION")
|
||||||
|
CURATED_PRJ = os.environ.get("CURATED_PRJ")
|
||||||
|
CURATED_BQ_DATASET = os.environ.get("CURATED_BQ_DATASET")
|
||||||
|
CURATED_GCS = os.environ.get("CURATED_GCS")
|
||||||
|
LAND_PRJ = os.environ.get("LAND_PRJ")
|
||||||
|
LAND_GCS = os.environ.get("LAND_GCS")
|
||||||
|
PROCESSING_GCS = os.environ.get("PROCESSING_GCS")
|
||||||
|
PROCESSING_SA = os.environ.get("PROCESSING_SA")
|
||||||
|
PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ")
|
||||||
|
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
||||||
|
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
||||||
|
DP_KMS_KEY = os.environ.get("DP_KMS_KEY", "")
|
||||||
|
DP_REGION = os.environ.get("DP_REGION")
|
||||||
|
DP_ZONE = os.environ.get("DP_REGION") + "-b"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Set default arguments
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# If you are running Airflow in more than one time zone
|
||||||
|
# see https://airflow.apache.org/docs/apache-airflow/stable/timezone.html
|
||||||
|
# for best practices
|
||||||
|
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
|
||||||
|
|
||||||
|
default_args = {
|
||||||
|
'owner': 'airflow',
|
||||||
|
'start_date': yesterday,
|
||||||
|
'depends_on_past': False,
|
||||||
|
'email': [''],
|
||||||
|
'email_on_failure': False,
|
||||||
|
'email_on_retry': False,
|
||||||
|
'retries': 1,
|
||||||
|
'retry_delay': datetime.timedelta(minutes=5),
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
# Main DAG
|
||||||
|
# --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
with models.DAG(
|
||||||
|
'delete_tables_dag',
|
||||||
|
default_args=default_args,
|
||||||
|
schedule_interval=None) as dag:
|
||||||
|
start = dummy.DummyOperator(
|
||||||
|
task_id='start',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
end = dummy.DummyOperator(
|
||||||
|
task_id='end',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Bigquery Tables deleted here for demo porpuse.
|
||||||
|
# Consider a dedicated pipeline or tool for a real life scenario.
|
||||||
|
with TaskGroup('delete_table') as delte_table:
|
||||||
|
delete_table_customers = BigQueryDeleteTableOperator(
|
||||||
|
task_id="delete_table_customers",
|
||||||
|
deletion_dataset_table=CURATED_PRJ+"."+CURATED_BQ_DATASET+".customers",
|
||||||
|
impersonation_chain=[PROCESSING_SA]
|
||||||
|
)
|
||||||
|
|
||||||
|
start >> delte_table >> end
|
|
@ -19,6 +19,7 @@ import time
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from airflow import models
|
from airflow import models
|
||||||
|
from airflow.operators import dummy
|
||||||
from airflow.providers.google.cloud.operators.dataproc import (
|
from airflow.providers.google.cloud.operators.dataproc import (
|
||||||
DataprocCreateBatchOperator, DataprocDeleteBatchOperator, DataprocGetBatchOperator, DataprocListBatchesOperator
|
DataprocCreateBatchOperator, DataprocDeleteBatchOperator, DataprocGetBatchOperator, DataprocListBatchesOperator
|
||||||
|
|
||||||
|
@ -45,8 +46,9 @@ PROCESSING_SA = os.environ.get("PROCESSING_SA")
|
||||||
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
||||||
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
||||||
|
|
||||||
PYTHON_FILE_LOCATION = "gs://"+PROCESSING_GCS+"/pyspark_sort.py"
|
PYTHON_FILE_LOCATION = PROCESSING_GCS+"/pyspark_sort.py"
|
||||||
PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_CLUSTER_NAME
|
PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_CLUSTER_NAME
|
||||||
|
BATCH_ID = "batch-create-phs-"+str(int(time.time()))
|
||||||
|
|
||||||
default_args = {
|
default_args = {
|
||||||
# Tell airflow to start one day ago, so that it runs as soon as you upload it
|
# Tell airflow to start one day ago, so that it runs as soon as you upload it
|
||||||
|
@ -58,6 +60,15 @@ with models.DAG(
|
||||||
default_args=default_args, # The interval with which to schedule the DAG
|
default_args=default_args, # The interval with which to schedule the DAG
|
||||||
schedule_interval=None, # Override to match your needs
|
schedule_interval=None, # Override to match your needs
|
||||||
) as dag:
|
) as dag:
|
||||||
|
start = dummy.DummyOperator(
|
||||||
|
task_id='start',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
|
end = dummy.DummyOperator(
|
||||||
|
task_id='end',
|
||||||
|
trigger_rule='all_success'
|
||||||
|
)
|
||||||
|
|
||||||
create_batch = DataprocCreateBatchOperator(
|
create_batch = DataprocCreateBatchOperator(
|
||||||
task_id="batch_create",
|
task_id="batch_create",
|
||||||
|
@ -75,19 +86,11 @@ with models.DAG(
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"pyspark_batch": {
|
"pyspark_batch": {
|
||||||
|
"args": ["pippo"],
|
||||||
"main_python_file_uri": PYTHON_FILE_LOCATION,
|
"main_python_file_uri": PYTHON_FILE_LOCATION,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
batch_id="batch-create-phs-"+str(int(time.time())),
|
batch_id=BATCH_ID,
|
||||||
)
|
)
|
||||||
|
|
||||||
list_batches = DataprocListBatchesOperator(
|
start >> create_batch >> end
|
||||||
task_id="list-all-batches",
|
|
||||||
)
|
|
||||||
|
|
||||||
get_batch = DataprocGetBatchOperator(
|
|
||||||
task_id="get_batch",
|
|
||||||
batch_id="batch-create-phs",
|
|
||||||
)
|
|
||||||
|
|
||||||
create_batch >> list_batches >> get_batch
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
1,Name1,Surname1,1636972001
|
||||||
|
2,Name2,Surname2,1636972002
|
||||||
|
3,Name3,Surname3,1636972003
|
||||||
|
4,Name4,Surname4,1636972004
|
||||||
|
5,Name5,Surname5,1636972005
|
||||||
|
6,Name6,Surname6,1636972006
|
||||||
|
7,Name7,Surname7,1636972007
|
||||||
|
8,Name8,Surname8,1636972008
|
||||||
|
9,Name9,Surname9,1636972009
|
||||||
|
10,Name11,Surname11,1636972010
|
||||||
|
11,Name12,Surname12,1636972011
|
||||||
|
12,Name13,Surname13,1636972012
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "id",
|
||||||
|
"type": "INTEGER",
|
||||||
|
"description": "ID"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "name",
|
||||||
|
"type": "STRING",
|
||||||
|
"description": "Name"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "surname",
|
||||||
|
"type": "STRING",
|
||||||
|
"description": "Surname"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "timestamp",
|
||||||
|
"type": "TIMESTAMP",
|
||||||
|
"description": "Timestamp"
|
||||||
|
}
|
||||||
|
]
|
|
@ -0,0 +1,28 @@
|
||||||
|
{
|
||||||
|
"BigQuery Schema": [
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "id",
|
||||||
|
"type": "INTEGER",
|
||||||
|
"description": "ID"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "name",
|
||||||
|
"type": "STRING",
|
||||||
|
"description": "Name"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "surname",
|
||||||
|
"type": "STRING",
|
||||||
|
"description": "Surname"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "REQUIRED",
|
||||||
|
"name": "timestamp",
|
||||||
|
"type": "TIMESTAMP",
|
||||||
|
"description": "Timestamp"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
|
@ -0,0 +1,12 @@
|
||||||
|
function transform(line) {
|
||||||
|
var values = line.split(',');
|
||||||
|
|
||||||
|
var obj = new Object();
|
||||||
|
obj.id = values[0]
|
||||||
|
obj.name = values[1];
|
||||||
|
obj.surname = values[2];
|
||||||
|
obj.timestamp = values[3];
|
||||||
|
var jsonString = JSON.stringify(obj);
|
||||||
|
|
||||||
|
return jsonString;
|
||||||
|
}
|
|
@ -0,0 +1,61 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
# Copyright 2019 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
""" Sample pyspark script to read data CSV data from Cloud Storage and
|
||||||
|
import into BigQuery. The script runs on Cloud Dataproc Serverless.
|
||||||
|
|
||||||
|
Note this file is not intended to be run directly, but run inside a PySpark
|
||||||
|
environment.
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from pyspark.sql import SparkSession
|
||||||
|
from pyspark.sql import functions as F
|
||||||
|
from pyspark.sql.types import StructType,TimestampType, StringType, IntegerType
|
||||||
|
|
||||||
|
# Create a Spark session
|
||||||
|
spark = SparkSession.builder \
|
||||||
|
.appName("Read CSV from GCS and Write to BigQuery") \
|
||||||
|
.getOrCreate()
|
||||||
|
|
||||||
|
# Read parameters
|
||||||
|
csv = spark.sparkContext.parallelize([sys.argv[1]]).first()
|
||||||
|
dataset_table = spark.sparkContext.parallelize([sys.argv[2]]).first()
|
||||||
|
tmp_gcs = spark.sparkContext.parallelize([sys.argv[3]]).first()
|
||||||
|
|
||||||
|
spark.conf.set('temporaryGcsBucket', tmp_gcs)
|
||||||
|
|
||||||
|
schema = StructType() \
|
||||||
|
.add("id",IntegerType(),True) \
|
||||||
|
.add("name",StringType(),True) \
|
||||||
|
.add("surname",StringType(),True) \
|
||||||
|
.add("timestamp",TimestampType(),True)
|
||||||
|
|
||||||
|
data = spark.read.format("csv") \
|
||||||
|
.schema(schema) \
|
||||||
|
.load(csv)
|
||||||
|
|
||||||
|
# add lineage metadata: input filename and loading ts
|
||||||
|
data = data.select('*',
|
||||||
|
(F.input_file_name()).alias('input_filename'),
|
||||||
|
(F.current_timestamp()).alias('load_ts')
|
||||||
|
)
|
||||||
|
|
||||||
|
# Saving the data to BigQuery
|
||||||
|
data.write.format('bigquery') \
|
||||||
|
.option('table', dataset_table) \
|
||||||
|
.mode('append') \
|
||||||
|
.save()
|
Binary file not shown.
Before Width: | Height: | Size: 102 KiB After Width: | Height: | Size: 174 KiB |
|
@ -17,7 +17,16 @@
|
||||||
output "bigquery-datasets" {
|
output "bigquery-datasets" {
|
||||||
description = "BigQuery datasets."
|
description = "BigQuery datasets."
|
||||||
value = {
|
value = {
|
||||||
curated = module.cur-bq-0.dataset_id,
|
curated = module.cur-bq-0.dataset_id
|
||||||
|
landing = module.land-bq-0.dataset_id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output "composer" {
|
||||||
|
description = "Composer variables."
|
||||||
|
value = {
|
||||||
|
air_flow_uri = try(google_composer_environment.processing-cmp-0[0].config.0.airflow_uri, null)
|
||||||
|
dag_bucket = try(regex("^gs://([^/]*)/dags$", google_composer_environment.processing-cmp-0[0].config[0].dag_gcs_prefix)[0], null)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -26,13 +35,12 @@ output "dataproc-history-server" {
|
||||||
value = one(module.processing-dp-historyserver)
|
value = one(module.processing-dp-historyserver)
|
||||||
}
|
}
|
||||||
|
|
||||||
output "gcs-buckets" {
|
output "gcs_buckets" {
|
||||||
description = "GCS buckets."
|
description = "GCS buckets."
|
||||||
sensitive = true
|
|
||||||
value = {
|
value = {
|
||||||
landing-cs-0 = module.land-sa-cs-0,
|
curated = module.cur-cs-0.name
|
||||||
processing-cs-0 = module.processing-cs-0,
|
landing = module.land-cs-0.name
|
||||||
cur-cs-0 = module.cur-cs-0,
|
processing = module.processing-cs-0.name
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,36 +49,38 @@ output "kms_keys" {
|
||||||
value = var.service_encryption_keys
|
value = var.service_encryption_keys
|
||||||
}
|
}
|
||||||
|
|
||||||
output "projects" {
|
output "network" {
|
||||||
description = "GCP Projects informations."
|
|
||||||
value = {
|
|
||||||
project_number = {
|
|
||||||
landing = module.land-project.number,
|
|
||||||
common = module.common-project.number,
|
|
||||||
curated = module.cur-project.number,
|
|
||||||
processing = module.processing-project.number,
|
|
||||||
}
|
|
||||||
project_id = {
|
|
||||||
landing = module.land-project.project_id,
|
|
||||||
common = module.common-project.project_id,
|
|
||||||
curated = module.cur-project.project_id,
|
|
||||||
processing = module.processing-project.project_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
output "vpc_network" {
|
|
||||||
description = "VPC network."
|
description = "VPC network."
|
||||||
value = {
|
value = {
|
||||||
processing_transformation = local.processing_vpc
|
processing_subnet = local.processing_subnet
|
||||||
processing_composer = local.processing_vpc
|
processing_vpc = local.processing_vpc
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output "vpc_subnet" {
|
output "projects" {
|
||||||
description = "VPC subnetworks."
|
description = "GCP Projects information."
|
||||||
value = {
|
value = {
|
||||||
processing_transformation = local.processing_subnet
|
project_number = {
|
||||||
processing_composer = local.processing_subnet
|
common = module.common-project.number
|
||||||
|
curated = module.cur-project.number
|
||||||
|
landing = module.land-project.number
|
||||||
|
processing = module.processing-project.number
|
||||||
|
}
|
||||||
|
project_id = {
|
||||||
|
common = module.common-project.project_id
|
||||||
|
curated = module.cur-project.project_id
|
||||||
|
landing = module.land-project.project_id
|
||||||
|
processing = module.processing-project.project_id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output "service_accounts" {
|
||||||
|
description = "Service account created."
|
||||||
|
value = {
|
||||||
|
composer = module.processing-sa-cmp-0.email
|
||||||
|
curated = module.cur-sa-0.email,
|
||||||
|
landing = module.land-sa-0.email,
|
||||||
|
processing = module.processing-sa-0.email,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,6 +24,7 @@ variable "composer_config" {
|
||||||
env_variables = optional(map(string), {})
|
env_variables = optional(map(string), {})
|
||||||
image_version = optional(string, "composer-2-airflow-2")
|
image_version = optional(string, "composer-2-airflow-2")
|
||||||
}), {})
|
}), {})
|
||||||
|
web_server_access_control = optional(map(string), {})
|
||||||
workloads_config = optional(object({
|
workloads_config = optional(object({
|
||||||
scheduler = optional(object({
|
scheduler = optional(object({
|
||||||
cpu = optional(number, 0.5)
|
cpu = optional(number, 0.5)
|
||||||
|
@ -53,12 +54,15 @@ variable "composer_config" {
|
||||||
|
|
||||||
variable "data_catalog_tags" {
|
variable "data_catalog_tags" {
|
||||||
description = "List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format."
|
description = "List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format."
|
||||||
type = map(map(list(string)))
|
type = map(object({
|
||||||
|
description = optional(string)
|
||||||
|
iam = optional(map(list(string)), {})
|
||||||
|
}))
|
||||||
nullable = false
|
nullable = false
|
||||||
default = {
|
default = {
|
||||||
"3_Confidential" = null
|
"3_Confidential" = {}
|
||||||
"2_Private" = null
|
"2_Private" = {}
|
||||||
"1_Sensitive" = null
|
"1_Sensitive" = {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,10 +102,7 @@ variable "network_config" {
|
||||||
type = object({
|
type = object({
|
||||||
host_project = optional(string)
|
host_project = optional(string)
|
||||||
network_self_link = optional(string)
|
network_self_link = optional(string)
|
||||||
subnet_self_links = optional(object({
|
subnet_self_link = optional(string)
|
||||||
processing_transformation = string
|
|
||||||
processing_composer = string
|
|
||||||
}), null)
|
|
||||||
composer_ip_ranges = optional(object({
|
composer_ip_ranges = optional(object({
|
||||||
connection_subnetwork = optional(string)
|
connection_subnetwork = optional(string)
|
||||||
cloud_sql = optional(string, "10.20.10.0/24")
|
cloud_sql = optional(string, "10.20.10.0/24")
|
||||||
|
@ -109,7 +110,6 @@ variable "network_config" {
|
||||||
pods_range_name = optional(string, "pods")
|
pods_range_name = optional(string, "pods")
|
||||||
services_range_name = optional(string, "services")
|
services_range_name = optional(string, "services")
|
||||||
}), {})
|
}), {})
|
||||||
# web_server_network_access_control = list(string)
|
|
||||||
})
|
})
|
||||||
nullable = false
|
nullable = false
|
||||||
default = {}
|
default = {}
|
||||||
|
|
|
@ -86,5 +86,5 @@ module "test" {
|
||||||
parent = "folders/467898377"
|
parent = "folders/467898377"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
# tftest modules=8 resources=41
|
# tftest modules=8 resources=43
|
||||||
```
|
```
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,12 +25,12 @@ Whether you’re transferring from another Cloud Service Provider or you’re ta
|
||||||
|
|
||||||
![GCS to BigQuery High-level diagram](images/diagram.png "GCS to BigQuery High-level diagram")
|
![GCS to BigQuery High-level diagram](images/diagram.png "GCS to BigQuery High-level diagram")
|
||||||
|
|
||||||
The main components that we would be setting up are (to learn more about these products, click on the hyperlinks):
|
The main components that we would be setting up are:
|
||||||
|
|
||||||
* [Cloud Storage (GCS) bucket](https://cloud.google.com/storage/): data lake solution to store extracted raw data that must undergo some kind of transformation.
|
* [Cloud Storage (GCS) bucket](https://cloud.google.com/storage/): data lake solution to store extracted raw data that must undergo some kind of transformation.
|
||||||
* [Cloud Dataflow pipeline](https://cloud.google.com/dataflow): to build fully managed batch and streaming pipelines to transform data stored in GCS buckets ready for processing in the Data Warehouse using Apache Beam.
|
* [Cloud Dataflow pipeline](https://cloud.google.com/dataflow): to build fully managed batch and streaming pipelines to transform data stored in GCS buckets ready for processing in the Data Warehouse using Apache Beam.
|
||||||
* [BigQuery datasets and tables](https://cloud.google.com/bigquery): to store the transformed data in and query it using SQL, use it to make reports or begin training [machine learning](https://cloud.google.com/bigquery-ml/docs/introduction) models without having to take your data out.
|
* [BigQuery datasets and tables](https://cloud.google.com/bigquery): to store the transformed data in and query it using SQL, use it to make reports or begin training [machine learning](https://cloud.google.com/bigquery-ml/docs/introduction) models.
|
||||||
* [Service accounts](https://cloud.google.com/iam/docs/service-account-overview) (__created with least privilege on each resource__): one for uploading data into the GCS bucket, one for Orchestration, one for Dataflow instances and one for the BigQuery tables. You can also configure users or groups of users to assign them a viewer role on the created resources and the ability to impersonate service accounts to test the Dataflow pipelines before automating them with a tool like [Cloud Composer](https://cloud.google.com/composer).
|
* [Service accounts](https://cloud.google.com/iam/docs/service-account-overview) (__created with least privilege on each resource__): one for uploading data into the GCS bucket, one for Orchestration, one for Dataflow instances and one for reading BigQuery tables. You can also configure groups of users to assign them a viewer role on the created resources and the ability to impersonate service accounts to test the Dataflow pipelines before automating them with a tool like [Cloud Composer](https://cloud.google.com/composer).
|
||||||
|
|
||||||
For a full list of the resources that will be created, please refer to the [github repository](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/data-solutions/gcs-to-bq-with-least-privileges) for this project. If you're migrating from another Cloud Provider, refer to [this](https://cloud.google.com/free/docs/aws-azure-gcp-service-comparison) documentation to see equivalent services and comparisons in Microsoft Azure and Amazon Web Services
|
For a full list of the resources that will be created, please refer to the [github repository](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/data-solutions/gcs-to-bq-with-least-privileges) for this project. If you're migrating from another Cloud Provider, refer to [this](https://cloud.google.com/free/docs/aws-azure-gcp-service-comparison) documentation to see equivalent services and comparisons in Microsoft Azure and Amazon Web Services
|
||||||
|
|
||||||
|
@ -76,7 +76,7 @@ Before we deploy the architecture, you will need the following information:
|
||||||
|
|
||||||
* The __service project ID__.
|
* The __service project ID__.
|
||||||
* A __unique prefix__ that you want all the deployed resources to have (for example: awesomestartup). This must be a string with no spaces or tabs.
|
* A __unique prefix__ that you want all the deployed resources to have (for example: awesomestartup). This must be a string with no spaces or tabs.
|
||||||
* A __list of Groups or Users__ with Service Account Token creator role on Service Accounts in IAM format, eg 'group:group@domain.com'.
|
* A __list of Groups__ with Service Account Token creator role on Service Accounts, eg '<group@domain.com>'.
|
||||||
|
|
||||||
#### Step 2: Deploying the resources
|
#### Step 2: Deploying the resources
|
||||||
|
|
||||||
|
@ -184,7 +184,7 @@ Once the job completes, you can navigate to BigQuery in the console and under __
|
||||||
|
|
||||||
The easiest way to remove all the deployed resources is to run the following command in Cloud Shell:
|
The easiest way to remove all the deployed resources is to run the following command in Cloud Shell:
|
||||||
|
|
||||||
terraform destroy -var-file=terraform.tfvars.sample -auto-approve
|
terraform destroy
|
||||||
|
|
||||||
The above command will delete the associated resources so there will be no billable charges made afterwards.
|
The above command will delete the associated resources so there will be no billable charges made afterwards.
|
||||||
<!-- BEGIN TFDOC -->
|
<!-- BEGIN TFDOC -->
|
||||||
|
@ -193,14 +193,13 @@ The above command will delete the associated resources so there will be no billa
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [prefix](variables.tf#L36) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
| [prefix](variables.tf#L37) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L54) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
| [project_config](variables.tf#L46) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. If project is created, `var.prefix` will be used. | <code title="object({ billing_account_id = optional(string), parent = string, project_id = optional(string, "gcs-df-bq") })">object({…})</code> | ✓ | |
|
||||||
| [cmek_encryption](variables.tf#L15) | Flag to enable CMEK on GCP resources created. | <code>bool</code> | | <code>false</code> |
|
| [cmek_encryption](variables.tf#L15) | Flag to enable CMEK on GCP resources created. | <code>bool</code> | | <code>false</code> |
|
||||||
| [data_eng_principals](variables.tf#L21) | Groups with Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'. | <code>list(string)</code> | | <code>[]</code> |
|
| [data_eng_principals](variables.tf#L21) | Groups with admin/developer role on enabled services and Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'. | <code>list(string)</code> | | <code>[]</code> |
|
||||||
| [network_config](variables.tf#L27) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string subnet_self_link = string })">object({…})</code> | | <code>null</code> |
|
| [network_config](variables.tf#L27) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = optional(string) subnet_self_link = optional(string) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [project_create](variables.tf#L45) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
| [region](variables.tf#L60) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
| [region](variables.tf#L59) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [vpc_subnet_range](variables.tf#L66) | Ip range used for the VPC subnet created for the example. | <code>string</code> | | <code>"10.0.0.0/20"</code> |
|
||||||
| [vpc_subnet_range](variables.tf#L65) | Ip range used for the VPC subnet created for the example. | <code>string</code> | | <code>"10.0.0.0/20"</code> |
|
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
@ -215,18 +214,17 @@ The above command will delete the associated resources so there will be no billa
|
||||||
| [service_accounts](outputs.tf#L69) | Service account. | |
|
| [service_accounts](outputs.tf#L69) | Service account. | |
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
||||||
## Test
|
## Test
|
||||||
|
|
||||||
```hcl
|
```hcl
|
||||||
module "test" {
|
module "test" {
|
||||||
source = "./fabric/blueprints/data-solutions/gcs-to-bq-with-least-privileges/"
|
source = "./fabric/blueprints/data-solutions/gcs-to-bq-with-least-privileges/"
|
||||||
project_create = {
|
project_config = {
|
||||||
billing_account_id = "123456-123456-123456"
|
billing_account_id = "123456-123456-123456"
|
||||||
parent = "folders/12345678"
|
parent = "folders/12345678"
|
||||||
}
|
|
||||||
project_id = "project-1"
|
project_id = "project-1"
|
||||||
|
}
|
||||||
prefix = "prefix"
|
prefix = "prefix"
|
||||||
}
|
}
|
||||||
# tftest modules=12 resources=47
|
# tftest modules=12 resources=43
|
||||||
```
|
```
|
||||||
|
|
|
@ -14,68 +14,37 @@
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
iam = {
|
iam = {
|
||||||
|
"roles/iam.serviceAccountUser" = [
|
||||||
|
module.service-account-orch.iam_email
|
||||||
|
]
|
||||||
|
"roles/iam.serviceAccountTokenCreator" = var.data_eng_principals
|
||||||
# GCS roles
|
# GCS roles
|
||||||
"roles/storage.objectAdmin" = [
|
"roles/storage.objectAdmin" = [
|
||||||
module.service-account-df.iam_email,
|
module.service-account-df.iam_email,
|
||||||
module.service-account-landing.iam_email
|
module.service-account-landing.iam_email
|
||||||
],
|
]
|
||||||
"roles/storage.objectViewer" = [
|
|
||||||
module.service-account-orch.iam_email,
|
|
||||||
],
|
|
||||||
# BigQuery roles
|
# BigQuery roles
|
||||||
"roles/bigquery.admin" = concat([
|
"roles/bigquery.admin" = var.data_eng_principals
|
||||||
module.service-account-orch.iam_email,
|
"roles/bigquery.dataOwner" = [
|
||||||
], var.data_eng_principals
|
|
||||||
)
|
|
||||||
"roles/bigquery.dataEditor" = [
|
|
||||||
module.service-account-df.iam_email,
|
|
||||||
module.service-account-bq.iam_email
|
|
||||||
]
|
|
||||||
"roles/bigquery.dataViewer" = [
|
|
||||||
module.service-account-bq.iam_email,
|
|
||||||
module.service-account-orch.iam_email
|
|
||||||
]
|
|
||||||
"roles/bigquery.jobUser" = [
|
|
||||||
module.service-account-df.iam_email,
|
|
||||||
module.service-account-bq.iam_email
|
|
||||||
]
|
|
||||||
"roles/bigquery.user" = [
|
|
||||||
module.service-account-bq.iam_email,
|
|
||||||
module.service-account-df.iam_email
|
module.service-account-df.iam_email
|
||||||
]
|
]
|
||||||
# common roles
|
"roles/bigquery.dataViewer" = [
|
||||||
"roles/logging.admin" = var.data_eng_principals
|
module.service-account-bq.iam_email
|
||||||
"roles/logging.logWriter" = [
|
|
||||||
module.service-account-bq.iam_email,
|
|
||||||
module.service-account-landing.iam_email,
|
|
||||||
module.service-account-orch.iam_email,
|
|
||||||
]
|
]
|
||||||
"roles/monitoring.metricWriter" = [
|
"roles/bigquery.jobUser" = [
|
||||||
module.service-account-bq.iam_email,
|
module.service-account-bq.iam_email
|
||||||
module.service-account-landing.iam_email,
|
|
||||||
module.service-account-orch.iam_email,
|
|
||||||
]
|
]
|
||||||
"roles/iam.serviceAccountUser" = [
|
# Compute
|
||||||
module.service-account-orch.iam_email,
|
"roles/compute.viewer" = var.data_eng_principals
|
||||||
]
|
|
||||||
"roles/iam.serviceAccountTokenCreator" = concat(
|
|
||||||
var.data_eng_principals
|
|
||||||
)
|
|
||||||
# Dataflow roles
|
# Dataflow roles
|
||||||
"roles/dataflow.admin" = concat(
|
"roles/dataflow.admin" = concat(
|
||||||
[module.service-account-orch.iam_email],
|
[module.service-account-orch.iam_email],
|
||||||
var.data_eng_principals
|
var.data_eng_principals
|
||||||
)
|
)
|
||||||
|
"roles/dataflow.developer" = var.data_eng_principals
|
||||||
"roles/dataflow.worker" = [
|
"roles/dataflow.worker" = [
|
||||||
module.service-account-df.iam_email,
|
module.service-account-df.iam_email,
|
||||||
]
|
]
|
||||||
"roles/dataflow.developer" = var.data_eng_principals
|
|
||||||
"roles/compute.viewer" = var.data_eng_principals
|
|
||||||
# network roles
|
|
||||||
"roles/compute.networkUser" = [
|
|
||||||
module.service-account-df.iam_email,
|
|
||||||
"serviceAccount:${module.project.service_accounts.robots.dataflow}"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
network_subnet_selflink = try(
|
network_subnet_selflink = try(
|
||||||
module.vpc[0].subnets["${var.region}/subnet"].self_link,
|
module.vpc[0].subnets["${var.region}/subnet"].self_link,
|
||||||
|
@ -86,29 +55,15 @@ locals {
|
||||||
"robot-df", "sa-df-worker"
|
"robot-df", "sa-df-worker"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
# reassemble in a format suitable for for_each
|
|
||||||
shared_vpc_bindings_map = {
|
|
||||||
for binding in flatten([
|
|
||||||
for role, members in local.shared_vpc_bindings : [
|
|
||||||
for member in members : { role = role, member = member }
|
|
||||||
]
|
|
||||||
]) : "${binding.role}-${binding.member}" => binding
|
|
||||||
}
|
|
||||||
shared_vpc_project = try(var.network_config.host_project, null)
|
|
||||||
shared_vpc_role_members = {
|
|
||||||
robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
|
|
||||||
sa-df-worker = module.service-account-df.iam_email
|
|
||||||
}
|
|
||||||
use_shared_vpc = var.network_config != null
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module "project" {
|
module "project" {
|
||||||
source = "../../../modules/project"
|
source = "../../../modules/project"
|
||||||
name = var.project_id
|
name = var.project_config.project_id
|
||||||
parent = try(var.project_create.parent, null)
|
parent = var.project_config.parent
|
||||||
billing_account = try(var.project_create.billing_account_id, null)
|
billing_account = var.project_config.billing_account_id
|
||||||
project_create = var.project_create != null
|
project_create = var.project_config.billing_account_id != null
|
||||||
prefix = var.project_create == null ? null : var.prefix
|
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||||
services = [
|
services = [
|
||||||
"bigquery.googleapis.com",
|
"bigquery.googleapis.com",
|
||||||
"bigquerystorage.googleapis.com",
|
"bigquerystorage.googleapis.com",
|
||||||
|
@ -120,19 +75,13 @@ module "project" {
|
||||||
"storage.googleapis.com",
|
"storage.googleapis.com",
|
||||||
"storage-component.googleapis.com",
|
"storage-component.googleapis.com",
|
||||||
]
|
]
|
||||||
|
iam = var.project_config.billing_account_id != null ? local.iam : {}
|
||||||
# additive IAM bindings avoid disrupting bindings in existing project
|
iam_additive = var.project_config.billing_account_id == null ? local.iam : {}
|
||||||
iam = var.project_create != null ? local.iam : {}
|
shared_vpc_service_config = var.network_config.host_project == null ? null : {
|
||||||
iam_additive = var.project_create == null ? local.iam : {}
|
|
||||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
|
||||||
attach = true
|
attach = true
|
||||||
host_project = local.shared_vpc_project
|
host_project = var.network_config.host_project
|
||||||
|
service_identity_iam = {
|
||||||
|
"roles/compute.networkUser" = ["dataflow"]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_project_iam_member" "shared_vpc" {
|
|
||||||
for_each = local.use_shared_vpc ? local.shared_vpc_bindings_map : {}
|
|
||||||
project = var.network_config.host_project
|
|
||||||
role = each.value.role
|
|
||||||
member = lookup(local.shared_vpc_role_members, each.value.member)
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,3 +0,0 @@
|
||||||
data_eng_principals = ["user:data-eng@domain.com"]
|
|
||||||
project_id = "datalake-001"
|
|
||||||
prefix = "prefix"
|
|
|
@ -19,7 +19,7 @@ variable "cmek_encryption" {
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "data_eng_principals" {
|
variable "data_eng_principals" {
|
||||||
description = "Groups with Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'."
|
description = "Groups with admin/developer role on enabled services and Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'."
|
||||||
type = list(string)
|
type = list(string)
|
||||||
default = []
|
default = []
|
||||||
}
|
}
|
||||||
|
@ -27,10 +27,11 @@ variable "data_eng_principals" {
|
||||||
variable "network_config" {
|
variable "network_config" {
|
||||||
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
|
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
|
||||||
type = object({
|
type = object({
|
||||||
host_project = string
|
host_project = optional(string)
|
||||||
subnet_self_link = string
|
subnet_self_link = optional(string)
|
||||||
})
|
})
|
||||||
default = null
|
nullable = false
|
||||||
|
default = {}
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "prefix" {
|
variable "prefix" {
|
||||||
|
@ -42,18 +43,18 @@ variable "prefix" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "project_create" {
|
variable "project_config" {
|
||||||
description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
|
description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. If project is created, `var.prefix` will be used."
|
||||||
type = object({
|
type = object({
|
||||||
billing_account_id = string
|
billing_account_id = optional(string),
|
||||||
parent = string
|
parent = string,
|
||||||
|
project_id = optional(string, "gcs-df-bq")
|
||||||
})
|
})
|
||||||
default = null
|
nullable = false
|
||||||
}
|
validation {
|
||||||
|
condition = var.project_config.billing_account_id != null || var.project_config.project_id != null
|
||||||
variable "project_id" {
|
error_message = "At least one of project_config.billing_account_id or var.project_config.project_id should be set."
|
||||||
description = "Project id, references existing project if `project_create` is null."
|
}
|
||||||
type = string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "region" {
|
variable "region" {
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.60.0" # tftest
|
version = ">= 4.76.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,7 +14,7 @@
|
||||||
|
|
||||||
module "vpc" {
|
module "vpc" {
|
||||||
source = "../../../modules/net-vpc"
|
source = "../../../modules/net-vpc"
|
||||||
count = local.use_shared_vpc ? 0 : 1
|
count = var.network_config.host_project == null ? 1 : 0
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "${var.prefix}-vpc"
|
name = "${var.prefix}-vpc"
|
||||||
subnets = [
|
subnets = [
|
||||||
|
@ -28,7 +28,7 @@ module "vpc" {
|
||||||
|
|
||||||
module "vpc-firewall" {
|
module "vpc-firewall" {
|
||||||
source = "../../../modules/net-vpc-firewall"
|
source = "../../../modules/net-vpc-firewall"
|
||||||
count = local.use_shared_vpc ? 0 : 1
|
count = var.network_config.host_project == null ? 1 : 0
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
network = module.vpc[0].name
|
network = module.vpc[0].name
|
||||||
default_rules_config = {
|
default_rules_config = {
|
||||||
|
@ -38,7 +38,7 @@ module "vpc-firewall" {
|
||||||
|
|
||||||
module "nat" {
|
module "nat" {
|
||||||
source = "../../../modules/net-cloudnat"
|
source = "../../../modules/net-cloudnat"
|
||||||
count = local.use_shared_vpc ? 0 : 1
|
count = var.network_config.host_project == null ? 1 : 0
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
name = "${var.prefix}-default"
|
name = "${var.prefix}-default"
|
||||||
|
|
|
@ -45,8 +45,8 @@ User groups provide a stable frame of reference that allows decoupling the final
|
||||||
|
|
||||||
We use groups to control access to resources:
|
We use groups to control access to resources:
|
||||||
|
|
||||||
- `data-engineers`: They handle and run workloads on the `workload` subfolder. They have editor access to all resources in the `workload` folder in order to troubleshoot possible issues within the workload. This team can also impersonate any service account in the workload folder.
|
- `gcp-data-engineers`: They handle and run workloads on the `workload` subfolder. They have editor access to all resources in the `workload` folder in order to troubleshoot possible issues within the workload. This team can also impersonate any service account in the workload folder.
|
||||||
- `data-security`: They handle security configurations for the shielded folder. They have owner access to the `audit-log` and `sec-core` projects.
|
- `gcp-data-security`: They handle security configurations for the shielded folder. They have owner access to the `audit-log` and `sec-core` projects.
|
||||||
|
|
||||||
## Encryption
|
## Encryption
|
||||||
|
|
||||||
|
@ -104,17 +104,20 @@ To deploy this blueprint in your GCP organization, you will need
|
||||||
- a folder or organization where resources will be created
|
- a folder or organization where resources will be created
|
||||||
- a billing account that will be associated with the new projects
|
- a billing account that will be associated with the new projects
|
||||||
|
|
||||||
The Shielded Folder blueprint is meant to be executed by a Service Account (or a regular user) having this minimal set of permission:
|
The Shielded Folder blueprint is meant to be executed by a Service Account having this minimal set of permission:
|
||||||
|
|
||||||
- Billing account
|
- **Billing account**
|
||||||
- `roles/billing.user`
|
- `roles/billing.user`
|
||||||
- Folder level
|
- **Organization level**:
|
||||||
|
- `roles/logging.configWriter`
|
||||||
- `roles/resourcemanager.folderAdmin`
|
- `roles/resourcemanager.folderAdmin`
|
||||||
|
- `roles/compute.orgFirewallPolicyAdmin`
|
||||||
- `roles/resourcemanager.projectCreator`
|
- `roles/resourcemanager.projectCreator`
|
||||||
|
- `roles/orgpolicy.policyAdmin`
|
||||||
|
|
||||||
The shielded Folder blueprint assumes [groups described](#user-groups) are created in your GCP organization.
|
The shielded Folder blueprint assumes [groups described](#user-groups) are created in your GCP organization. Please create them from the [https://admin.google.com/][Google Admin] console.
|
||||||
|
|
||||||
### Variable configuration PIPPO
|
### Variable configuration
|
||||||
|
|
||||||
There are several sets of variables you will need to fill in:
|
There are several sets of variables you will need to fill in:
|
||||||
|
|
||||||
|
|
|
@ -121,7 +121,7 @@ module "vpc-sc" {
|
||||||
service_perimeters_regular = {
|
service_perimeters_regular = {
|
||||||
shielded = {
|
shielded = {
|
||||||
# Move `spec` definition to `status` and comment `use_explicit_dry_run_spec` variable to enforce VPC-SC configuration
|
# Move `spec` definition to `status` and comment `use_explicit_dry_run_spec` variable to enforce VPC-SC configuration
|
||||||
# Before enforing configuration check logs and create Access Level, Ingress/Egress policy as needed
|
# Before enforcing configuration check logs and create Access Level, Ingress/Egress policy as needed
|
||||||
|
|
||||||
status = null
|
status = null
|
||||||
spec = {
|
spec = {
|
||||||
|
|
|
@ -8,9 +8,9 @@ using Fabric modules. It builds a two node cluster with a fileshare witness inst
|
||||||
The actual setup process (apart from Active Directory operations) has been scripted, so that least amount of
|
The actual setup process (apart from Active Directory operations) has been scripted, so that least amount of
|
||||||
manual works needs to performed:
|
manual works needs to performed:
|
||||||
|
|
||||||
- Joining the domain using appropriate credentials
|
- Joining the domain using appropriate credentials
|
||||||
- Running an automatically generated initialization script (`C:\InitializeCluster.ps1`)
|
- Running an automatically generated initialization script (`C:\InitializeCluster.ps1`)
|
||||||
- Creating the [Availability Groups using the wizard](https://cloud.google.com/compute/docs/instances/sql-server/configure-availability#creating_an_availability_group)
|
- Creating the [Availability Groups using the wizard](https://cloud.google.com/compute/docs/instances/sql-server/configure-availability#creating_an_availability_group)
|
||||||
(please note that healthchecks are automatically configured when the appropriate AGs are created)
|
(please note that healthchecks are automatically configured when the appropriate AGs are created)
|
||||||
|
|
||||||
To monitor the installation process, the startup scripts log output to Application Log (visible under Windows Logs in Event Viewer)
|
To monitor the installation process, the startup scripts log output to Application Log (visible under Windows Logs in Event Viewer)
|
||||||
|
@ -29,7 +29,7 @@ and to `C:\GcpSetupLog.txt` file.
|
||||||
| [secrets.tf](./secrets.tf) | Creates SQL admin user password secret. | <code>secret-manager</code> |
|
| [secrets.tf](./secrets.tf) | Creates SQL admin user password secret. | <code>secret-manager</code> |
|
||||||
| [service-accounts.tf](./service-accounts.tf) | Creates service accounts for the instances. | <code>iam-service-account</code> |
|
| [service-accounts.tf](./service-accounts.tf) | Creates service accounts for the instances. | <code>iam-service-account</code> |
|
||||||
| [variables.tf](./variables.tf) | Module variables. | |
|
| [variables.tf](./variables.tf) | Module variables. | |
|
||||||
| [vpc.tf](./vpc.tf) | Creates the VPC and manages the firewall rules and ILB. | <code>net-address</code> · <code>net-ilb</code> · <code>net-vpc</code> · <code>net-vpc-firewall</code> |
|
| [vpc.tf](./vpc.tf) | Creates the VPC and manages the firewall rules and LB. | <code>net-address</code> · <code>net-lb-int</code> · <code>net-vpc</code> · <code>net-vpc-firewall</code> |
|
||||||
|
|
||||||
## Variables
|
## Variables
|
||||||
|
|
||||||
|
@ -87,5 +87,5 @@ module "test" {
|
||||||
ad_domain_fqdn = "ad.example.com"
|
ad_domain_fqdn = "ad.example.com"
|
||||||
ad_domain_netbios = "ad"
|
ad_domain_netbios = "ad"
|
||||||
}
|
}
|
||||||
# tftest modules=12 resources=38
|
# tftest modules=12 resources=40
|
||||||
```
|
```
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# Copyright 2022 Google LLC
|
# Copyright 2023 Google LLC
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
|
@ -68,7 +68,6 @@ module "nodes" {
|
||||||
nat = false
|
nat = false
|
||||||
addresses = {
|
addresses = {
|
||||||
internal = module.ip-addresses.internal_addresses[each.value].address
|
internal = module.ip-addresses.internal_addresses[each.value].address
|
||||||
external = null
|
|
||||||
}
|
}
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
@ -122,7 +121,6 @@ module "witness" {
|
||||||
nat = false
|
nat = false
|
||||||
addresses = {
|
addresses = {
|
||||||
internal = module.ip-addresses.internal_addresses[each.value].address
|
internal = module.ip-addresses.internal_addresses[each.value].address
|
||||||
external = null
|
|
||||||
}
|
}
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue