From 5968661e8d45a66eaf3b04cbf7a677f5a4bf41c6 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 19 Sep 2022 10:34:46 +0200 Subject: [PATCH 01/29] First commit --- blueprints/README.md | 2 +- blueprints/data-solutions/README.md | 9 +- .../data-solutions/composer-2/README.md | 106 ++++++++++++ .../composer-2/backend.tf.sample | 30 ++++ .../data-solutions/composer-2/composer.tf | 96 +++++++++++ .../data-solutions/composer-2/diagram.png | Bin 0 -> 17693 bytes blueprints/data-solutions/composer-2/main.tf | 159 ++++++++++++++++++ .../data-solutions/composer-2/outputs.tf | 25 +++ .../data-solutions/composer-2/variables.tf | 85 ++++++++++ .../data_solutions/composer_2/__init__.py | 13 ++ .../data_solutions/composer_2/fixture/main.tf | 27 +++ .../data_solutions/composer_2/test_plan.py | 19 +++ 12 files changed, 569 insertions(+), 2 deletions(-) create mode 100644 blueprints/data-solutions/composer-2/README.md create mode 100644 blueprints/data-solutions/composer-2/backend.tf.sample create mode 100644 blueprints/data-solutions/composer-2/composer.tf create mode 100644 blueprints/data-solutions/composer-2/diagram.png create mode 100644 blueprints/data-solutions/composer-2/main.tf create mode 100644 blueprints/data-solutions/composer-2/outputs.tf create mode 100644 blueprints/data-solutions/composer-2/variables.tf create mode 100644 tests/blueprints/data_solutions/composer_2/__init__.py create mode 100644 tests/blueprints/data_solutions/composer_2/fixture/main.tf create mode 100644 tests/blueprints/data_solutions/composer_2/test_plan.py diff --git a/blueprints/README.md b/blueprints/README.md index b38370bb..61323dab 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -5,7 +5,7 @@ This section **[networking blueprints](./networking/)** that implement core patt Currently available blueprints: - **cloud operations** - [Resource tracking and remediation via Cloud Asset feeds](./cloud-operations/asset-inventory-feed-remediation), [Granular Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Granular Cloud DNS IAM for Shared VPC](./cloud-operations/dns-shared-vpc), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Packer image builder](./cloud-operations/packer-image-builder), [On-prem SA key management](./cloud-operations/onprem-sa-key-management), [TCP healthcheck for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [HTTP Load Balancer with Cloud Armor](./cloud-operations/glb_and_armor) -- **data solutions** - [GCE/GCS CMEK via centralized Cloud KMS](./data-solutions/gcs-to-bq-with-least-privileges/), [Cloud Storage to Bigquery with Cloud Dataflow with least privileges](./data-solutions/gcs-to-bq-with-least-privileges/), [Data Platform Foundations](./data-solutions/data-platform-foundations/), [SQL Server AlwaysOn availability groups blueprint](./data-solutions/sqlserver-alwayson), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion/) +- **data solutions** - [GCE/GCS CMEK via centralized Cloud KMS](./data-solutions/gcs-to-bq-with-least-privileges/), [Cloud Storage to Bigquery with Cloud Dataflow with least privileges](./data-solutions/gcs-to-bq-with-least-privileges/), [Data Platform Foundations](./data-solutions/data-platform-foundations/), [SQL Server AlwaysOn availability groups blueprint](./data-solutions/sqlserver-alwayson), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion/), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2/) - **factories** - [The why and the how of resource factories](./factories/README.md) - **GKE** - [GKE multitenant fleet](./gke/multitenant-fleet/), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [Binary Authorization Pipeline](./gke/binauthz/), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api/) - **networking** - [hub and spoke via peering](./networking/hub-and-spoke-peering/), [hub and spoke via VPN](./networking/hub-and-spoke-vpn/), [DNS and Google Private Access for on-premises](./networking/onprem-google-access-dns/), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [ILB as next hop](./networking/ilb-next-hop), [PSC for on-premises Cloud Function invocation](./networking/private-cloud-function-from-onprem/), [decentralized firewall](./networking/decentralized-firewall) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index 4abebf9d..44311b63 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -30,7 +30,7 @@ This [blueprint](./data-platform-foundations/) implements SQL Server Always On A ### Cloud SQL instance with multi-region read replicas - + This [blueprint](./cloudsql-multiregion/) creates a [Cloud SQL instance](https://cloud.google.com/sql) with multi-region read replicas as described in the [Cloud SQL for PostgreSQL disaster recovery](https://cloud.google.com/architecture/cloud-sql-postgres-disaster-recovery-complete-failover-fallback) article.
@@ -41,3 +41,10 @@ This [blueprint](./data-playground/) creates a [Vertex AI Notebook](https://cloud.google.com/vertex-ai/docs/workbench/introduction) running on a VPC with a private IP and a dedicated Service Account. A GCS bucket and a BigQuery dataset are created to store inputs and outputs of data experiments.
+ +### Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key + + +This [blueprint](./composer-2/) creates a [Cloud Composer](https://cloud.google.com/sql) version 2 instance on a VPC with a dedicated service account. The solution supports as inputs: a Shared VPC and Cloud KMS CMEK keys. +
\ No newline at end of file diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md new file mode 100644 index 00000000..1c05ec6a --- /dev/null +++ b/blueprints/data-solutions/composer-2/README.md @@ -0,0 +1,106 @@ +# Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key + +This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. + +The solution will use: + - Cloud Composer + - VPC with Private Service Access to deploy resources, if no Shared VPC configuration provided. + - Google Cloud NAT to access internet resources, if no Shared VPC configuration provided. + +The solution supports as inputs: + - Shared VPC + - Cloud KMS CMEK keys + +This is the high level diagram: + +![Cloud Composer 2 architecture overview](./diagram.png "Cloud Composer 2 architecture overview") + +# Requirements +This blueprint will deploy all its resources into the project defined by the project_id variable. Please note that we assume this project already exists. However, if you provide the appropriate values to the `project_create` variable, the project will be created as part of the deployment. + +If `project_create` is left to null, the identity performing the deployment needs the owner role on the project defined by the `project_id` variable. Otherwise, the identity performing the deployment needs `resourcemanager.projectCreator` on the resource hierarchy node specified by `project_create.parent` and `billing.user` on the billing account specified by `project_create.billing_account_id`. + +# Deployment +Run Terraform init: + +``` +$ terraform init +``` + +Configure the Terraform variable in your terraform.tfvars file. You need to spefify at least the following variables: + +``` +project_id = "lcaggioni-sandbox" +prefix = "lc" +``` + +You can run now: + +``` +$ terraform apply +``` + +You can now connect to your instance. + +# Customizations + +## Shared VPC +As is often the case in real-world configurations, this blueprint accepts as input an existing [`Shared-VPC`](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. + +Example: +``` +network_config = { + host_project = "PROJECT" + network_self_link = "projects/PROJECT/global/networks/VPC_NAME" + subnet_self_link = "projects/PROJECT/regions/REGION/subnetworks/VPC_NAME" + composer_secondary_ranges = { + pods = "pods" + services = "services" + } +} +``` + +Make sure that: +- The GKE API (`container.googleapis.com`) is enabled in the VPC host project. +- The subnet has secondary ranges configured with 2 ranges: + - pods: `/22` example: `10.10.8.0/22` + - services = `/24` example: 10.10.12.0/24` +- Firewall rules are set, as described in the [documentation](https://cloud.google.com/composer/docs/how-to/managing/configuring-private-ip#step_3_configure_firewall_rules) + +In order to run the example and deploy Cloud Composer on a shared VPC the identity running Terraform must have the following IAM role on the Shared VPC Host project. + - Compute Network Admin (roles/compute.networkAdmin) + - Compute Shared VPC Admin (roles/compute.xpnAdmin) + +## Encryption +As is often the case in real-world configurations, this blueprint accepts as input an existing [`Cloud KMS keys`](https://cloud.google.com/kms/docs/cmek) via the `service_encryption_keys` variable. + +Example: +``` +service_encryption_keys = { + `europe/west1` = `projects/PROJECT/locations/REGION/keyRings/KR_NAME/cryptoKeys/KEY_NAME` +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [organization_domain](variables.tf#L51) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L56) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | +| [project_id](variables.tf#L70) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [composer_config](variables.tf#L17) | Composer environemnt configuration. | object({…}) | | {…} | +| [groups](variables.tf#L29) | User groups. | map(string) | | {…} | +| [network_config](variables.tf#L37) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L61) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L75) | Region where instances will be deployed. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L81) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [composer_airflow_uri](outputs.tf#L22) | The URI of the Apache Airflow Web UI hosted within the Cloud Composer environment.. | | +| [composer_dag_gcs](outputs.tf#L17) | The Cloud Storage prefix of the DAGs for the Cloud Composer environment. | | + + diff --git a/blueprints/data-solutions/composer-2/backend.tf.sample b/blueprints/data-solutions/composer-2/backend.tf.sample new file mode 100644 index 00000000..49a0883d --- /dev/null +++ b/blueprints/data-solutions/composer-2/backend.tf.sample @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The `impersonate_service_account` option require the identity launching terraform +# role `roles/iam.serviceAccountTokenCreator` on the Service Account specified. + +terraform { + backend "gcs" { + bucket = "BUCKET_NAME" + prefix = "PREFIX" + impersonate_service_account = "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com" + } +} +provider "google" { + impersonate_service_account = "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com" +} +provider "google-beta" { + impersonate_service_account = "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/blueprints/data-solutions/composer-2/composer.tf b/blueprints/data-solutions/composer-2/composer.tf new file mode 100644 index 00000000..28a9fffc --- /dev/null +++ b/blueprints/data-solutions/composer-2/composer.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "comp-sa" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + prefix = var.prefix + name = "cmp" + display_name = "Composer service account" + iam = { + "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] + } +} + +resource "google_composer_environment" "env" { + name = "${var.prefix}-composer" + project = module.project.project_id + region = var.region + config { + software_config { + image_version = var.composer_config.image_version + } + workloads_config { + scheduler { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 1 + } + web_server { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + worker { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 1 + max_count = 3 + } + } + environment_size = var.composer_config.environment_size + + node_config { + network = local.orch_vpc + subnetwork = local.orch_subnet + service_account = module.comp-sa.email + enable_ip_masq_agent = "true" + tags = ["composer-worker"] + ip_allocation_policy { + cluster_secondary_range_name = try( + var.network_config.composer_secondary_ranges.pods, "pods" + ) + services_secondary_range_name = try( + var.network_config.composer_secondary_ranges.services, "services" + ) + } + } + private_environment_config { + enable_private_endpoint = "true" + cloud_sql_ipv4_cidr_block = try( + var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24" + ) + master_ipv4_cidr_block = try( + var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28" + ) + } + dynamic "encryption_config" { + for_each = ( + try(lookup(var.service_encryption_keys, var.region, null) != null, false) + ? { 1 = 1 } + : {} + ) + content { + kms_key_name = try(lookup(var.service_encryption_keys, var.region, null), null) + } + } + } + depends_on = [ + google_project_iam_member.shared_vpc, + ] +} diff --git a/blueprints/data-solutions/composer-2/diagram.png b/blueprints/data-solutions/composer-2/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..b8ffc12efdd4244a9536379a6e6ff9474c58336c GIT binary patch literal 17693 zcmeIabyQr>@;HblXn>Fef=+M<4DK2Wzd=I+ zek*Op$bkzYR7FY@p=^j`19-48)s!}qmq%a#?$HoX5Qz{_@2UV_Aw=T)dr3q(1mu6p zfp&r|5m5eJM*+CreIkMH9n8P3NEt}~se!DPf&8C)gqypn*s8wxzy;mmtripk;RW^G z7g1V;Y99ds3ByuN(^*qqj?dWM7Hnu@Z)6H~w{^IKLJ)B018!|ioee?mwl;Q9K6gQi ze`@dn_jkq26rg{qI9m%+Xv!;r#O$3+L0n)KFbjnc1_%Taa56FDQxTW^H#qPlNMYgZ z?7+v&?B?bMc4G(IJDD@H^78UBv#>FYN zmiBg_yLJtY>|LA%DJbqby8rxZoX(bJ4?WpI|2-{Wg3Nbsm|4Lr%>NUZsk`O>1=!u2 ze}VlouYYwXaMu{0iYe6I#^r8Q@9Zp{h1dlC>Er)g{4hv?ySDh0EZt3QG{r4#P3@on zs1Oe)0Qf(?`5z&*9w6B`IsO~+zux>CQh@mmTmOZXi;B z0)hyFwD=n}cf_3*w-^$QhVpq2r(p|@K3-bcI7Y2X6P*`?zuqI#%5@RGc$Dn?rvxfI z#ODUTa)kY5oP;>|TR*|v?R;-*L*Cvs%|p3H)vr!s8lLkqa0Equh_qE-F}yD~Ep0b( zg9Rd}$n4Qli#s}LnTIT)@#bUPH{KP|P8ixYR2#R$ zEZB0Esu3m8zPERDc1l&uIqQEv94$$MuugruHzOr*S@Qw?B*v1aMbC{~T5|N`6YAnO z|5bBFyq&$4cFC#XMY*1EtCNzcC4IFH?` zH}z_~_{haPR!9Y&)}74mH)EAQ_EI@s{vK~EY3NuQS0^VYWWx3Nt@Syy%{(0&PEFQ| zsXUJEi=&(xa_Lp=AWffb5q(8iVzksRo#a?{w*#3^ZL3|4g7$4C+Hh2fD9`)|@**LX z2CELvAS28RWUJZ|qaym7oXxO9)WA+iD(QsOnMMjB%W)mmhbj^)Bl zR<2g!i#*-TFTKiz+Ad3NHGHm4JBz{k&3y+8*05sZ;n)Pxn$lt??aBQH>FL{x z!av*e;WXA`=P3dz4VOFd7vKPAcDW$U$IQR~+}_XVHwDt}-tGQEC|l6<{@ICDZmDaU zu$wkzgSMBW$zH^fOO5Gs)z#iqy;~_ExajxaDV?texTvN)8~K(OBo=OR*0hPYR*3RS zb1iQwWob`jB{`|h{VPd{*pw&vnaB%;y*9nCwPNGjM0R2++aoE}SZ(8yVvpLYCc3Q@ z)6pJ%5QIP`q+Zu#xc^<=aJn%fsm`v+ZTO?BZ||mWZ*S< zU)mRnOluq+%C3QlaAR^;qNi-(>;7kp;|#`yVx>Z9Ee`M>vDSlmWbSW0eF2q8iN2wpl_^$7#9r&3TxEkvGS;rmw4=olX<0P_H3SAhX(N; zvNECF;-|if1D&pv@%2pO8100%vb2`UY!2SNj7IREM(L8fIyrXU@q{MxX>b_+9SD>) zK8tF(pCtm4^edJ*UO&EsycyoqBpGg_Jr-qtBE5lMqdAC+nX5Yzr4lp`;z@)8MG|!L zlo!k8Ua}&fu=$HP4Y(&d&>SP)Z`~pwFyc>rUGZlRd+UrC!a=;?R22KecIgOg@P8?? z?zd0e9x`Bq-foCXLBH=aL>A$IfB+BmUJVdA#TB_Sh1D#}JrwBzVD%w%_q#qyWiY~D zX0I}8bEJEO{e3EO)hQu<{|t&0M8Wl&fim&p-(w6n9smPVU_6lF_N({j{b|;{i4R5f z0GK15@`Erl!2#xasw+E$azFo{)bGzjLqqPLML=GFZc@3X?L>2r`~TNb^qv+wU*-4i z+YV(OlzV)CL-mgk`A&GsPWrdJXMWZ3qI;i;m*f6*4+)a->l?z;!4!V~vsCG@V_hlKZgQ?l+SH>tEaW6Wz~?N1tC8-Mg7?)-TwD{(yoNkm3@wh z^1;u`^>R_)-JXuFw+|;!)Z7ZzcDULqMdnp{ItMr5l;yuCQiE|6f$TEN#m{ziTT!M^ zCunmaf1m+Nz1QL=%5Ipevvd9~T{YG{Q%TAg2z(Z1Z9I1PMl&`aI}>NS#>|J@ zoKNIZlsL@QC6o=*VwI)L-6lTGcIBOBeVEf?r|}TjU802zCbQ>O=ZJK=?iNO^*l+wz zG>bK0$w6`nXBxF#F_nOtsXu$RoaA*z%Cq6x5XYFL?DALF)+c(AJVl_;;BGqM73x~AIVa8X;(xoUE5 zCu-jL*7o+y=>^$yB{)gCMv<~>Q!y7fnb#+3kP0%b)Y*)Lomdm}p62B? zDYoq|Z0(WMpLD&}E&E2l8@>}<-ROF>B1%L0PAD)+W<70g3AJkPFzV(7x zi$@G~{Pz*_o+tdIPseU9AwAt#?Q4uPPilJ83PU<wVZX;!Y5Zx*vH;hyPeRDERemh^6(%? zyeMg8%2K!1$#*i5sb1~R(~L|v-d1V-nY&2-?FHf6GaVGp(%~htZ<3~; zn-;s$(&`1}4}O-WjxSBJAJov!?zq@G3S2{#oU+1ue!*R1C4Rei6m080`Q?YxIrtoj zLo%h*t8@dPraHT*2WERI+wS`prdjq{c47H5x^e}Pkbbp=*x1bg(i+i^(7@b0XLTXCi?}zkLmlSnEX$ zCHC4$Jq|fP5Dt4QaYL?q723WV8Ks3!`;4p3m8_7jn3{m?q!(O(+Ms=Pd6UKvt3{}8pSd*g z=Uuy)kBPW$o!O1GJ?Z)75q}(G3iy@#p<1edY`^vI6fJ^Mq?bjghY$Tg(56+cwz8h&iwvYRMw#GyJ^n@4}HSrnP8{< zx`{qn0~iwm=7o&~MrEtcEiX_N)A>d;uQKueu$#lyRxYw$aOtT2SUXuy=M4U@t(pdD zhTfS0XJS0f8c_B7mZftc5J9OgQ!=-<+HF+HWh844+L7d&_JxjCvD9xARXelLD~E}f z4c5xli&gTVOdN*r7iLBfwtMdk@)rlx zr>B`xkb9wLzz#Tnk|8XOnD?8{Q$Saf4Iq2aP2Ud!P&TgL*TMJt%i91z!^86&e{dmj zQNBN~nnBHO`riZk2f60X@S;^Jh6B`oe&bf(w*Ew3u}y7}=Ax;8|r(@G+kj5aDA-F>VETxUuNlpk*Ai637c5~pQBNzAIWUf($1czAfg;qVTW zH>_wssWbQ)xS46~DypCVexgXc>F5O7-FI}#?B1~q+S}J%tQYjz`oz8lllk~x%-{qN z8Dm>}3fVm=d;<&uEz8IU`KYA;sH`gl31c7c^%He~$X;WQK9EK~V6s$RAwfOwi`r!n z4$O1kJSYn$yns0nr*EKub5GEk08|Obo%vt;%AxTVzotoEUY?Vaa~{B{j*d?82j$RQ zimJ-W!v(Ls`6f4e2BQ3bTy4^K6pbNC1jvL@`QM4=Y491Cue!dz@OgXN?e*SX)1DCE z>VULe&hPmiB`;!ujaN~S{8jT6YO1Rfct25%Yinr4JFbh*X5Mjt|2Q6?rAII^0pp;j z`sTW$UDIkkRTg4Q=e?H5ZvKKLzfJMcbC z;YR@(=p*TspGC=nJ?-1quq zO8DmBOWW09KdoX~^U)B$zWX24cT;6kdR{y0BiX>Elw-y3dde?V*zbmT@jPhc!;R3* z`4SP)&1qem?XpGKc_)FsfYb6@DXH!ldezBNojTuZ&ng!5J0gFnfY7fWqz`I*jEIIM z-A)LvA-b)u&Cbj;oG5xn25V0)r9rgGNKv(6)f@qsk;~uqLpZ?O&^H|}W(#5aw z7GJ%&zN|@G!qh+i@goqk#=0X2`wlgxc50Y-)H4Bh%_rVjq-FgxtG|1L3{phVx54`j ztLpU_P<)FIMYx1q{*7@4F2=)t2c$kFlOy9U%9B5lrfOc#uSg6Ta+83kr^)zKh`*_$ zqmbsqf;VSHCkLbhhltg^LXS#3vK9N!^YNxPH#fDa%nF~Qe?L5Qq7+oiCcoJa4xZ&q z@ebeDe(V7O$3I*C{_%0JYCKz_B#6-I>e}ZPyI7FSEzc$VMg$aiH^Sepr#x24%%0Aj zg)Bejso~!f@lQfXXH;DS9%u5PJVJ9m<{7F9xB#nIM0~9Ahn5n|FOSKHB@=u(H=Led z3p=k|7m*r5RpipkVKk3z5Za#l69t74|5Z#<{IGX^urxS0I0bQoY;W0?AVolEPW7;< zty46r9&y4I)!P>q^zzwx^YJh1x8=e9N<9bTUN`#9(^XA;=dr~tV>AvgqcIaE+3xIr zXgRhZAOZe99cFBqsWO+nvV0lb<$ZF@MT68;%9DkZBxTXK8$GqIe{&e$+1UwDa#ZZd z+R^n2v)`u#N}jAPdI5D5DNwX?^2^?c^1RL6^&5bTI#K&Z4;|6Mfz zh+!1nA}D=%s=~Ot8qN6T3Z9#ri%upAIORY#wmB6QBXizIgWO)8p1X}(Wm#Dil&Y9g z11(fG;s%N?8LfienWkC3O$@kLC%$W*`W~#3y2KE%Mm`y!edP~T8k~V7!whe+6)Y_n zjRv;%{BAE#cYfV&q~G*kXUxDqb)M@xG-@6z0RD*MG7YYw=w_kR*)UC4I!oV7l zz7a|7EWRUB^IXr_-@ivm*Ntg8TktZ|=it5W*T1R!Gb)c+@iQzYlrkrj(01gx;Zf#o934;5rmyjzQxqbht_Tg^370fwtWj`&w!?M*y%<=zXc9_Hf+QhqYub3>-z6Yt9_K1?5zR+;oh`COjTTcyPL^q7V%q=r#$ z7Uw((J6?|=7emU0N7zI5`cHVt=NDe9mcI~f@ff%*CP;D9UxOOn5O-j9WU4A62>TV9 z3%E)R4^Q~rtogOx-dvJj3-iNFQjX8z9r_vkA+@;f*YL2@+ZrVPw#=9IYpTvx zUj%akE&>WgH-Pv54JD2+&v+a~^J7ceEG0iB%hLU9s_o{4f3Fz|$X@fKUZ~yDr}rg} zPPGIlVSf7`$rIp*%LEgNi-7EUo-4K|VQOkBY3O5JZKcV3(Px2?G4tVT~!|ZBSJ;QUgzywt~oRDq;RLovS&eg?@e%i6@}o- ze>#Fqonw&ou+tN}sS*921kV9=B&zHxYWlT_O6ji*B71Q-_JhEWw;U z7CfUoG=NVNdSKcZY737(Kf=Dg{C&i8^tYKmPaM8cf^}A&D^blJs zKg}{6A;sRwTsu7{)+TkcJBZaq zt8K7tYiqSN*%Qv*dQ}68^hMIV`OC7XO6uvV;XDuRUs*6^54s%NuOq!#a#0SWh?dnQ zhb)c!c1rOL^R8>wC{PIqt(+dDvxvp8_&KR9m;5x)#l_|6hsmq&me?5m!vzhht6=Nd z0jj&;0LT_)5(XtN!cN)461-6O^2q7}X^7;D@fd~uPaU^3*v$NzQUDHWn(y+?zP+!i zhJ{Y`m>l_4 zKsVJz8g|UjHk}<{)Q2VnB#f(J2oY&k{3&v@ z3AX@kezE@ihIj>Xi)N}tYM51reOwgo+kp1jhO(obG=kUHp9M=@F`y*edP&fkx$oBX zD+2=FDO=zJp|)=3ih^Tk)1^?=2)@^JtyR8`2_El2l(%)cr;c<@ zr_bGLIW@KLwK;>`VjW%j!$PA;FNj&d3c~%0D&Rqr0k0eQTIpJ$=}U~aHAbBn)_Gs! zsyGA_o;Qxq<+^KOq=15%yFitaF3+Ec7rzfj!ZYr3qR1wRCrW}1|Cd&yb76DYdJ~m?jf&TWU~yU5WZvYF-CVt+!kF#A<&8}$Wa$og3X>8Xru>&0fV-_21v=Okl)7E;<~u<^cfkH1JI z+SQJ|U-WAVYwI9rpN@UYgtLO>ymij=Ua>vl+f(~nF88Srr{n1{H<<%rNwv@)FBL@W z-;BvZ#y;!jYyapEI8(P~>0pg1&feAA!PqFJaiPtn=X*(`*g44!_S+?+ffkV|Q=BZcaDPd#*7n+JC!bp>x zmA|%v%^Y&Zrwz-GjgL$1ePpY!PC_$&Q&>%+HCoxGI|t!LZYL`(C#eu|*{`d{#4qP- zNEG(_eJ52>Zgq@|xk?rEs(p1ceo<-K&mnxG=yx$z25EBsB_&AaZ=Pk;nzgD{1ot_$xlsD?1w- z@7;#AV~Y@<3nJ9X98eVIxG)2#}6M(kTB zBO?>UK*GH}VepApO?CMTHDY|n7B;*YS&^~M3&rf!++KZc?|N1rTN*U|K%!7`Bl=KsSv$@8WLF zywoihVQxLo+Me@t8>N&v@S7^vFONzx%_R)_yi*JV(4ab4&hn9(K|h+>r06RuB-&!- zD1~~T=GQURvcSo0%dE~A?y!k^d4Gjv=Gy8>MHUD4?mOQIW6|&J0q%ZuU>2bpWZ4)Pl&0nj7}*%X zg46W|FKE8rX@^v}>O29Nx*GM6i;1_3#?0unV)(eJ@`6*1D_$=otKN&Du4IvnD=JibMHI5|2GUM?{eebjL%J( zQ1`&6j#yK;(3|$PQ}CJa&$ji$g5JFasg7SEh3iaWbr5IPx$vof8=Gm_&51Y z9nD*}=)`$B`K;aqJZ%hT7Z%Td6vH7?tN}{=)dtuPo$eD~pR3x7cWsM=-}Ye?)D;FG zDDxr$^GEr8q7iE;=KIE6p_v7``S;X_=UII{NCz-3ozR;1e_&eC89llXzC99_;$$z; zs*jVfRb`^4+KVZyR#%t7^!`s}MUXb~z#Ot^$2I1$! z4DLIlpqN)T9Bi@4$;F1cJELgjtUcD%*$whUa6$L|j2Rno(T={S5k%HpG3lpzt}~`j z{hr2zw4tdN{<7I|Jw7dNj%A0C>#X?hha33lh=|HWD`?$&|NCQOHCES++Nt>U+<3jw zqL>guWl6-u3zHrWz2cbcf&&Id4f!BeA-l_GZ}|{^F0OyIqIfHBoRa>rmzs#5@cLB& zIwIf_1wEGRQk9pDss1g7|K-@CVlparsL@hh7A}4JPF>v!q?sj;BP-7`Jq0r%w8(>`5!q`$)Z3L< zatggV1hVSnZkXB=6s zR!o(rFFbuNLAuv+DBYV%n&#%t8?&!yEtoR5Z>8GNT1lD&uY^kod>VvMsk|{5c{Vk` zrlBg8QA_PyCa-$><8gy)gWFKtIy>~y*{kx~!s%ZGhrtv2?#EnHOLRSsb1tRjK9N{n zw+Xi}{zuIsYwnQ6rXjZzHaC@+bUw3k+|_bz~)pWw^NzWS_NS*n?=JDv+p=3cdDA_gMFqOa}lLQPZD4bkC| z!@Zcd+jBj?{&VI^AN^zhFHZO2OJxojXfTJVenfwT!RO1zhR2 zBDz7rKMuG(97t>eJ+t}2amVQ{c^T@LR*_RFG`SK#lPby(8LMuehISPhQc6*%pf`wc zW71_hE<3zO6SAaIO4+Nw6kD)|BtzHUwcfh=40TV1O?QpVELm(4*4vQfL+Nl}e6;hg zOyNUcAe-0Z+|>H;LZ6w@>eUYLE54KP`kmTe&K+B|+|2c9LKjY!3Z>pReglW_DZgtC zpA#&I1+X=6AyM9_=aib0(9xs(4SFQgr)+_OR7;cbRlmV>ggya(GDZE_t ztU1l(WANmD|3$sf71!VKa7!A)B)M3=ImQ3|M+h z8X%8)Yp-Q~r*HTaCoPsd>Oh1kR+l>ai4@;U#C+88orPa(2anC}KgwQ?$9FOP9)$TCv& zWbi|HJvmYl?FgB6t+A)}v)bdHl)Wi`i%)o%f1Z;_cZ&C^la{-l=M5%v zGk=v-JNx8zE$nAF%(!UCec`v;qtC~5HF>^BzPaGYhOr6QjEhaL%%!Vms5*ptB2iab z)K$lyyGU*A4~iyvU2f^$dTvdYn#mzS2|i7uaLlyrIXs~!WMq+?KjpAf3)u* z-EGfU4vzN{@`3jtz4|FM=HPiCi+a9rQ~9s0KzL*jtc4>uG?vtBHzb0ILZaQf?id)) zWT~8^o9f)3Z+qLlfz>OUxUpuFFnl~7BZYOeG6aQ z@r1Hg*m{Tq8NPR^*3K%w#ASj zzIBH~DLyWuC7SW%PrhEpB;&4oOsqqJ5M?0l1*phDa`FO13KpQ2Ki7W=+9HF-3gXC| zMxjn>c?oKg)vEILDi?q!Iub3KV!SxVOz0Yj;7q_iqa>Q~6kY&^ST2c#QslMRZ#6v` zWn?{u@KFBdcecdffxP`Uki-?iBH<@01;gQd?tP7ZR5Ji^u z>J>TzY{_}e;(`VO9Kqwrye*l=-X8PQi5Ptqmr--=_LFS#>mHV=Ui*AeMvKj$Q}FPUfE$y1uMw?tjq>`C?-XCX{MCvn;_~de}N| zZ+j&Rvax8r z40Z&EmQKj!_7mQ%j`|<3+is(&Pdq+`St!8cVNinAeL2#XdNN-k+MC|_6PyN<-q0H* z`x?%Ar7G@j|I%36BzAyt4_igRGQZgz(ekG+{+b%*-G2hTE?Ta)CWF8VQU3{Hb{aN0 z#mPQNF?tq7Ih?-$B`a=YF?f+};px_Ek&mqXST37PKNNp$Akml(Q1+D4V-mYeOe@$s z)2}9zT;6F~Vv8go*92*>^X_x>(-~eg#BwU{vghn@aLW8p4jv2n_~^sn4`Ob+xmw%g zL+Tr6=fPAJ=^8Wf>KH_oYt|PUPXL@?Br`?dmDZK&0AZ$xh2f^| z%S|0-qilkxtCZnHnl)Aa$7-40_1+d9CFHlEn^lO2IzSw@vq6_v&s>4ABb|nq7wVH} z(Lo4s2zG2=kU7|eL(?KIEg&__mWF60!BR|Ko51{K#Bj+I7{#-E<#}AcE&<`KZTIk2 zi;UX5ye^)fcRmBhwcGPAw~-w??~1 ze9z$7x$GtZiy?%EOfWaSrcwUfpKxLFy?4$AOI0q3`8EMvdybfi_;sW}AUj2Nat+4? zBEV;$7CF~{#2fv*i&lYFvc`)i)1L!eQcGE#G+_VKuM51?(`C}Cd_tky`wfq%%xL`E@O>dECu9!@e*6K9=4CkKW2g2fit>Xl!V2diD?Fk8&tM75`O_q{>0jLY>PxQt z&!>fUrJ$GRBA_ue#+1K&BZMWk*z;#bUuZIGUFc1cjYZRbZ4nawnkb6E4xJ+=Xw>KZ z9WODwemico_D**G56gFNP-OT&@zA1X!5v{A!=RYl5?|)Y0aryZX&2>)%Im@2dSK|) z3{9zcm~-37G?-RB(_!tlt2n}j&m)SclhbdAWNB(RDjOTMMTa+yL`apk)nZ4&ND4(t zCzLqe`HY84k^87sJvBud|6N&861A_@(T1B1_Q>Ns-urS!*a~(I?T5_-WSuL1@9i9_ zffCXXbGk6(1+>TIrr~2}7Bz~9d1>xgdR%^fX(w^X8SZz%ny$7D*ih%wu5Eb*o7uU$ zI&mIrY7XrA%b$lWXmEQhcB0VVFE?gQk+0~d3vahmbO?8#V^iTAR#qfF;dPO8!p&|& z@6zGe_O_CGlFQ+vRQwSehGeDwNT36?>EaVcutJryx zF*pEvzrxI3CT8Wynldb+t^5jpQmlWyIcr@$vCYR+G`y)D-V(tFie(M*0vt-#2jv4^ zP8+nUe`$pd&^yNK3_Bi`u1#zjFl`La;_*U9rpvG*G2%dvlGgI!eoo{0U)O;^ZE@DA z)lLYWlK;L|VT1hke9=J17vY=w7g&k;nL6utvBZroIaDfaQ#CG3D%MwGHAG)nTxctN zl4LOa>G4UXUU)TncC-c^LP_wTgvT_u)BHZ=WSd(X`rCTHp>x$k+~z9Ys%|P5l9afF z_<-fj`aP}nV;43y)%Ua1J#Q(i?TT))=mrJ+!|G?;-+z-kVbBSlbqn7pf02}~+bV0a z1H%4Yys}6OK4bCpuTp%cz?5s4s30<@l+5j4Uk>lM)i1l6NQ8Nb4e~g&o6U1Ys1lOnv5B*I{h>qU&FL?MVz4o>xFRS8du&S zE{Q}C|EAr~(t>Z`Lqt1lgMv$W+0!=9$`}FMm0UkC92vzb$&-z-#-cpTaC-a%5OE30I%TzV$6T8He zpXfbaBZn``67iW;b{v!6p_Fojwi!NR30rSi<)V9I@s2)tGo4MM&0EG#5^#{eI z|4|q%$t4NI=HQ|}_l5WCTd8hEhUFe>hug(b8S*0)n|aYZKXKuB)MB@mhq}uQ*-7U4 z5gAm=Z_0;-_z^hL$p5MrrCOKI-5QQI-`D_U4l$*oyjQ!O}L zp_0cfqfC!A^eDSpRe76nAT)|RU*~1>FFupF@AD-2kkk>K9g+F!iYbgSDt_TW0{1{JV44p#K`R2`(^emF}bcDW>Cz_KUv-mtky#dEnX{L z7J#Xc$2U%*4xFWHwp<~uuqs)h9Cf13?no?-uhtk-BnR9nAx1W{f$w*J0O>0^=R%dE z&6L}tZWri4NLDgFfgi6*y)Ylnb%Pq>=F+LilYW{OJxo=K}2U7w9a2Nt~uLzsyu@U=+EwB|VG{DRKGVXHG(~gsT5UGXLl?o_;cj@DZnU9VxIOcKIM14rC*vTIU9mYhzA>< z=5V!))Gp>Ld0zV6zM4*asJ9fAL(_&y?k)xGJ=0yvqhsPa6$%J9@Vx+kA7efX%cPHX z+$)?vuhmPwnGtPJ%f?xJm`b*9wcL%m?FaJB`u~KohLk8nB(_MEmnK*71jfoYa6yrB*LW$&(pZt?}+1eQkM-&p(NEHgA9gy3l@c z+E)nv>nMK#7A9y<|5d?94&4wz67Ay6E{zIqF@kQ9d~! z%8SHtYQ_8QKd}wwj}NhpJCvFTA)3p}_I0qzA53_r+;CJC@agUB%SU70aFEA$n6e@^ z+?Mh~8l2!C84OB<^S-Exd?9l;;%O~|!QPO76ek34Jt?iD*WGedts zz-^^4-k6xuOPbcJ5!_0x0Q$!+MBYpJls-kL-G0c8VW9^2AlV|?-{xwDq4`(t%cta8DAP^X1-{EbOOys(c-fkr#_GL!I7meG7Dzw@i4;8?6XXXp#hDGO5smx2*GLec zlzl)bd~%=mx(!4?%=3#S^zRd$MLqy@gpQQM_mJy|kN7h^wd{q@L*6Sokp4)G7PY42)|j=XzPjG$48O;|qiBb(>lF91dHVvuLFL=g zC-)`a5P|t&gP%g~^5On<46GM8EiU^)diEionhQAXUG-feEA)QKAT0n binding + } + + shared_vpc_project = try(var.network_config.host_project, null) + use_shared_vpc = var.network_config != null + + vpc_self_link = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.vpc.0.self_link + ) + + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_link + : values(module.vpc.0.subnet_self_links)[0] + ) + + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.vpc.0.self_link + ) + groups = { + for k, v in var.groups : k => "${v}@${var.organization_domain}" + } + groups_iam = { + for k, v in local.groups : k => "group:${v}" + } +} + +module "project" { + source = "../../../modules/project" + name = var.project_id + parent = try(var.project_create.parent, null) + billing_account = try(var.project_create.billing_account_id, null) + project_create = var.project_create != null + prefix = var.project_create == null ? null : var.prefix + iam = var.project_create != null ? local.iam : {} + iam_additive = var.project_create == null ? local.iam : {} + services = [ + "cloudkms.googleapis.com", + "container.googleapis.com", + "containerregistry.googleapis.com", + "composer.googleapis.com", + "compute.googleapis.com", + "iap.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "networkmanagement.googleapis.com", + "servicenetworking.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com", + ] + + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + service_identity_iam = {} + } + + service_encryption_key_ids = { + composer = [try(lookup(var.service_encryption_keys, var.region, null), null)] + } + + service_config = { + disable_on_destroy = false, disable_dependent_services = false + } +} + +module "vpc" { + source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "vpc" + subnets = [ + { + ip_cidr_range = "10.0.0.0/20" + name = "subnet" + region = var.region + secondary_ip_range = { + pods = "10.10.8.0/22" + services = "10.10.12.0/24" + } + } + ] +} + +module "firewall" { + source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + network = module.vpc.0.name + admin_ranges = ["10.0.0.0/20"] +} + +module "nat" { + source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + region = var.region + name = "${var.prefix}-default" + router_network = module.vpc.0.name +} + +resource "google_project_iam_member" "shared_vpc" { + for_each = local.use_shared_vpc ? local.shared_vpc_bindings_map : {} + project = var.network_config.host_project + role = each.value.role + member = lookup(local.shared_vpc_role_members, each.value.member) +} diff --git a/blueprints/data-solutions/composer-2/outputs.tf b/blueprints/data-solutions/composer-2/outputs.tf new file mode 100644 index 00000000..a2943006 --- /dev/null +++ b/blueprints/data-solutions/composer-2/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "composer_dag_gcs" { + description = "The Cloud Storage prefix of the DAGs for the Cloud Composer environment." + value = google_composer_environment.env.config[0].dag_gcs_prefix +} + +output "composer_airflow_uri" { + description = "The URI of the Apache Airflow Web UI hosted within the Cloud Composer environment.." + value = google_composer_environment.env.config[0].airflow_uri +} diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf new file mode 100644 index 00000000..36ba2edf --- /dev/null +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "composer_config" { + description = "Composer environemnt configuration." + type = object({ + environment_size = string + image_version = string + }) + default = { + environment_size = "ENVIRONMENT_SIZE_SMALL" + image_version = "composer-2-airflow-2" + } +} + +variable "groups" { + description = "User groups." + type = map(string) + default = { + data-engineers = "gcp-data-engineers" + } +} + +variable "network_config" { + description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." + type = object({ + host_project = string + network_self_link = string + subnet_self_link = string + composer_secondary_ranges = object({ + pods = string + services = string + }) + }) + default = null +} + +variable "organization_domain" { + description = "Organization domain." + type = string +} + +variable "prefix" { + description = "Unique prefix used for resource names. Not used for project if 'project_create' is null." + type = string +} + +variable "project_create" { + description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "project_id" { + description = "Project id, references existing project if `project_create` is null." + type = string +} + +variable "region" { + description = "Region where instances will be deployed." + type = string + default = "europe-west1" +} + +variable "service_encryption_keys" { + description = "Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use." + type = map(string) + default = null +} diff --git a/tests/blueprints/data_solutions/composer_2/__init__.py b/tests/blueprints/data_solutions/composer_2/__init__.py new file mode 100644 index 00000000..6d6d1266 --- /dev/null +++ b/tests/blueprints/data_solutions/composer_2/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/blueprints/data_solutions/composer_2/fixture/main.tf b/tests/blueprints/data_solutions/composer_2/fixture/main.tf new file mode 100644 index 00000000..384a069a --- /dev/null +++ b/tests/blueprints/data_solutions/composer_2/fixture/main.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../../blueprints/data-solutions/composer-2/" + project_id = "project" + + organization_domain = "example.com" + project_create = { + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "prefix" +} diff --git a/tests/blueprints/data_solutions/composer_2/test_plan.py b/tests/blueprints/data_solutions/composer_2/test_plan.py new file mode 100644 index 00000000..017d9979 --- /dev/null +++ b/tests/blueprints/data_solutions/composer_2/test_plan.py @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_resources(e2e_plan_runner): + "Test that plan works and the numbers of resources is as expected." + modules, resources = e2e_plan_runner() + assert len(modules) == 5 + assert len(resources) == 28 From df8bcc1bb2ad60217f7908d349869dad3d6e5fe8 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 19 Sep 2022 12:00:42 +0200 Subject: [PATCH 02/29] Fix resource dependencies. --- blueprints/data-solutions/composer-2/composer.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/blueprints/data-solutions/composer-2/composer.tf b/blueprints/data-solutions/composer-2/composer.tf index 28a9fffc..dd26a4c1 100644 --- a/blueprints/data-solutions/composer-2/composer.tf +++ b/blueprints/data-solutions/composer-2/composer.tf @@ -92,5 +92,6 @@ resource "google_composer_environment" "env" { } depends_on = [ google_project_iam_member.shared_vpc, + module.project ] } From 489acfdb32405b9e689f35c0453444849cdf5917 Mon Sep 17 00:00:00 2001 From: Arindam Sirohia Date: Tue, 20 Sep 2022 00:09:59 +0530 Subject: [PATCH 03/29] Added new binary_authorization argument in gke-cluster module --- modules/gke-cluster/README.md | 59 ++++++++++++++++---------------- modules/gke-cluster/main.tf | 7 ++++ modules/gke-cluster/variables.tf | 6 ++++ 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 20a3f2fc..a34b72b9 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -68,13 +68,13 @@ module "cluster-1" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [location](variables.tf#L155) | Cluster zone or region. | string | ✓ | | -| [name](variables.tf#L222) | Cluster name. | string | ✓ | | -| [network](variables.tf#L227) | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | -| [project_id](variables.tf#L271) | Cluster project id. | string | ✓ | | -| [secondary_range_pods](variables.tf#L294) | Subnet secondary range name used for pods. | string | ✓ | | -| [secondary_range_services](variables.tf#L299) | Subnet secondary range name used for services. | string | ✓ | | -| [subnetwork](variables.tf#L304) | VPC subnetwork name or self link. | string | ✓ | | +| [location](variables.tf#L161) | Cluster zone or region. | string | ✓ | | +| [name](variables.tf#L228) | Cluster name. | string | ✓ | | +| [network](variables.tf#L233) | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | +| [project_id](variables.tf#L277) | Cluster project id. | string | ✓ | | +| [secondary_range_pods](variables.tf#L300) | Subnet secondary range name used for pods. | string | ✓ | | +| [secondary_range_services](variables.tf#L305) | Subnet secondary range name used for services. | string | ✓ | | +| [subnetwork](variables.tf#L310) | VPC subnetwork name or self link. | string | ✓ | | | [addons](variables.tf#L17) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} | | [authenticator_security_group](variables.tf#L53) | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null | | [cluster_autoscaling](variables.tf#L59) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | {…} | @@ -83,28 +83,29 @@ module "cluster-1" { | [description](variables.tf#L97) | Cluster description. | string | | null | | [dns_config](variables.tf#L103) | Configuration for Using Cloud DNS for GKE. | object({…}) | | null | | [enable_autopilot](variables.tf#L113) | Create cluster in autopilot mode. With autopilot there's no need to create node-pools and some features are not supported (e.g. setting default_max_pods_per_node). | bool | | false | -| [enable_dataplane_v2](variables.tf#L119) | Enable Dataplane V2 on the cluster, will disable network_policy addons config. | bool | | false | -| [enable_intranode_visibility](variables.tf#L125) | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null | -| [enable_l4_ilb_subsetting](variables.tf#L131) | Enable L4ILB Subsetting. | bool | | null | -| [enable_shielded_nodes](variables.tf#L137) | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null | -| [enable_tpu](variables.tf#L143) | Enable Cloud TPU resources in this cluster. | bool | | null | -| [labels](variables.tf#L149) | Cluster resource labels. | map(string) | | null | -| [logging_config](variables.tf#L160) | Logging configuration (enabled components). | list(string) | | null | -| [logging_service](variables.tf#L166) | Logging service (disable with an empty string). | string | | "logging.googleapis.com/kubernetes" | -| [maintenance_config](variables.tf#L172) | Maintenance window configuration. | object({…}) | | {…} | -| [master_authorized_ranges](variables.tf#L198) | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} | -| [min_master_version](variables.tf#L204) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | -| [monitoring_config](variables.tf#L210) | Monitoring configuration (enabled components). | list(string) | | null | -| [monitoring_service](variables.tf#L216) | Monitoring service (disable with an empty string). | string | | "monitoring.googleapis.com/kubernetes" | -| [node_locations](variables.tf#L232) | Zones in which the cluster's nodes are located. | list(string) | | [] | -| [notification_config](variables.tf#L238) | GKE Cluster upgrade notifications via PubSub. | bool | | false | -| [peering_config](variables.tf#L244) | Configure peering with the master VPC for private clusters. | object({…}) | | null | -| [pod_security_policy](variables.tf#L254) | Enable the PodSecurityPolicy feature. | bool | | null | -| [private_cluster_config](variables.tf#L260) | Enable and configure private cluster, private nodes must be true if used. | object({…}) | | null | -| [release_channel](variables.tf#L276) | Release channel for GKE upgrades. | string | | null | -| [resource_usage_export_config](variables.tf#L282) | Configure the ResourceUsageExportConfig feature. | object({…}) | | {…} | -| [vertical_pod_autoscaling](variables.tf#L309) | Enable the Vertical Pod Autoscaling feature. | bool | | null | -| [workload_identity](variables.tf#L315) | Enable the Workload Identity feature. | bool | | true | +| [enable_binary_authorization](variables.tf#L119) | Enable Google Binary Authorization. | bool | | false | +| [enable_dataplane_v2](variables.tf#L125) | Enable Dataplane V2 on the cluster, will disable network_policy addons config. | bool | | false | +| [enable_intranode_visibility](variables.tf#L131) | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null | +| [enable_l4_ilb_subsetting](variables.tf#L137) | Enable L4ILB Subsetting. | bool | | null | +| [enable_shielded_nodes](variables.tf#L143) | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null | +| [enable_tpu](variables.tf#L149) | Enable Cloud TPU resources in this cluster. | bool | | null | +| [labels](variables.tf#L155) | Cluster resource labels. | map(string) | | null | +| [logging_config](variables.tf#L166) | Logging configuration (enabled components). | list(string) | | null | +| [logging_service](variables.tf#L172) | Logging service (disable with an empty string). | string | | "logging.googleapis.com/kubernetes" | +| [maintenance_config](variables.tf#L178) | Maintenance window configuration. | object({…}) | | {…} | +| [master_authorized_ranges](variables.tf#L204) | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} | +| [min_master_version](variables.tf#L210) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L216) | Monitoring configuration (enabled components). | list(string) | | null | +| [monitoring_service](variables.tf#L222) | Monitoring service (disable with an empty string). | string | | "monitoring.googleapis.com/kubernetes" | +| [node_locations](variables.tf#L238) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [notification_config](variables.tf#L244) | GKE Cluster upgrade notifications via PubSub. | bool | | false | +| [peering_config](variables.tf#L250) | Configure peering with the master VPC for private clusters. | object({…}) | | null | +| [pod_security_policy](variables.tf#L260) | Enable the PodSecurityPolicy feature. | bool | | null | +| [private_cluster_config](variables.tf#L266) | Enable and configure private cluster, private nodes must be true if used. | object({…}) | | null | +| [release_channel](variables.tf#L282) | Release channel for GKE upgrades. | string | | null | +| [resource_usage_export_config](variables.tf#L288) | Configure the ResourceUsageExportConfig feature. | object({…}) | | {…} | +| [vertical_pod_autoscaling](variables.tf#L315) | Enable the Vertical Pod Autoscaling feature. | bool | | null | +| [workload_identity](variables.tf#L321) | Enable the Workload Identity feature. | bool | | true | ## Outputs diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf index 56f7ea75..decb55a0 100644 --- a/modules/gke-cluster/main.tf +++ b/modules/gke-cluster/main.tf @@ -292,6 +292,13 @@ resource "google_container_cluster" "cluster" { } } + dynamic "binary_authorization" { + for_each = var.enable_binary_authorization == true ? [""] : [] + content { + evaluation_mode = "PROJECT_SINGLETON_POLICY_ENFORCE" + } + } + dynamic "dns_config" { for_each = var.dns_config != null ? [""] : [] content { diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf index 58be03c9..2c9fccbe 100644 --- a/modules/gke-cluster/variables.tf +++ b/modules/gke-cluster/variables.tf @@ -116,6 +116,12 @@ variable "enable_autopilot" { default = false } +variable "enable_binary_authorization" { + description = "Enable Google Binary Authorization." + type = bool + default = false +} + variable "enable_dataplane_v2" { description = "Enable Dataplane V2 on the cluster, will disable network_policy addons config." type = bool From eb73e13b670b007e584adf623e60cb1f1573bd3a Mon Sep 17 00:00:00 2001 From: Arindam Sirohia Date: Tue, 20 Sep 2022 00:35:06 +0530 Subject: [PATCH 04/29] Updated for each condition of binary authorization in gke-cluster module --- modules/gke-cluster/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf index decb55a0..e0b31a31 100644 --- a/modules/gke-cluster/main.tf +++ b/modules/gke-cluster/main.tf @@ -293,7 +293,7 @@ resource "google_container_cluster" "cluster" { } dynamic "binary_authorization" { - for_each = var.enable_binary_authorization == true ? [""] : [] + for_each = var.enable_binary_authorization ? [""] : [] content { evaluation_mode = "PROJECT_SINGLETON_POLICY_ENFORCE" } From 6c42fa350441c0f5a851f2541ed9f8d2b95f2614 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 20 Sep 2022 10:35:06 +0200 Subject: [PATCH 05/29] Add articatregistry service --- blueprints/data-solutions/composer-2/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/blueprints/data-solutions/composer-2/main.tf b/blueprints/data-solutions/composer-2/main.tf index 7bb2ea8c..faf9e6dc 100644 --- a/blueprints/data-solutions/composer-2/main.tf +++ b/blueprints/data-solutions/composer-2/main.tf @@ -87,6 +87,7 @@ module "project" { iam = var.project_create != null ? local.iam : {} iam_additive = var.project_create == null ? local.iam : {} services = [ + "artifactregistry.googleapis.com", "cloudkms.googleapis.com", "container.googleapis.com", "containerregistry.googleapis.com", From 495a5bbfffa87e1dca3eb530c31f655f380cfa5e Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 20 Sep 2022 10:36:12 +0200 Subject: [PATCH 06/29] Add Artifact registry Service Identity SA creation. --- modules/project/service-accounts.tf | 1 + tests/blueprints/gke/binauthz/test_plan.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index 7d584fa7..4bb73062 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -67,6 +67,7 @@ locals { } ) service_accounts_jit_services = [ + "artifactregistry.googleapis.com", "cloudasset.googleapis.com", "gkehub.googleapis.com", "pubsub.googleapis.com", diff --git a/tests/blueprints/gke/binauthz/test_plan.py b/tests/blueprints/gke/binauthz/test_plan.py index 6e176b1c..cf012c06 100644 --- a/tests/blueprints/gke/binauthz/test_plan.py +++ b/tests/blueprints/gke/binauthz/test_plan.py @@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner() assert len(modules) == 13 - assert len(resources) == 42 + assert len(resources) == 43 From f28dc5cb04002f04359e436d16552ae59ef2cb0a Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 20 Sep 2022 10:51:11 +0200 Subject: [PATCH 07/29] Fix tests --- blueprints/data-solutions/data-platform-foundations/README.md | 2 +- .../data_solutions/data_platform_foundations/test_plan.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 76810f54..d2b88550 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -222,7 +222,7 @@ module "data-platform" { prefix = "myprefix" } -# tftest modules=42 resources=314 +# tftest modules=42 resources=315 ``` ## Customizations diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index e5db6ffc..0e4b77f5 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -24,4 +24,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 41 - assert len(resources) == 313 + assert len(resources) == 314 From 0704a0ad3057481b772b89ea317771da7f18ede0 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 20 Sep 2022 12:26:57 +0200 Subject: [PATCH 08/29] Fixes based on comments. --- .../data-solutions/composer-2/README.md | 31 ++++++++++--------- .../data-solutions/composer-2/composer.tf | 7 ++--- blueprints/data-solutions/composer-2/main.tf | 21 +++++-------- .../data-solutions/composer-2/variables.tf | 15 +++------ .../data_solutions/composer_2/fixture/main.tf | 1 - .../data_solutions/composer_2/test_plan.py | 2 +- 6 files changed, 31 insertions(+), 46 deletions(-) diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index 1c05ec6a..e8c15eec 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -1,6 +1,10 @@ # Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key -This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. +This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. Cloud Composer 2 is the new major verion for Cloud Composer that supports: + - environment autoscaling + - workloads configuration: CPU, memory, and storage parameters for Airflow workers, schedulers, web server, and database. + +Please consult the [documentation page](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) for an exaustive comparison between Composer Version 1 and Version 2. The solution will use: - Cloud Composer @@ -23,20 +27,20 @@ If `project_create` is left to null, the identity performing the deployment need # Deployment Run Terraform init: -``` +```bash $ terraform init ``` Configure the Terraform variable in your terraform.tfvars file. You need to spefify at least the following variables: -``` +```tfvars project_id = "lcaggioni-sandbox" prefix = "lc" ``` You can run now: -``` +```bash $ terraform apply ``` @@ -48,7 +52,7 @@ You can now connect to your instance. As is often the case in real-world configurations, this blueprint accepts as input an existing [`Shared-VPC`](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Example: -``` +```tfvars network_config = { host_project = "PROJECT" network_self_link = "projects/PROJECT/global/networks/VPC_NAME" @@ -75,7 +79,7 @@ In order to run the example and deploy Cloud Composer on a shared VPC the identi As is often the case in real-world configurations, this blueprint accepts as input an existing [`Cloud KMS keys`](https://cloud.google.com/kms/docs/cmek) via the `service_encryption_keys` variable. Example: -``` +```tfvars service_encryption_keys = { `europe/west1` = `projects/PROJECT/locations/REGION/keyRings/KR_NAME/cryptoKeys/KEY_NAME` } @@ -86,15 +90,14 @@ service_encryption_keys = { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [organization_domain](variables.tf#L51) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L56) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | -| [project_id](variables.tf#L70) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [prefix](variables.tf#L49) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | +| [project_id](variables.tf#L63) | Project id, references existing project if `project_create` is null. | string | ✓ | | | [composer_config](variables.tf#L17) | Composer environemnt configuration. | object({…}) | | {…} | -| [groups](variables.tf#L29) | User groups. | map(string) | | {…} | -| [network_config](variables.tf#L37) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L61) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L75) | Region where instances will be deployed. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L81) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | +| [iam_groups_map](variables.tf#L29) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | +| [network_config](variables.tf#L35) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L54) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L68) | Region where instances will be deployed. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L74) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | ## Outputs diff --git a/blueprints/data-solutions/composer-2/composer.tf b/blueprints/data-solutions/composer-2/composer.tf index dd26a4c1..e22d91ff 100644 --- a/blueprints/data-solutions/composer-2/composer.tf +++ b/blueprints/data-solutions/composer-2/composer.tf @@ -20,9 +20,6 @@ module "comp-sa" { prefix = var.prefix name = "cmp" display_name = "Composer service account" - iam = { - "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] - } } resource "google_composer_environment" "env" { @@ -81,12 +78,12 @@ resource "google_composer_environment" "env" { } dynamic "encryption_config" { for_each = ( - try(lookup(var.service_encryption_keys, var.region, null) != null, false) + try(var.service_encryption_keys[var.region], null) != null ? { 1 = 1 } : {} ) content { - kms_key_name = try(lookup(var.service_encryption_keys, var.region, null), null) + kms_key_name = try(var.service_encryption_keys[var.region], null) } } } diff --git a/blueprints/data-solutions/composer-2/main.tf b/blueprints/data-solutions/composer-2/main.tf index faf9e6dc..e8556296 100644 --- a/blueprints/data-solutions/composer-2/main.tf +++ b/blueprints/data-solutions/composer-2/main.tf @@ -15,14 +15,13 @@ */ locals { - iam = { - "roles/composer.worker" = [ - module.comp-sa.iam_email - ] - "roles/composer.ServiceAgentV2Ext" = [ - "serviceAccount:${module.project.service_accounts.robots.composer}" - ] - } + iam = merge( + { + "roles/composer.worker" = [module.comp-sa.iam_email] + "roles/composer.ServiceAgentV2Ext" = ["serviceAccount:${module.project.service_accounts.robots.composer}"] + }, + var.iam_groups_map + ) _shared_vpc_bindings = { "roles/compute.networkUser" = [ @@ -69,12 +68,6 @@ locals { ? var.network_config.network_self_link : module.vpc.0.self_link ) - groups = { - for k, v in var.groups : k => "${v}@${var.organization_domain}" - } - groups_iam = { - for k, v in local.groups : k => "group:${v}" - } } module "project" { diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index 36ba2edf..b4c8ed98 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -26,12 +26,10 @@ variable "composer_config" { } } -variable "groups" { - description = "User groups." - type = map(string) - default = { - data-engineers = "gcp-data-engineers" - } +variable "iam_groups_map" { + description = "Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}." + type = map(list(string)) + default = null } variable "network_config" { @@ -48,11 +46,6 @@ variable "network_config" { default = null } -variable "organization_domain" { - description = "Organization domain." - type = string -} - variable "prefix" { description = "Unique prefix used for resource names. Not used for project if 'project_create' is null." type = string diff --git a/tests/blueprints/data_solutions/composer_2/fixture/main.tf b/tests/blueprints/data_solutions/composer_2/fixture/main.tf index 384a069a..4b35e6f8 100644 --- a/tests/blueprints/data_solutions/composer_2/fixture/main.tf +++ b/tests/blueprints/data_solutions/composer_2/fixture/main.tf @@ -18,7 +18,6 @@ module "test" { source = "../../../../../blueprints/data-solutions/composer-2/" project_id = "project" - organization_domain = "example.com" project_create = { billing_account_id = "123456-123456-123456" parent = "folders/12345678" diff --git a/tests/blueprints/data_solutions/composer_2/test_plan.py b/tests/blueprints/data_solutions/composer_2/test_plan.py index 017d9979..5f69f185 100644 --- a/tests/blueprints/data_solutions/composer_2/test_plan.py +++ b/tests/blueprints/data_solutions/composer_2/test_plan.py @@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner() assert len(modules) == 5 - assert len(resources) == 28 + assert len(resources) == 29 From ee309ecc06ee177df2cf320b05c8dcaaf0fe3e17 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 20 Sep 2022 16:57:35 +0200 Subject: [PATCH 09/29] Update firewall rules. --- .../data-solutions/data-platform-foundations/variables.tf | 2 +- .../data/firewall-rules/dev/rules.yaml | 8 ++++---- fast/stages/03-data-platform/dev/demo | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) create mode 120000 fast/stages/03-data-platform/dev/demo diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index ac8d06b4..dc59de29 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -28,7 +28,7 @@ variable "composer_config" { }) default = { node_count = 3 - airflow_version = "composer-1.17.5-airflow-2.1.4" + airflow_version = "composer-1-airflow-2" env_variables = {} } } diff --git a/fast/stages/02-networking-peering/data/firewall-rules/dev/rules.yaml b/fast/stages/02-networking-peering/data/firewall-rules/dev/rules.yaml index d4df8cdc..3e2d9cc9 100644 --- a/fast/stages/02-networking-peering/data/firewall-rules/dev/rules.yaml +++ b/fast/stages/02-networking-peering/data/firewall-rules/dev/rules.yaml @@ -4,8 +4,8 @@ ingress-allow-composer-nodes: description: "Allow traffic to Composer nodes." direction: INGRESS action: allow - sources: [] - ranges: ["0.0.0.0/0"] + sources: + - composer-worker targets: - composer-worker use_service_accounts: false @@ -17,8 +17,8 @@ ingress-allow-dataflow-load: description: "Allow traffic to Dataflow nodes." direction: INGRESS action: allow - sources: [] - ranges: ["0.0.0.0/0"] + sources: + - dataflow targets: - dataflow use_service_accounts: false diff --git a/fast/stages/03-data-platform/dev/demo b/fast/stages/03-data-platform/dev/demo new file mode 120000 index 00000000..7a0e7c1e --- /dev/null +++ b/fast/stages/03-data-platform/dev/demo @@ -0,0 +1 @@ +../../../../blueprints/data-solutions/data-platform-foundations/demo/ \ No newline at end of file From 149d84d41deaf99122c03ac9388901f2b24215fc Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 20 Sep 2022 17:08:24 +0200 Subject: [PATCH 10/29] Fix NVA and VPC firewall rules. --- .../02-networking-nva/data/firewall-rules/dev/rules.yaml | 8 ++++---- .../02-networking-vpn/data/firewall-rules/dev/rules.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml b/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml index d4df8cdc..3e2d9cc9 100644 --- a/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml +++ b/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml @@ -4,8 +4,8 @@ ingress-allow-composer-nodes: description: "Allow traffic to Composer nodes." direction: INGRESS action: allow - sources: [] - ranges: ["0.0.0.0/0"] + sources: + - composer-worker targets: - composer-worker use_service_accounts: false @@ -17,8 +17,8 @@ ingress-allow-dataflow-load: description: "Allow traffic to Dataflow nodes." direction: INGRESS action: allow - sources: [] - ranges: ["0.0.0.0/0"] + sources: + - dataflow targets: - dataflow use_service_accounts: false diff --git a/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml b/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml index d4df8cdc..3e2d9cc9 100644 --- a/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml +++ b/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml @@ -4,8 +4,8 @@ ingress-allow-composer-nodes: description: "Allow traffic to Composer nodes." direction: INGRESS action: allow - sources: [] - ranges: ["0.0.0.0/0"] + sources: + - composer-worker targets: - composer-worker use_service_accounts: false @@ -17,8 +17,8 @@ ingress-allow-dataflow-load: description: "Allow traffic to Dataflow nodes." direction: INGRESS action: allow - sources: [] - ranges: ["0.0.0.0/0"] + sources: + - dataflow targets: - dataflow use_service_accounts: false From d6c6ae943e9a196dd42ab30f543a5d96527e59cd Mon Sep 17 00:00:00 2001 From: Natalia Strelkova Date: Wed, 21 Sep 2022 08:14:24 +0000 Subject: [PATCH 11/29] regional MySQL DBs - automatic backup conf --- modules/cloudsql-instance/main.tf | 9 +++++---- tests/modules/cloudsql_instance/test_plan.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index de4b5560..6cd280be 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -18,10 +18,11 @@ locals { prefix = var.prefix == null ? "" : "${var.prefix}-" is_mysql = can(regex("^MYSQL", var.database_version)) has_replicas = try(length(var.replicas) > 0, false) + is_regional = var.availability_type == "REGIONAL" ? true : false // Enable backup if the user asks for it or if the user is deploying - // MySQL with replicas - enable_backup = var.backup_configuration.enabled || (local.is_mysql && local.has_replicas) + // MySQL in HA configuration (regional or with specified replicas) + enable_backup = var.backup_configuration.enabled || (local.is_mysql && local.has_replicas) || (local.is_mysql && local.is_regional) users = { for user, password in coalesce(var.users, {}) : @@ -76,11 +77,11 @@ resource "google_sql_database_instance" "primary" { content { enabled = true - // enable binary log if the user asks for it or we have replicas, + // enable binary log if the user asks for it or we have replicas (default in regional), // but only for MySQL binary_log_enabled = ( local.is_mysql - ? var.backup_configuration.binary_log_enabled || local.has_replicas + ? var.backup_configuration.binary_log_enabled || local.has_replicas || local.is_regional : null ) start_time = var.backup_configuration.start_time diff --git a/tests/modules/cloudsql_instance/test_plan.py b/tests/modules/cloudsql_instance/test_plan.py index f23c69c7..72d31dad 100644 --- a/tests/modules/cloudsql_instance/test_plan.py +++ b/tests/modules/cloudsql_instance/test_plan.py @@ -90,6 +90,17 @@ def test_mysql_replicas_enables_backup(plan_runner): assert backup_config['binary_log_enabled'] +def test_mysql_binary_log_for_regional(plan_runner): + "Test that the binary log will be enabled for regional MySQL DBs." + + _, resources = plan_runner(database_version="MYSQL_8_0", availability_type="REGIONAL") + assert len(resources) == 1 + primary = [r for r in resources if r['name'] == 'primary'][0] + backup_config = primary['values']['settings'][0]['backup_configuration'][0] + assert backup_config['enabled'] + assert backup_config['binary_log_enabled'] + + def test_users(plan_runner): "Test user creation." From 78026a4d5a691b8f2519b51b737767bb9927e33a Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 21 Sep 2022 15:00:52 +0200 Subject: [PATCH 12/29] Improve composer config variable. --- .../data-solutions/composer-2/README.md | 16 ++--- .../data-solutions/composer-2/composer.tf | 59 ++++++++++++------- .../data-solutions/composer-2/variables.tf | 38 +++++++++++- 3 files changed, 81 insertions(+), 32 deletions(-) diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index e8c15eec..00dbcc0f 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -90,14 +90,14 @@ service_encryption_keys = { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L49) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | -| [project_id](variables.tf#L63) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [composer_config](variables.tf#L17) | Composer environemnt configuration. | object({…}) | | {…} | -| [iam_groups_map](variables.tf#L29) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | -| [network_config](variables.tf#L35) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L54) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L68) | Region where instances will be deployed. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L74) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | +| [prefix](variables.tf#L81) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | +| [project_id](variables.tf#L95) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [composer_config](variables.tf#L17) | Composer environemnt configuration. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables. | object({…}) | | {…} | +| [iam_groups_map](variables.tf#L61) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | +| [network_config](variables.tf#L67) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L86) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L100) | Region where instances will be deployed. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L106) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | ## Outputs diff --git a/blueprints/data-solutions/composer-2/composer.tf b/blueprints/data-solutions/composer-2/composer.tf index e22d91ff..1217e0d4 100644 --- a/blueprints/data-solutions/composer-2/composer.tf +++ b/blueprints/data-solutions/composer-2/composer.tf @@ -27,29 +27,46 @@ resource "google_composer_environment" "env" { project = module.project.project_id region = var.region config { - software_config { - image_version = var.composer_config.image_version - } - workloads_config { - scheduler { - cpu = 0.5 - memory_gb = 1.875 - storage_gb = 1 - count = 1 - } - web_server { - cpu = 0.5 - memory_gb = 1.875 - storage_gb = 1 - } - worker { - cpu = 0.5 - memory_gb = 1.875 - storage_gb = 1 - min_count = 1 - max_count = 3 + dynamic "software_config" { + for_each = ( + try(var.composer_config.software_config, null) != null + ? { 1 = 1 } + : {} + ) + content { + airflow_config_overrides = try(var.composer_config.software_config.airflow_config_overrides, null) + pypi_packages = try(var.composer_config.software_config.pypi_packages, null) + env_variables = try(var.composer_config.software_config.env_variables, null) + image_version = try(var.composer_config.software_config.image_version, null) + python_version = try(var.composer_config.software_config.python_version, null) + scheduler_count = try(var.composer_config.software_config.scheduler_count, null) } } + dynamic "workloads_config" { + for_each = (try(var.composer_config.workloads_config, null) != null ? { 1 = 1 } : {}) + + content { + scheduler { + cpu = try(var.composer_config.workloads_config.scheduler.cpu, null) + memory_gb = try(var.composer_config.workloads_config.scheduler.memory_gb, null) + storage_gb = try(var.composer_config.workloads_config.scheduler.storage_gb, null) + count = try(var.composer_config.workloads_config.scheduler.count, null) + } + web_server { + cpu = try(var.composer_config.workloads_config.web_server.cpu, null) + memory_gb = try(var.composer_config.workloads_config.web_server.memory_gb, null) + storage_gb = try(var.composer_config.workloads_config.web_server.storage_gb, null) + } + worker { + cpu = try(var.composer_config.workloads_config.worker.cpu, null) + memory_gb = try(var.composer_config.workloads_config.worker.memory_gb, null) + storage_gb = try(var.composer_config.workloads_config.worker.storage_gb, null) + min_count = try(var.composer_config.workloads_config.worker.min_count, null) + max_count = try(var.composer_config.workloads_config.worker.max_count, null) + } + } + } + environment_size = var.composer_config.environment_size node_config { diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index b4c8ed98..c39f3b32 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -15,14 +15,46 @@ */ variable "composer_config" { - description = "Composer environemnt configuration." + description = "Composer environemnt configuration. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables." type = object({ environment_size = string - image_version = string + software_config = any + workloads_config = object({ + scheduler = object( + { + cpu = number + memory_gb = number + storage_gb = number + count = number + } + ) + web_server = object( + { + cpu = number + memory_gb = number + storage_gb = number + } + ) + worker = object( + { + cpu = number + memory_gb = number + storage_gb = number + min_count = number + max_count = number + } + ) + }) }) default = { environment_size = "ENVIRONMENT_SIZE_SMALL" - image_version = "composer-2-airflow-2" + software_config = { + image_version = "composer-2-airflow-2" + env_variables = { + FOO = "bar" + } + } + workloads_config = null } } From 6e90af2bc39ec5600f9a460831e0579f3be14e97 Mon Sep 17 00:00:00 2001 From: Krzysiek Mucha <47009740+kmucha555@users.noreply.github.com> Date: Wed, 21 Sep 2022 23:30:35 +0200 Subject: [PATCH 13/29] Add support for service_label property in internal load balancer (#834) Co-authored-by: Krzysztof Mucha --- modules/net-ilb-l7/README.md | 22 +++++++++---------- modules/net-ilb-l7/forwarding-rule.tf | 1 + modules/net-ilb-l7/variables.tf | 12 +++++----- tests/modules/net_ilb_l7/fixture/variables.tf | 12 +++++----- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/modules/net-ilb-l7/README.md b/modules/net-ilb-l7/README.md index 5dabc52d..64a3f776 100644 --- a/modules/net-ilb-l7/README.md +++ b/modules/net-ilb-l7/README.md @@ -403,18 +403,18 @@ An Internal HTTP Load Balancer is made of multiple components, that change depen |---|---|:---:|:---:|:---:| | [name](variables.tf#L17) | Load balancer name. | string | ✓ | | | [project_id](variables.tf#L22) | Project id. | string | ✓ | | -| [region](variables.tf#L157) | The region where to allocate the ILB resources. | string | ✓ | | -| [subnetwork](variables.tf#L187) | The subnetwork where the ILB VIP is allocated. | string | ✓ | | +| [region](variables.tf#L159) | The region where to allocate the ILB resources. | string | ✓ | | +| [subnetwork](variables.tf#L189) | The subnetwork where the ILB VIP is allocated. | string | ✓ | | | [backend_services_config](variables.tf#L27) | The backends services configuration. | map(object({…})) | | {} | -| [forwarding_rule_config](variables.tf#L98) | Forwarding rule configurations. | object({…}) | | {…} | -| [health_checks_config](variables.tf#L116) | Custom health checks configuration. | map(object({…})) | | {} | -| [health_checks_config_defaults](variables.tf#L127) | Auto-created health check default configuration. | object({…}) | | {…} | -| [https](variables.tf#L145) | Whether to enable HTTPS. | bool | | false | -| [network](variables.tf#L151) | The network where the ILB is created. | string | | "default" | -| [ssl_certificates_config](variables.tf#L162) | The SSL certificates configuration. | map(object({…})) | | {} | -| [static_ip_config](variables.tf#L172) | Static IP address configuration. | object({…}) | | {…} | -| [target_proxy_https_config](variables.tf#L192) | The HTTPS target proxy configuration. | object({…}) | | null | -| [url_map_config](variables.tf#L200) | The url-map configuration. | object({…}) | | null | +| [forwarding_rule_config](variables.tf#L98) | Forwarding rule configurations. | object({…}) | | {…} | +| [health_checks_config](variables.tf#L118) | Custom health checks configuration. | map(object({…})) | | {} | +| [health_checks_config_defaults](variables.tf#L129) | Auto-created health check default configuration. | object({…}) | | {…} | +| [https](variables.tf#L147) | Whether to enable HTTPS. | bool | | false | +| [network](variables.tf#L153) | The network where the ILB is created. | string | | "default" | +| [ssl_certificates_config](variables.tf#L164) | The SSL certificates configuration. | map(object({…})) | | {} | +| [static_ip_config](variables.tf#L174) | Static IP address configuration. | object({…}) | | {…} | +| [target_proxy_https_config](variables.tf#L194) | The HTTPS target proxy configuration. | object({…}) | | null | +| [url_map_config](variables.tf#L202) | The url-map configuration. | object({…}) | | null | ## Outputs diff --git a/modules/net-ilb-l7/forwarding-rule.tf b/modules/net-ilb-l7/forwarding-rule.tf index 41044060..31ff6525 100644 --- a/modules/net-ilb-l7/forwarding-rule.tf +++ b/modules/net-ilb-l7/forwarding-rule.tf @@ -62,6 +62,7 @@ resource "google_compute_forwarding_rule" "forwarding_rule" { port_range = local.port_range ports = [] region = try(var.region, null) + service_label = try(var.forwarding_rule_config.service_label, null) subnetwork = try(var.subnetwork, null) target = local.target } diff --git a/modules/net-ilb-l7/variables.tf b/modules/net-ilb-l7/variables.tf index 8d9979fd..7202e3fa 100644 --- a/modules/net-ilb-l7/variables.tf +++ b/modules/net-ilb-l7/variables.tf @@ -98,10 +98,11 @@ variable "backend_services_config" { variable "forwarding_rule_config" { description = "Forwarding rule configurations." type = object({ - ip_version = string - labels = map(string) - network_tier = string - port_range = string + ip_version = string + labels = map(string) + network_tier = string + port_range = string + service_label = string }) default = { allow_global_access = true @@ -109,7 +110,8 @@ variable "forwarding_rule_config" { labels = {} network_tier = "PREMIUM" # If not specified, 443 if var.https = true; 80 otherwise - port_range = null + port_range = null + service_label = null } } diff --git a/tests/modules/net_ilb_l7/fixture/variables.tf b/tests/modules/net_ilb_l7/fixture/variables.tf index 0e40c5c0..d513a2a4 100644 --- a/tests/modules/net_ilb_l7/fixture/variables.tf +++ b/tests/modules/net_ilb_l7/fixture/variables.tf @@ -88,10 +88,11 @@ variable "backend_services_config" { variable "forwarding_rule_config" { description = "Forwarding rule configurations." type = object({ - ip_version = string - labels = map(string) - network_tier = string - port_range = string + ip_version = string + labels = map(string) + network_tier = string + port_range = string + service_label = string }) default = { allow_global_access = true @@ -99,7 +100,8 @@ variable "forwarding_rule_config" { labels = {} network_tier = "PREMIUM" # If not specified, 443 if var.https = true; 80 otherwise - port_range = null + port_range = null + service_label = null } } From b755470390149bb9fc05dcee90a76ebe688a293e Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 21 Sep 2022 23:31:30 +0200 Subject: [PATCH 14/29] update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcb1d931..b9b8cae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. ### BLUEPRINTS +- [[#828](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/828)] Update firewall rules. ([lcaggio](https://github.com/lcaggio)) - [[#813](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/813)] Add documentation example test for pf ([ludoo](https://github.com/ludoo)) - [[#809](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/809)] Renaming and moving blueprints ([juliocc](https://github.com/juliocc)) @@ -17,8 +18,17 @@ All notable changes to this project will be documented in this file. ### FAST +- [[#828](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/828)] Update firewall rules. ([lcaggio](https://github.com/lcaggio)) - [[#807](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/807)] FAST: refactor Gitlab template ([ludoo](https://github.com/ludoo)) +### MODULES + +- [[#834](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/834)] Add support for service_label property in internal load balancer ([kmucha555](https://github.com/kmucha555)) +- [[#833](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/833)] regional MySQL DBs - automatic backup conf ([skalolazka](https://github.com/skalolazka)) +- [[#827](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/827)] Project module: Add Artifactregistry Service Identity SA creation. ([lcaggio](https://github.com/lcaggio)) +- [[#826](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/826)] Added new binary_authorization argument in gke-cluster module ([sirohia](https://github.com/sirohia)) +- [[#819](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/819)] Removed old and unused modules ([juliocc](https://github.com/juliocc)) + ### TOOLS - [[#811](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/811)] Fix changelog generator ([ludoo](https://github.com/ludoo)) From c54103cb9ad81d4434a338109b63b9e59722c563 Mon Sep 17 00:00:00 2001 From: Catalin Muresan Date: Thu, 22 Sep 2022 13:06:36 +0100 Subject: [PATCH 15/29] Fix workflow-gitlab.yaml template rendering --- fast/assets/templates/workflow-gitlab.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fast/assets/templates/workflow-gitlab.yaml b/fast/assets/templates/workflow-gitlab.yaml index b685be8a..8981e70b 100644 --- a/fast/assets/templates/workflow-gitlab.yaml +++ b/fast/assets/templates/workflow-gitlab.yaml @@ -14,7 +14,7 @@ default: before_script: - - echo "${CI_JOB_JWT_V2}" > token.txt + - echo "$${CI_JOB_JWT_V2}" > token.txt image: name: hashicorp/terraform entrypoint: @@ -49,10 +49,10 @@ gcp-auth: script: - | gcloud iam workload-identity-pools create-cred-config \ - ${FAST_WIF_PROVIDER} \ - --service-account=${FAST_SERVICE_ACCOUNT} \ + $${FAST_WIF_PROVIDER} \ + --service-account=$${FAST_SERVICE_ACCOUNT} \ --service-account-token-lifetime-seconds=3600 \ - --output-file=${GOOGLE_CREDENTIALS} \ + --output-file=$${GOOGLE_CREDENTIALS} \ --credential-source-file=token.txt tf-files: dependencies: @@ -62,14 +62,14 @@ tf-files: stage: tf-files script: # - gcloud components install -q alpha - - gcloud config set auth/credential_file_override ${GOOGLE_CREDENTIALS} + - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS} - mkdir -p .tf-setup - | gcloud alpha storage cp -r \ - "gs://${FAST_OUTPUTS_BUCKET}/providers/${TF_PROVIDERS_FILE}" .tf-setup/ + "gs://$${FAST_OUTPUTS_BUCKET}/providers/$${TF_PROVIDERS_FILE}" .tf-setup/ - | gcloud alpha storage cp -r \ - "gs://${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ + "gs://$${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ tf-plan: # uncomment the following lines and set the SSH key secret for private modules repo @@ -82,9 +82,9 @@ tf-plan: # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts stage: tf-plan script: - - cp .tf-setup/${TF_PROVIDERS_FILE} ./ + - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - | - for f in ${TF_VAR_FILES}; do + for f in $${TF_VAR_FILES}; do ln -s ".tf-setup/tfvars/$f" ./ done - terraform init @@ -104,9 +104,9 @@ tf-apply: # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts stage: tf-apply script: - - cp .tf-setup/${TF_PROVIDERS_FILE} ./ + - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - | - for f in ${TF_VAR_FILES}; do + for f in $${TF_VAR_FILES}; do ln -s ".tf-setup/tfvars/$f" ./ done - terraform init From a2598991fc725f337d5ffc971b4bcc49c14c73d4 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 27 Sep 2022 12:44:54 +0200 Subject: [PATCH 16/29] Implements review comments. --- blueprints/data-solutions/README.md | 2 +- blueprints/data-solutions/composer-2/README.md | 14 ++++++++++---- blueprints/data-solutions/composer-2/main.tf | 9 ++------- blueprints/data-solutions/composer-2/variables.tf | 4 ++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index 44311b63..819861eb 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -46,5 +46,5 @@ running on a VPC with a private IP and a dedicated Service Account. A GCS bucket -This [blueprint](./composer-2/) creates a [Cloud Composer](https://cloud.google.com/sql) version 2 instance on a VPC with a dedicated service account. The solution supports as inputs: a Shared VPC and Cloud KMS CMEK keys. +This [blueprint](./composer-2/) creates a [Cloud Composer](https://cloud.google.com/composer/) version 2 instance on a VPC with a dedicated service account. The solution supports as inputs: a Shared VPC and Cloud KMS CMEK keys.
\ No newline at end of file diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index 00dbcc0f..4ee62126 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -1,10 +1,10 @@ # Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key -This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. Cloud Composer 2 is the new major verion for Cloud Composer that supports: +This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. Cloud Composer 2 is the new major version for Cloud Composer that supports: - environment autoscaling - workloads configuration: CPU, memory, and storage parameters for Airflow workers, schedulers, web server, and database. -Please consult the [documentation page](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) for an exaustive comparison between Composer Version 1 and Version 2. +Please consult the [documentation page](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) for an exhaustive comparison between Composer Version 1 and Version 2. The solution will use: - Cloud Composer @@ -31,7 +31,7 @@ Run Terraform init: $ terraform init ``` -Configure the Terraform variable in your terraform.tfvars file. You need to spefify at least the following variables: +Configure the Terraform variable in your terraform.tfvars file. You need to specify at least the following variables: ```tfvars project_id = "lcaggioni-sandbox" @@ -48,6 +48,12 @@ You can now connect to your instance. # Customizations +## VPC +If a shared VPC is not configured, a VPC will be created within the project. The following IP ranges will be used: +- Cloudsql: `10.20.10.0/24` +- GKE: `10.20.11.0/28` + +Change the code as needed to match your needed configuration, remember that these addresses should not overlap with any other range used in network. ## Shared VPC As is often the case in real-world configurations, this blueprint accepts as input an existing [`Shared-VPC`](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. @@ -69,7 +75,7 @@ Make sure that: - The subnet has secondary ranges configured with 2 ranges: - pods: `/22` example: `10.10.8.0/22` - services = `/24` example: 10.10.12.0/24` -- Firewall rules are set, as described in the [documentation](https://cloud.google.com/composer/docs/how-to/managing/configuring-private-ip#step_3_configure_firewall_rules) +- Firewall rules are set, as described in the [documentation](https://cloud.google.com/composer/docs/composer-2/configure-private-ip#step_3_configure_firewall_rules) In order to run the example and deploy Cloud Composer on a shared VPC the identity running Terraform must have the following IAM role on the Shared VPC Host project. - Compute Network Admin (roles/compute.networkAdmin) diff --git a/blueprints/data-solutions/composer-2/main.tf b/blueprints/data-solutions/composer-2/main.tf index e8556296..c55113e0 100644 --- a/blueprints/data-solutions/composer-2/main.tf +++ b/blueprints/data-solutions/composer-2/main.tf @@ -23,6 +23,7 @@ locals { var.iam_groups_map ) + # Adding Roles on Service Identities Service account as per documentation: https://cloud.google.com/composer/docs/composer-2/configure-shared-vpc#edit_permissions_for_the_google_apis_service_account _shared_vpc_bindings = { "roles/compute.networkUser" = [ "prj-cloudservices", "prj-robot-gke" @@ -128,13 +129,7 @@ module "vpc" { ] } -module "firewall" { - source = "../../../modules/net-vpc-firewall" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - network = module.vpc.0.name - admin_ranges = ["10.0.0.0/20"] -} +# No explicit firewall rules set, created automatically by GKE autopilot module "nat" { source = "../../../modules/net-cloudnat" diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index c39f3b32..0a57eef2 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -15,7 +15,7 @@ */ variable "composer_config" { - description = "Composer environemnt configuration. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables." + description = "Composer environment configuration. It accepts only following attributes: `environment_size`, `software_config` and `workloads_config`. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables." type = object({ environment_size = string software_config = any @@ -98,7 +98,7 @@ variable "project_id" { } variable "region" { - description = "Region where instances will be deployed." + description = "Reagion where instances will be deployed." type = string default = "europe-west1" } From bf5f30e0bd4ee3784ca430abc9936ba1290cf2ea Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 27 Sep 2022 12:55:26 +0200 Subject: [PATCH 17/29] Fix test. --- tests/blueprints/data_solutions/composer_2/test_plan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/blueprints/data_solutions/composer_2/test_plan.py b/tests/blueprints/data_solutions/composer_2/test_plan.py index 5f69f185..04f4a39f 100644 --- a/tests/blueprints/data_solutions/composer_2/test_plan.py +++ b/tests/blueprints/data_solutions/composer_2/test_plan.py @@ -15,5 +15,5 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner() - assert len(modules) == 5 - assert len(resources) == 29 + assert len(modules) == 4 + assert len(resources) == 25 From 3a2b2d443b35c0d4fa452a2ec170e7b220bfdffa Mon Sep 17 00:00:00 2001 From: lcaggio Date: Tue, 27 Sep 2022 15:32:30 +0200 Subject: [PATCH 18/29] Remove env_variables in the default value. --- blueprints/data-solutions/composer-2/variables.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index 0a57eef2..db7ac55a 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -50,9 +50,6 @@ variable "composer_config" { environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" - env_variables = { - FOO = "bar" - } } workloads_config = null } From ac835b6d50f7889b6f71fcaf0089a7ecceff5154 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 15 Sep 2022 17:04:28 +0200 Subject: [PATCH 19/29] Update terraform version --- blueprints/cloud-operations/adfs/versions.tf | 2 +- .../versions.tf | 2 +- .../dns-fine-grained-iam/versions.tf | 2 +- .../dns-shared-vpc/versions.tf | 2 +- .../iam-delegated-role-grants/versions.tf | 2 +- .../onprem-sa-key-management/versions.tf | 2 +- .../packer-image-builder/versions.tf | 2 +- .../quota-monitoring/versions.tf | 2 +- .../versions.tf | 2 +- .../cmek-via-centralized-kms/versions.tf | 2 +- .../data-playground/versions.tf | 2 +- .../versions.tf | 2 +- .../net-vpc-firewall-yaml/versions.tf | 2 +- .../decentralized-firewall/versions.tf | 2 +- .../networking/filtering-proxy/versions.tf | 2 +- .../hub-and-spoke-peering/versions.tf | 2 +- .../networking/hub-and-spoke-vpn/versions.tf | 2 +- .../networking/ilb-next-hop/versions.tf | 2 +- .../onprem-google-access-dns/versions.tf | 2 +- .../versions.tf | 2 +- .../networking/shared-vpc-gke/versions.tf | 2 +- .../openshift/tf/versions.tf | 2 +- default-versions.tf | 2 +- fast/stages/00-cicd/versions.tf | 2 +- modules/__experimental/net-neg/versions.tf | 2 +- modules/api-gateway/versions.tf | 2 +- modules/apigee-organization/versions.tf | 2 +- modules/apigee-x-instance/versions.tf | 2 +- modules/artifact-registry/versions.tf | 2 +- modules/bigquery-dataset/versions.tf | 2 +- modules/bigtable-instance/versions.tf | 2 +- modules/billing-budget/versions.tf | 2 +- modules/binauthz/versions.tf | 2 +- .../coredns/versions.tf | 2 +- .../cos-generic-metadata/versions.tf | 2 +- .../envoy-traffic-director/versions.tf | 2 +- .../cloud-config-container/mysql/versions.tf | 2 +- .../nginx-tls/versions.tf | 2 +- .../cloud-config-container/nginx/versions.tf | 2 +- .../cloud-config-container/onprem/versions.tf | 2 +- .../cloud-config-container/squid/versions.tf | 2 +- modules/cloud-function/versions.tf | 2 +- modules/cloud-identity-group/versions.tf | 2 +- modules/cloud-run/versions.tf | 2 +- modules/cloudsql-instance/versions.tf | 2 +- modules/compute-mig/versions.tf | 2 +- modules/compute-vm/versions.tf | 2 +- modules/container-registry/versions.tf | 2 +- modules/data-catalog-policy-tag/versions.tf | 2 +- modules/datafusion/versions.tf | 2 +- modules/dns/versions.tf | 2 +- modules/endpoints/versions.tf | 2 +- modules/folder/versions.tf | 2 +- modules/gcs/versions.tf | 2 +- modules/gke-cluster/versions.tf | 2 +- modules/gke-hub/versions.tf | 2 +- modules/gke-nodepool/versions.tf | 2 +- modules/iam-service-account/versions.tf | 2 +- modules/kms/versions.tf | 2 +- modules/logging-bucket/versions.tf | 2 +- modules/net-address/versions.tf | 2 +- modules/net-cloudnat/versions.tf | 2 +- modules/net-glb/versions.tf | 2 +- modules/net-ilb-l7/versions.tf | 2 +- modules/net-ilb/versions.tf | 2 +- .../versions.tf | 2 +- modules/net-vpc-firewall/versions.tf | 2 +- modules/net-vpc-peering/versions.tf | 2 +- modules/net-vpc/versions.tf | 2 +- modules/net-vpn-dynamic/versions.tf | 2 +- modules/net-vpn-ha/versions.tf | 2 +- modules/net-vpn-static/versions.tf | 2 +- modules/organization-policy/experimental.tf | 19 ------------------- modules/organization-policy/versions.tf | 2 +- modules/organization/versions.tf | 2 +- modules/project/versions.tf | 2 +- modules/projects-data-source/versions.tf | 2 +- modules/pubsub/versions.tf | 2 +- modules/secret-manager/versions.tf | 2 +- modules/service-directory/versions.tf | 2 +- modules/source-repository/versions.tf | 2 +- modules/vpc-sc/versions.tf | 2 +- .../fixture/experimental.tf | 18 ------------------ 83 files changed, 81 insertions(+), 118 deletions(-) delete mode 100644 modules/organization-policy/experimental.tf delete mode 100644 tests/modules/organization_policy/fixture/experimental.tf diff --git a/blueprints/cloud-operations/adfs/versions.tf b/blueprints/cloud-operations/adfs/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/adfs/versions.tf +++ b/blueprints/cloud-operations/adfs/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/asset-inventory-feed-remediation/versions.tf b/blueprints/cloud-operations/asset-inventory-feed-remediation/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/asset-inventory-feed-remediation/versions.tf +++ b/blueprints/cloud-operations/asset-inventory-feed-remediation/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/dns-fine-grained-iam/versions.tf b/blueprints/cloud-operations/dns-fine-grained-iam/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/dns-fine-grained-iam/versions.tf +++ b/blueprints/cloud-operations/dns-fine-grained-iam/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/dns-shared-vpc/versions.tf b/blueprints/cloud-operations/dns-shared-vpc/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/dns-shared-vpc/versions.tf +++ b/blueprints/cloud-operations/dns-shared-vpc/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/iam-delegated-role-grants/versions.tf b/blueprints/cloud-operations/iam-delegated-role-grants/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/iam-delegated-role-grants/versions.tf +++ b/blueprints/cloud-operations/iam-delegated-role-grants/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/onprem-sa-key-management/versions.tf b/blueprints/cloud-operations/onprem-sa-key-management/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/onprem-sa-key-management/versions.tf +++ b/blueprints/cloud-operations/onprem-sa-key-management/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/packer-image-builder/versions.tf b/blueprints/cloud-operations/packer-image-builder/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/packer-image-builder/versions.tf +++ b/blueprints/cloud-operations/packer-image-builder/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/quota-monitoring/versions.tf b/blueprints/cloud-operations/quota-monitoring/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/quota-monitoring/versions.tf +++ b/blueprints/cloud-operations/quota-monitoring/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/versions.tf b/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/versions.tf +++ b/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf b/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf +++ b/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/data-solutions/data-playground/versions.tf b/blueprints/data-solutions/data-playground/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/data-solutions/data-playground/versions.tf +++ b/blueprints/data-solutions/data-playground/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/data-solutions/gcs-to-bq-with-least-privileges/versions.tf b/blueprints/data-solutions/gcs-to-bq-with-least-privileges/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/data-solutions/gcs-to-bq-with-least-privileges/versions.tf +++ b/blueprints/data-solutions/gcs-to-bq-with-least-privileges/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/factories/net-vpc-firewall-yaml/versions.tf b/blueprints/factories/net-vpc-firewall-yaml/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/factories/net-vpc-firewall-yaml/versions.tf +++ b/blueprints/factories/net-vpc-firewall-yaml/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/decentralized-firewall/versions.tf b/blueprints/networking/decentralized-firewall/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/decentralized-firewall/versions.tf +++ b/blueprints/networking/decentralized-firewall/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/filtering-proxy/versions.tf b/blueprints/networking/filtering-proxy/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/filtering-proxy/versions.tf +++ b/blueprints/networking/filtering-proxy/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/hub-and-spoke-peering/versions.tf b/blueprints/networking/hub-and-spoke-peering/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/hub-and-spoke-peering/versions.tf +++ b/blueprints/networking/hub-and-spoke-peering/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/hub-and-spoke-vpn/versions.tf b/blueprints/networking/hub-and-spoke-vpn/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/hub-and-spoke-vpn/versions.tf +++ b/blueprints/networking/hub-and-spoke-vpn/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/ilb-next-hop/versions.tf b/blueprints/networking/ilb-next-hop/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/ilb-next-hop/versions.tf +++ b/blueprints/networking/ilb-next-hop/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/onprem-google-access-dns/versions.tf b/blueprints/networking/onprem-google-access-dns/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/onprem-google-access-dns/versions.tf +++ b/blueprints/networking/onprem-google-access-dns/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/private-cloud-function-from-onprem/versions.tf b/blueprints/networking/private-cloud-function-from-onprem/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/private-cloud-function-from-onprem/versions.tf +++ b/blueprints/networking/private-cloud-function-from-onprem/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/networking/shared-vpc-gke/versions.tf b/blueprints/networking/shared-vpc-gke/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/networking/shared-vpc-gke/versions.tf +++ b/blueprints/networking/shared-vpc-gke/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/blueprints/third-party-solutions/openshift/tf/versions.tf b/blueprints/third-party-solutions/openshift/tf/versions.tf index 4278054b..8abac788 100644 --- a/blueprints/third-party-solutions/openshift/tf/versions.tf +++ b/blueprints/third-party-solutions/openshift/tf/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/default-versions.tf b/default-versions.tf index 4278054b..8abac788 100644 --- a/default-versions.tf +++ b/default-versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/fast/stages/00-cicd/versions.tf b/fast/stages/00-cicd/versions.tf index 4278054b..8abac788 100644 --- a/fast/stages/00-cicd/versions.tf +++ b/fast/stages/00-cicd/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/__experimental/net-neg/versions.tf b/modules/__experimental/net-neg/versions.tf index 4278054b..8abac788 100644 --- a/modules/__experimental/net-neg/versions.tf +++ b/modules/__experimental/net-neg/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/api-gateway/versions.tf b/modules/api-gateway/versions.tf index 4278054b..8abac788 100644 --- a/modules/api-gateway/versions.tf +++ b/modules/api-gateway/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/apigee-organization/versions.tf b/modules/apigee-organization/versions.tf index 4278054b..8abac788 100644 --- a/modules/apigee-organization/versions.tf +++ b/modules/apigee-organization/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/apigee-x-instance/versions.tf b/modules/apigee-x-instance/versions.tf index 4278054b..8abac788 100644 --- a/modules/apigee-x-instance/versions.tf +++ b/modules/apigee-x-instance/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/artifact-registry/versions.tf b/modules/artifact-registry/versions.tf index 4278054b..8abac788 100644 --- a/modules/artifact-registry/versions.tf +++ b/modules/artifact-registry/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/bigquery-dataset/versions.tf b/modules/bigquery-dataset/versions.tf index 4278054b..8abac788 100644 --- a/modules/bigquery-dataset/versions.tf +++ b/modules/bigquery-dataset/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/bigtable-instance/versions.tf b/modules/bigtable-instance/versions.tf index 4278054b..8abac788 100644 --- a/modules/bigtable-instance/versions.tf +++ b/modules/bigtable-instance/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/billing-budget/versions.tf b/modules/billing-budget/versions.tf index 4278054b..8abac788 100644 --- a/modules/billing-budget/versions.tf +++ b/modules/billing-budget/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/binauthz/versions.tf b/modules/binauthz/versions.tf index 4278054b..8abac788 100644 --- a/modules/binauthz/versions.tf +++ b/modules/binauthz/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/coredns/versions.tf b/modules/cloud-config-container/coredns/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/coredns/versions.tf +++ b/modules/cloud-config-container/coredns/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/cos-generic-metadata/versions.tf b/modules/cloud-config-container/cos-generic-metadata/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/cos-generic-metadata/versions.tf +++ b/modules/cloud-config-container/cos-generic-metadata/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/envoy-traffic-director/versions.tf b/modules/cloud-config-container/envoy-traffic-director/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/envoy-traffic-director/versions.tf +++ b/modules/cloud-config-container/envoy-traffic-director/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/mysql/versions.tf b/modules/cloud-config-container/mysql/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/mysql/versions.tf +++ b/modules/cloud-config-container/mysql/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/nginx-tls/versions.tf b/modules/cloud-config-container/nginx-tls/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/nginx-tls/versions.tf +++ b/modules/cloud-config-container/nginx-tls/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/nginx/versions.tf b/modules/cloud-config-container/nginx/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/nginx/versions.tf +++ b/modules/cloud-config-container/nginx/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/onprem/versions.tf b/modules/cloud-config-container/onprem/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/onprem/versions.tf +++ b/modules/cloud-config-container/onprem/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-config-container/squid/versions.tf b/modules/cloud-config-container/squid/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-config-container/squid/versions.tf +++ b/modules/cloud-config-container/squid/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-function/versions.tf b/modules/cloud-function/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-function/versions.tf +++ b/modules/cloud-function/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-identity-group/versions.tf b/modules/cloud-identity-group/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-identity-group/versions.tf +++ b/modules/cloud-identity-group/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloud-run/versions.tf b/modules/cloud-run/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloud-run/versions.tf +++ b/modules/cloud-run/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/cloudsql-instance/versions.tf b/modules/cloudsql-instance/versions.tf index 4278054b..8abac788 100644 --- a/modules/cloudsql-instance/versions.tf +++ b/modules/cloudsql-instance/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/compute-mig/versions.tf b/modules/compute-mig/versions.tf index 4278054b..8abac788 100644 --- a/modules/compute-mig/versions.tf +++ b/modules/compute-mig/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/compute-vm/versions.tf b/modules/compute-vm/versions.tf index 4278054b..8abac788 100644 --- a/modules/compute-vm/versions.tf +++ b/modules/compute-vm/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/container-registry/versions.tf b/modules/container-registry/versions.tf index 4278054b..8abac788 100644 --- a/modules/container-registry/versions.tf +++ b/modules/container-registry/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/data-catalog-policy-tag/versions.tf b/modules/data-catalog-policy-tag/versions.tf index 4278054b..8abac788 100644 --- a/modules/data-catalog-policy-tag/versions.tf +++ b/modules/data-catalog-policy-tag/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/datafusion/versions.tf b/modules/datafusion/versions.tf index 4278054b..8abac788 100644 --- a/modules/datafusion/versions.tf +++ b/modules/datafusion/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/dns/versions.tf b/modules/dns/versions.tf index 4278054b..8abac788 100644 --- a/modules/dns/versions.tf +++ b/modules/dns/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/endpoints/versions.tf b/modules/endpoints/versions.tf index 4278054b..8abac788 100644 --- a/modules/endpoints/versions.tf +++ b/modules/endpoints/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/folder/versions.tf b/modules/folder/versions.tf index 4278054b..8abac788 100644 --- a/modules/folder/versions.tf +++ b/modules/folder/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/gcs/versions.tf b/modules/gcs/versions.tf index 4278054b..8abac788 100644 --- a/modules/gcs/versions.tf +++ b/modules/gcs/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/gke-cluster/versions.tf b/modules/gke-cluster/versions.tf index 4278054b..8abac788 100644 --- a/modules/gke-cluster/versions.tf +++ b/modules/gke-cluster/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/gke-hub/versions.tf b/modules/gke-hub/versions.tf index 4278054b..8abac788 100644 --- a/modules/gke-hub/versions.tf +++ b/modules/gke-hub/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/gke-nodepool/versions.tf b/modules/gke-nodepool/versions.tf index 4278054b..8abac788 100644 --- a/modules/gke-nodepool/versions.tf +++ b/modules/gke-nodepool/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/iam-service-account/versions.tf b/modules/iam-service-account/versions.tf index 4278054b..8abac788 100644 --- a/modules/iam-service-account/versions.tf +++ b/modules/iam-service-account/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/kms/versions.tf b/modules/kms/versions.tf index 4278054b..8abac788 100644 --- a/modules/kms/versions.tf +++ b/modules/kms/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/logging-bucket/versions.tf b/modules/logging-bucket/versions.tf index 4278054b..8abac788 100644 --- a/modules/logging-bucket/versions.tf +++ b/modules/logging-bucket/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-address/versions.tf b/modules/net-address/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-address/versions.tf +++ b/modules/net-address/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-cloudnat/versions.tf b/modules/net-cloudnat/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-cloudnat/versions.tf +++ b/modules/net-cloudnat/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-glb/versions.tf b/modules/net-glb/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-glb/versions.tf +++ b/modules/net-glb/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-ilb-l7/versions.tf b/modules/net-ilb-l7/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-ilb-l7/versions.tf +++ b/modules/net-ilb-l7/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-ilb/versions.tf b/modules/net-ilb/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-ilb/versions.tf +++ b/modules/net-ilb/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-interconnect-attachment-direct/versions.tf b/modules/net-interconnect-attachment-direct/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-interconnect-attachment-direct/versions.tf +++ b/modules/net-interconnect-attachment-direct/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-vpc-firewall/versions.tf b/modules/net-vpc-firewall/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-vpc-firewall/versions.tf +++ b/modules/net-vpc-firewall/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-vpc-peering/versions.tf b/modules/net-vpc-peering/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-vpc-peering/versions.tf +++ b/modules/net-vpc-peering/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-vpc/versions.tf +++ b/modules/net-vpc/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-vpn-dynamic/versions.tf b/modules/net-vpn-dynamic/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-vpn-dynamic/versions.tf +++ b/modules/net-vpn-dynamic/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-vpn-ha/versions.tf b/modules/net-vpn-ha/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-vpn-ha/versions.tf +++ b/modules/net-vpn-ha/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/net-vpn-static/versions.tf b/modules/net-vpn-static/versions.tf index 4278054b..8abac788 100644 --- a/modules/net-vpn-static/versions.tf +++ b/modules/net-vpn-static/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/organization-policy/experimental.tf b/modules/organization-policy/experimental.tf deleted file mode 100644 index 5fc3fc44..00000000 --- a/modules/organization-policy/experimental.tf +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -terraform { - # TODO: Remove once Terraform 1.3 is released https://github.com/hashicorp/terraform/releases/tag/v1.3.0-alpha20220622 - experiments = [module_variable_optional_attrs] -} diff --git a/modules/organization-policy/versions.tf b/modules/organization-policy/versions.tf index 4278054b..8abac788 100644 --- a/modules/organization-policy/versions.tf +++ b/modules/organization-policy/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/organization/versions.tf b/modules/organization/versions.tf index 4278054b..8abac788 100644 --- a/modules/organization/versions.tf +++ b/modules/organization/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/project/versions.tf b/modules/project/versions.tf index 4278054b..8abac788 100644 --- a/modules/project/versions.tf +++ b/modules/project/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/projects-data-source/versions.tf b/modules/projects-data-source/versions.tf index 4278054b..8abac788 100644 --- a/modules/projects-data-source/versions.tf +++ b/modules/projects-data-source/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/pubsub/versions.tf b/modules/pubsub/versions.tf index 4278054b..8abac788 100644 --- a/modules/pubsub/versions.tf +++ b/modules/pubsub/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/secret-manager/versions.tf b/modules/secret-manager/versions.tf index 4278054b..8abac788 100644 --- a/modules/secret-manager/versions.tf +++ b/modules/secret-manager/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/service-directory/versions.tf b/modules/service-directory/versions.tf index 4278054b..8abac788 100644 --- a/modules/service-directory/versions.tf +++ b/modules/service-directory/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/source-repository/versions.tf b/modules/source-repository/versions.tf index 4278054b..8abac788 100644 --- a/modules/source-repository/versions.tf +++ b/modules/source-repository/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/modules/vpc-sc/versions.tf b/modules/vpc-sc/versions.tf index 4278054b..8abac788 100644 --- a/modules/vpc-sc/versions.tf +++ b/modules/vpc-sc/versions.tf @@ -13,7 +13,7 @@ # limitations under the License. terraform { - required_version = ">= 1.1.0" + required_version = ">= 1.3.0" required_providers { google = { source = "hashicorp/google" diff --git a/tests/modules/organization_policy/fixture/experimental.tf b/tests/modules/organization_policy/fixture/experimental.tf deleted file mode 100644 index 6ed0c934..00000000 --- a/tests/modules/organization_policy/fixture/experimental.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - # TODO: Remove once Terraform 1.3 is released https://github.com/hashicorp/terraform/releases/tag/v1.3.0-alpha20220622 - experiments = [module_variable_optional_attrs] -} From 040c9284e69f6089ebe1671fe13af12a311990e8 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 28 Sep 2022 11:50:23 +0200 Subject: [PATCH 20/29] Fix tf version in pipelines --- .github/workflows/linting.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 5de7336d..33693873 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Terraform uses: hashicorp/setup-terraform@v1 with: - terraform_version: 1.1.8 + terraform_version: 1.3 - name: Install dependencies run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8e4d144..a7e427d7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,7 +30,7 @@ env: PYTEST_ADDOPTS: "--color=yes" PYTHON_VERSION: "3.10" TF_PLUGIN_CACHE_DIR: "/home/runner/.terraform.d/plugin-cache" - TF_VERSION: 1.1.8 + TF_VERSION: 1.3.0 jobs: doc-examples: From b983ae46f581717be31040cb1f1f3ea57a628433 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 28 Sep 2022 13:27:02 +0200 Subject: [PATCH 21/29] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b8cae4..ff0fc85b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. ### BLUEPRINTS +- [[#839](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/839)] **incompatible change:** Update to terraform 1.3 ([juliocc](https://github.com/juliocc)) - [[#828](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/828)] Update firewall rules. ([lcaggio](https://github.com/lcaggio)) - [[#813](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/813)] Add documentation example test for pf ([ludoo](https://github.com/ludoo)) - [[#809](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/809)] Renaming and moving blueprints ([juliocc](https://github.com/juliocc)) @@ -18,11 +19,14 @@ All notable changes to this project will be documented in this file. ### FAST +- [[#835](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/835)] Fix workflow-gitlab.yaml template rendering ([muresan](https://github.com/muresan)) - [[#828](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/828)] Update firewall rules. ([lcaggio](https://github.com/lcaggio)) - [[#807](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/807)] FAST: refactor Gitlab template ([ludoo](https://github.com/ludoo)) ### MODULES +- [[#839](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/839)] **incompatible change:** Update to terraform 1.3 ([juliocc](https://github.com/juliocc)) +- [[#824](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/824)] Add simple composer 2 blueprint ([lcaggio](https://github.com/lcaggio)) - [[#834](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/834)] Add support for service_label property in internal load balancer ([kmucha555](https://github.com/kmucha555)) - [[#833](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/833)] regional MySQL DBs - automatic backup conf ([skalolazka](https://github.com/skalolazka)) - [[#827](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/827)] Project module: Add Artifactregistry Service Identity SA creation. ([lcaggio](https://github.com/lcaggio)) From fcf71b983e5b2f01576ac5f39901637977db018c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 28 Sep 2022 14:10:05 +0200 Subject: [PATCH 22/29] refactor net-address modules for 1.3 (#840) --- modules/net-address/README.md | 19 ++---- modules/net-address/main.tf | 8 +-- modules/net-address/variables.tf | 14 ++--- tests/modules/net_address/fixture/main.tf | 13 ++-- .../modules/net_address/fixture/variables.tf | 24 ++------ tests/modules/net_address/test_plan.py | 60 +++++++++---------- 6 files changed, 52 insertions(+), 86 deletions(-) diff --git a/modules/net-address/README.md b/modules/net-address/README.md index 76637ccd..e5138348 100644 --- a/modules/net-address/README.md +++ b/modules/net-address/README.md @@ -27,22 +27,16 @@ module "addresses" { project_id = var.project_id internal_addresses = { ilb-1 = { + purpose = "SHARED_LOADBALANCER_VIP" region = var.region subnetwork = var.subnet.self_link } ilb-2 = { + address = "10.0.0.2" region = var.region subnetwork = var.subnet.self_link } } - # optional configuration - internal_addresses_config = { - ilb-1 = { - address = null - purpose = "SHARED_LOADBALANCER_VIP" - tier = null - } - } } # tftest modules=1 resources=2 ``` @@ -89,13 +83,12 @@ module "addresses" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [project_id](variables.tf#L60) | Project where the addresses will be created. | string | ✓ | | +| [project_id](variables.tf#L54) | Project where the addresses will be created. | string | ✓ | | | [external_addresses](variables.tf#L17) | Map of external address regions, keyed by name. | map(string) | | {} | | [global_addresses](variables.tf#L29) | List of global addresses to create. | list(string) | | [] | -| [internal_addresses](variables.tf#L35) | Map of internal addresses to create, keyed by name. | map(object({…})) | | {} | -| [internal_addresses_config](variables.tf#L44) | Optional configuration for internal addresses, keyed by name. Unused options can be set to null. | map(object({…})) | | {} | -| [psa_addresses](variables.tf#L65) | Map of internal addresses used for Private Service Access. | map(object({…})) | | {} | -| [psc_addresses](variables.tf#L75) | Map of internal addresses used for Private Service Connect. | map(object({…})) | | {} | +| [internal_addresses](variables.tf#L35) | Map of internal addresses to create, keyed by name. | map(object({…})) | | {} | +| [psa_addresses](variables.tf#L59) | Map of internal addresses used for Private Service Access. | map(object({…})) | | {} | +| [psc_addresses](variables.tf#L69) | Map of internal addresses used for Private Service Connect. | map(object({…})) | | {} | ## Outputs diff --git a/modules/net-address/main.tf b/modules/net-address/main.tf index caab92a0..8619f95b 100644 --- a/modules/net-address/main.tf +++ b/modules/net-address/main.tf @@ -39,10 +39,10 @@ resource "google_compute_address" "internal" { address_type = "INTERNAL" region = each.value.region subnetwork = each.value.subnetwork - address = try(var.internal_addresses_config[each.key].address, null) - network_tier = try(var.internal_addresses_config[each.key].tier, null) - purpose = try(var.internal_addresses_config[each.key].purpose, null) - # labels = lookup(var.internal_address_labels, each.key, {}) + address = each.value.address + network_tier = each.value.tier + purpose = each.value.purpose + labels = coalesce(each.value.labels, {}) } resource "google_compute_global_address" "psc" { diff --git a/modules/net-address/variables.tf b/modules/net-address/variables.tf index bb3043dc..35093e83 100644 --- a/modules/net-address/variables.tf +++ b/modules/net-address/variables.tf @@ -37,16 +37,10 @@ variable "internal_addresses" { type = map(object({ region = string subnetwork = string - })) - default = {} -} - -variable "internal_addresses_config" { - description = "Optional configuration for internal addresses, keyed by name. Unused options can be set to null." - type = map(object({ - address = string - purpose = string - tier = string + address = optional(string) + labels = optional(map(string)) + purpose = optional(string) + tier = optional(string) })) default = {} } diff --git a/tests/modules/net_address/fixture/main.tf b/tests/modules/net_address/fixture/main.tf index e716ad1a..4d41b096 100644 --- a/tests/modules/net_address/fixture/main.tf +++ b/tests/modules/net_address/fixture/main.tf @@ -15,11 +15,10 @@ */ module "test" { - source = "../../../../modules/net-address" - external_addresses = var.external_addresses - global_addresses = var.global_addresses - internal_addresses = var.internal_addresses - internal_addresses_config = var.internal_addresses_config - psa_addresses = var.psa_addresses - project_id = var.project_id + source = "../../../../modules/net-address" + external_addresses = var.external_addresses + global_addresses = var.global_addresses + internal_addresses = var.internal_addresses + psa_addresses = var.psa_addresses + project_id = var.project_id } diff --git a/tests/modules/net_address/fixture/variables.tf b/tests/modules/net_address/fixture/variables.tf index d8272803..ebc90814 100644 --- a/tests/modules/net_address/fixture/variables.tf +++ b/tests/modules/net_address/fixture/variables.tf @@ -15,29 +15,17 @@ */ variable "external_addresses" { - type = map(string) + type = any default = {} } variable "global_addresses" { - type = list(string) + type = any default = [] } variable "internal_addresses" { - type = map(object({ - region = string - subnetwork = string - })) - default = {} -} - -variable "internal_addresses_config" { - type = map(object({ - address = string - purpose = string - tier = string - })) + type = any default = {} } @@ -47,10 +35,6 @@ variable "project_id" { } variable "psa_addresses" { - type = map(object({ - address = string - network = string - prefix_length = number - })) + type = any default = {} } diff --git a/tests/modules/net_address/test_plan.py b/tests/modules/net_address/test_plan.py index 4aa03d27..2274da6f 100644 --- a/tests/modules/net_address/test_plan.py +++ b/tests/modules/net_address/test_plan.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. + def test_external_addresses(plan_runner): addresses = '{one = "europe-west1", two = "europe-west2"}' _, resources = plan_runner(external_addresses=addresses) assert [r['values']['name'] for r in resources] == ['one', 'two'] - assert set(r['values']['address_type'] - for r in resources) == set(['EXTERNAL']) - assert [r['values']['region'] - for r in resources] == ['europe-west1', 'europe-west2'] + assert set(r['values']['address_type'] for r in resources) == set( + ['EXTERNAL']) + assert [r['values']['region'] for r in resources + ] == ['europe-west1', 'europe-west2'] def test_global_addresses(plan_runner): @@ -29,42 +30,37 @@ def test_global_addresses(plan_runner): def test_internal_addresses(plan_runner): - addresses = ( - '{one = {region = "europe-west1", subnetwork = "foobar"}, ' - 'two = {region = "europe-west2", subnetwork = "foobarz"}}' - ) + addresses = ('{one = {region = "europe-west1", subnetwork = "foobar"}, ' + 'two = {region = "europe-west2", subnetwork = "foobarz"}}') _, resources = plan_runner(internal_addresses=addresses) assert [r['values']['name'] for r in resources] == ['one', 'two'] - assert set(r['values']['address_type'] - for r in resources) == set(['INTERNAL']) - assert [r['values']['region'] - for r in resources] == ['europe-west1', 'europe-west2'] + assert set(r['values']['address_type'] for r in resources) == set( + ['INTERNAL']) + assert [r['values']['region'] for r in resources + ] == ['europe-west1', 'europe-west2'] def test_internal_addresses_config(plan_runner): - addresses = ( - '{one = {region = "europe-west1", subnetwork = "foobar"}, ' - 'two = {region = "europe-west2", subnetwork = "foobarz"}}' - ) - config = ( - '{one = {address = "10.0.0.2", purpose = "SHARED_LOADBALANCER_VIP", ' - 'tier=null}}' - ) - _, resources = plan_runner(internal_addresses=addresses, - internal_addresses_config=config) + addresses = '''{ + one = { + region = "europe-west1" + subnetwork = "foobar" + address = "10.0.0.2" + purpose = "SHARED_LOADBALANCER_VIP" + }, + two = {region = "europe-west2", subnetwork = "foobarz"} + }''' + _, resources = plan_runner(internal_addresses=addresses) assert [r['values']['name'] for r in resources] == ['one', 'two'] - assert set(r['values']['address_type'] - for r in resources) == set(['INTERNAL']) - assert [r['values'].get('address') - for r in resources] == ['10.0.0.2', None] - assert [r['values'].get('purpose') - for r in resources] == ['SHARED_LOADBALANCER_VIP', None] + assert set(r['values']['address_type'] for r in resources) == set( + ['INTERNAL']) + assert [r['values'].get('address') for r in resources] == ['10.0.0.2', None] + assert [r['values'].get('purpose') for r in resources + ] == ['SHARED_LOADBALANCER_VIP', None] def test_psa_config(plan_runner): psa_addresses = '{cloudsql-mysql={address="10.199.0.0", network="foobar", prefix_length = 24}}' _, resources = plan_runner(psa_addresses=psa_addresses) - assert set(r['values']['purpose'] - for r in resources) == set(['VPC_PEERING']) - assert set(r['values']['address'] - for r in resources) == set(['10.199.0.0']) + assert set(r['values']['purpose'] for r in resources) == set(['VPC_PEERING']) + assert set(r['values']['address'] for r in resources) == set(['10.199.0.0']) From accafd9ab95d44a1724bfb3d31498f49b2e462da Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 28 Sep 2022 14:10:51 +0200 Subject: [PATCH 23/29] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff0fc85b..619bb9c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. ### MODULES +- [[#840](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/840)] Refactor net-address module for 1.3 ([ludoo](https://github.com/ludoo)) - [[#839](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/839)] **incompatible change:** Update to terraform 1.3 ([juliocc](https://github.com/juliocc)) - [[#824](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/824)] Add simple composer 2 blueprint ([lcaggio](https://github.com/lcaggio)) - [[#834](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/834)] Add support for service_label property in internal load balancer ([kmucha555](https://github.com/kmucha555)) From f2eb424d559ed9eda9f53d31f5754f25f9e65ada Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 28 Sep 2022 14:27:17 +0200 Subject: [PATCH 24/29] FAST: revert 00-cicd provider changes --- fast/stages/00-cicd/versions.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fast/stages/00-cicd/versions.tf b/fast/stages/00-cicd/versions.tf index 8abac788..3a6a1ed8 100644 --- a/fast/stages/00-cicd/versions.tf +++ b/fast/stages/00-cicd/versions.tf @@ -24,6 +24,14 @@ terraform { version = ">= 4.32.0" # tftest } } + github = { + source = "integrations/github" + version = "~> 4.0" + } + gitlab = { + source = "gitlabhq/gitlab" + version = ">= 3.16.1" + } } From 4a1465ae323451fc6392ed1977a14f05ff28dfe7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 29 Sep 2022 08:30:01 +0200 Subject: [PATCH 25/29] Comment redundant role in bootstrap stage, align IAM.md files, improve IAM tool (#842) * comment redundant role * account for duplicate folder names * update IAM.md files --- fast/stages/00-bootstrap/IAM.md | 4 +- fast/stages/00-bootstrap/organization.tf | 7 +- fast/stages/01-resman/IAM.md | 92 ++++++++++++++++++++++-- tools/state_iam.py | 16 ++++- 4 files changed, 106 insertions(+), 13 deletions(-) diff --git a/fast/stages/00-bootstrap/IAM.md b/fast/stages/00-bootstrap/IAM.md index 8ed7a126..7e7d7c19 100644 --- a/fast/stages/00-bootstrap/IAM.md +++ b/fast/stages/00-bootstrap/IAM.md @@ -6,13 +6,13 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|GCP organization domain
domain|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser)
[roles/resourcemanager.organizationViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationViewer) | +|GCP organization domain
domain|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser) | |gcp-billing-admins
group|[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +| |gcp-network-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) +
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) +| |gcp-organization-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.admin)
[roles/compute.osAdminLogin](https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin)
[roles/compute.osLoginExternalUser](https://cloud.google.com/iam/docs/understanding-roles#compute.osLoginExternalUser)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +| |gcp-security-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/iam.securityReviewer](https://cloud.google.com/iam/docs/understanding-roles#iam.securityReviewer)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/securitycenter.admin](https://cloud.google.com/iam/docs/understanding-roles#securitycenter.admin)
[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) +
[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +| |gcp-support
group|[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/logging.viewer](https://cloud.google.com/iam/docs/understanding-roles#logging.viewer)
[roles/monitoring.viewer](https://cloud.google.com/iam/docs/understanding-roles#monitoring.viewer) | -|prod-bootstrap-0
serviceAccount|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) +| +|prod-bootstrap-0
serviceAccount|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/resourcemanager.projectMover](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectMover)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) +| |prod-resman-0
serviceAccount|organizations/[org_id #0]/roles/organizationIamAdmin •
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.tagAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagAdmin)
[roles/resourcemanager.tagUser](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.tagUser)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +| ## Project prod-audit-logs-0 diff --git a/fast/stages/00-bootstrap/organization.tf b/fast/stages/00-bootstrap/organization.tf index 2aa21f8d..39845012 100644 --- a/fast/stages/00-bootstrap/organization.tf +++ b/fast/stages/00-bootstrap/organization.tf @@ -34,9 +34,10 @@ locals { [module.automation-tf-bootstrap-sa.iam_email], local._iam_bootstrap_user ) - "roles/resourcemanager.organizationViewer" = [ - "domain:${var.organization.domain}" - ] + # the following is useful if roles/browser is not desirable + # "roles/resourcemanager.organizationViewer" = [ + # "domain:${var.organization.domain}" + # ] "roles/resourcemanager.projectCreator" = concat( [module.automation-tf-bootstrap-sa.iam_email], local._iam_bootstrap_user diff --git a/fast/stages/01-resman/IAM.md b/fast/stages/01-resman/IAM.md index cbd989d9..78fac484 100644 --- a/fast/stages/01-resman/IAM.md +++ b/fast/stages/01-resman/IAM.md @@ -7,18 +7,45 @@ Legend: + additive, • conditional. | members | roles | |---|---| |dev-resman-dp-0
serviceAccount|[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +•
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| +|dev-resman-gke-0
serviceAccount|[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| |dev-resman-pf-0
serviceAccount|[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +•
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| |prod-resman-dp-0
serviceAccount|[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +•
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| +|prod-resman-gke-0
serviceAccount|[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| |prod-resman-net-0
serviceAccount|[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +
[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) +
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) +| |prod-resman-pf-0
serviceAccount|[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +•
[roles/billing.costsManager](https://cloud.google.com/iam/docs/understanding-roles#billing.costsManager) +
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| |prod-resman-sec-0
serviceAccount|[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) +
[roles/billing.user](https://cloud.google.com/iam/docs/understanding-roles#billing.user) +| -## Folder development +## Folder development [#0] | members | roles | |---|---| -|dev-resman-dp-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) | -|dev-resman-pf-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) | +|dev-resman-dp-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder development [#1] + +| members | roles | +|---|---| +|dev-resman-gke-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder development [#2] + +| members | roles | +|---|---| +|dev-resman-dp-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin | +|dev-resman-gke-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin | +|dev-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin | + +## Folder development [#3] + +| members | roles | +|---|---| +|dev-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder development [#4] + +| members | roles | +|---|---| +|dev-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | ## Folder networking @@ -27,12 +54,37 @@ Legend: + additive, • conditional. |gcp-network-admins
group|[roles/editor](https://cloud.google.com/iam/docs/understanding-roles#editor) | |prod-resman-net-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | -## Folder production +## Folder production [#0] | members | roles | |---|---| -|prod-resman-dp-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) | -|prod-resman-pf-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) | +|prod-resman-dp-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder production [#1] + +| members | roles | +|---|---| +|prod-resman-gke-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder production [#2] + +| members | roles | +|---|---| +|prod-resman-dp-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin | +|prod-resman-gke-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin | +|prod-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin | + +## Folder production [#3] + +| members | roles | +|---|---| +|prod-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder production [#4] + +| members | roles | +|---|---| +|prod-resman-pf-0
serviceAccount|organizations/[org_id #0]/roles/serviceProjectNetworkAdmin
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | ## Folder sandbox @@ -46,3 +98,31 @@ Legend: + additive, • conditional. |---|---| |gcp-security-admins
group|[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | |prod-resman-sec-0
serviceAccount|[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder team a + +| members | roles | +|---|---| +|team-a
group|[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | +|prod-teams-team-a-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder team b + +| members | roles | +|---|---| +|prod-teams-team-b-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Folder teams + +| members | roles | +|---|---| +|prod-resman-teams-0
serviceAccount|[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator) | + +## Project prod-iac-core-0 + +| members | roles | +|---|---| +|dev-resman-dp-1
serviceAccount|[roles/logging.logWriter](https://cloud.google.com/iam/docs/understanding-roles#logging.logWriter) +| +|dev-resman-gke-1
serviceAccount|[roles/logging.logWriter](https://cloud.google.com/iam/docs/understanding-roles#logging.logWriter) +| +|prod-resman-gke-1
serviceAccount|[roles/logging.logWriter](https://cloud.google.com/iam/docs/understanding-roles#logging.logWriter) +| +|prod-resman-net-1
serviceAccount|[roles/logging.logWriter](https://cloud.google.com/iam/docs/understanding-roles#logging.logWriter) +| diff --git a/tools/state_iam.py b/tools/state_iam.py index a428f782..1ea887f8 100755 --- a/tools/state_iam.py +++ b/tools/state_iam.py @@ -92,11 +92,22 @@ def get_bindings(resources, prefix=None, folders=None): def get_folders(resources): 'Parse resources and return folder id, name tuples.' + folders = {} for r in resources: if r['type'] != 'google_folder': continue for i in r['instances']: - yield i['attributes']['id'], i['attributes']['display_name'] + folder_id = i['attributes']['id'] + folder_name = i['attributes']['display_name'] + if folder_name not in folders: + folders[folder_name] = [] + folders[folder_name].append(folder_id) + for name, ids in folders.items(): + for i, folder_id in enumerate(ids): + if len(ids) == 1: + yield folder_id, name + else: + yield folder_id, f'{name} [#{i}]' def output_csv(bindings): @@ -113,7 +124,8 @@ def output_principals(bindings): print('# IAM bindings reference') print('\nLegend: + additive, • conditional.') for resource, resource_groups in resource_grouper: - print(f'\n## {resource[0].title()} {resource[1].lower()}\n') + resource_type, resource_name = resource + print(f'\n## {resource_type.title()} {resource_name.lower()}\n') principal_grouper = itertools.groupby( resource_groups, key=lambda b: (b.member_type, b.member_id)) print('| members | roles |') From 1842586241927fcecf36e2270a8885e72b9fdf3b Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 29 Sep 2022 09:01:15 +0200 Subject: [PATCH 26/29] Bjbloemker-google/master (#843) * Add ability to add KMS encryption to google_compute_instance_template * fix code block Co-authored-by: BJ Bloemker Co-authored-by: bjbloemker-google <109173391+bjbloemker-google@users.noreply.github.com> Co-authored-by: Julio Castillo --- modules/compute-vm/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/compute-vm/main.tf b/modules/compute-vm/main.tf index 8cbd4215..3a588160 100644 --- a/modules/compute-vm/main.tf +++ b/modules/compute-vm/main.tf @@ -311,6 +311,12 @@ resource "google_compute_instance_template" "default" { config.value.source_type != "attach" ? config.value.name : null ) type = "PERSISTENT" + dynamic "disk_encryption_key" { + for_each = var.encryption != null ? [""] : [] + content { + kms_key_self_link = var.encryption.kms_key_self_link + } + } } } From 46f8cb1d71af4284a442eb0d71f651a86ed86e66 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 29 Sep 2022 09:29:12 +0200 Subject: [PATCH 27/29] update changelog --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 619bb9c1..5c602a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,16 @@ All notable changes to this project will be documented in this file. ### FAST +- [[#842](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/842)] Comment redundant role in bootstrap stage, align IAM.md files, improve IAM tool ([ludoo](https://github.com/ludoo)) +- [[#841](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/841)] FAST: revert 00-cicd provider changes ([ludoo](https://github.com/ludoo)) - [[#835](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/835)] Fix workflow-gitlab.yaml template rendering ([muresan](https://github.com/muresan)) - [[#828](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/828)] Update firewall rules. ([lcaggio](https://github.com/lcaggio)) - [[#807](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/807)] FAST: refactor Gitlab template ([ludoo](https://github.com/ludoo)) ### MODULES -- [[#840](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/840)] Refactor net-address module for 1.3 ([ludoo](https://github.com/ludoo)) +- [[#843](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/843)] Add support for disk encryption to instance templates in compute-vm module ([ludoo](https://github.com/ludoo)) +- [[#840](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/840)] **incompatible change:** Refactor net-address module for 1.3 ([ludoo](https://github.com/ludoo)) - [[#839](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/839)] **incompatible change:** Update to terraform 1.3 ([juliocc](https://github.com/juliocc)) - [[#824](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/824)] Add simple composer 2 blueprint ([lcaggio](https://github.com/lcaggio)) - [[#834](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/834)] Add support for service_label property in internal load balancer ([kmucha555](https://github.com/kmucha555)) @@ -36,6 +39,7 @@ All notable changes to this project will be documented in this file. ### TOOLS +- [[#842](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/842)] Comment redundant role in bootstrap stage, align IAM.md files, improve IAM tool ([ludoo](https://github.com/ludoo)) - [[#811](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/811)] Fix changelog generator ([ludoo](https://github.com/ludoo)) - [[#810](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/810)] Fully recursive e2e test runner for examples ([juliocc](https://github.com/juliocc)) From 2e51bdd8c4a4b3785924028432c264959a1f4cf0 Mon Sep 17 00:00:00 2001 From: cmalpe Date: Thu, 29 Sep 2022 10:57:19 +0000 Subject: [PATCH 28/29] added root password support for MS SQL Server --- modules/cloudsql-instance/README.md | 7 ++++--- modules/cloudsql-instance/main.tf | 1 + modules/cloudsql-instance/variables.tf | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index 2cbb09b9..bf2a12e4 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -151,7 +151,7 @@ module "db" { | [network](variables.tf#L102) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | string | ✓ | | | [project_id](variables.tf#L113) | The ID of the project where this instances will be created. | string | ✓ | | | [region](variables.tf#L118) | Region of the primary instance. | string | ✓ | | -| [tier](variables.tf#L132) | The machine type to use for the instances. | string | ✓ | | +| [tier](variables.tf#L138) | The machine type to use for the instances. | string | ✓ | | | [authorized_networks](variables.tf#L17) | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | map(string) | | null | | [availability_type](variables.tf#L23) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | | [backup_configuration](variables.tf#L29) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | object({…}) | | {…} | @@ -161,11 +161,12 @@ module "db" { | [disk_type](variables.tf#L73) | The type of data disk: `PD_SSD` or `PD_HDD`. | string | | "PD_SSD" | | [encryption_key_name](variables.tf#L79) | The full path to the encryption key used for the CMEK disk encryption of the primary instance. | string | | null | | [flags](variables.tf#L85) | Map FLAG_NAME=>VALUE for database-specific tuning. | map(string) | | null | -| [ipv4_enabled](variables.tf#L143) | Add a public IP address to database instance. | bool | | false | +| [ipv4_enabled](variables.tf#L149) | Add a public IP address to database instance. | bool | | false | | [labels](variables.tf#L91) | Labels to be attached to all instances. | map(string) | | null | | [prefix](variables.tf#L107) | Prefix used to generate instance names. | string | | null | | [replicas](variables.tf#L123) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | -| [users](variables.tf#L137) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | +| [root_password](variables.tf#L132) | Root password of the Cloud SQL instance. Required for MS SQL Server | string | | null | +| [users](variables.tf#L143) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | ## Outputs diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index 6cd280be..f15b386b 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -50,6 +50,7 @@ resource "google_sql_database_instance" "primary" { region = var.region database_version = var.database_version encryption_key_name = var.encryption_key_name + root_password = var.root_password settings { tier = var.tier diff --git a/modules/cloudsql-instance/variables.tf b/modules/cloudsql-instance/variables.tf index cd1581a2..05fbaf29 100644 --- a/modules/cloudsql-instance/variables.tf +++ b/modules/cloudsql-instance/variables.tf @@ -129,6 +129,12 @@ variable "replicas" { default = {} } +variable "root_password" { + description = "Root password of the Cloud SQL instance. Required for MS SQL Server" + type = string + default = null +} + variable "tier" { description = "The machine type to use for the instances." type = string From 9c2a53f7a94fb14c8bcb1e8591fcc1193ed4eb69 Mon Sep 17 00:00:00 2001 From: ddaluka <108656287+ddaluka@users.noreply.github.com> Date: Thu, 29 Sep 2022 18:40:07 +0530 Subject: [PATCH 29/29] Management of GCP project default service accounts (#844) * Added change for default service account deprivilege setting * Updated readme and variable name * Updated readme and variable name --- modules/project/README.md | 57 +++++++++++++++-------------- modules/project/service-accounts.tf | 8 ++++ modules/project/variables.tf | 6 +++ 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/modules/project/README.md b/modules/project/README.md index 8ce143bb..64ee2d4f 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -352,7 +352,7 @@ output "compute_robot" { | [main.tf](./main.tf) | Module-level locals and resources. | google_compute_project_metadata_item · google_essential_contacts_contact · google_monitoring_monitored_project · google_project · google_project_service · google_resource_manager_lien | | [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | google_project_organization_policy | | [outputs.tf](./outputs.tf) | Module outputs. | | -| [service-accounts.tf](./service-accounts.tf) | Service identities and supporting resources. | google_kms_crypto_key_iam_member · google_project_iam_member · google_project_service_identity | +| [service-accounts.tf](./service-accounts.tf) | Service identities and supporting resources. | google_kms_crypto_key_iam_member · google_project_default_service_accounts · google_project_iam_member · google_project_service_identity | | [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | google_compute_shared_vpc_host_project · google_compute_shared_vpc_service_project · google_project_iam_member | | [tags.tf](./tags.tf) | None | google_tags_tag_binding | | [variables.tf](./variables.tf) | Module variables. | | @@ -363,38 +363,39 @@ output "compute_robot" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L125) | Project name and id suffix. | string | ✓ | | +| [name](variables.tf#L131) | Project name and id suffix. | string | ✓ | | | [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | | [billing_account](variables.tf#L23) | Billing account id. | string | | null | | [contacts](variables.tf#L29) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | | [custom_roles](variables.tf#L36) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | +| [default_service_account](variables.tf#L49) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | string | | "keep" | | [descriptive_name](variables.tf#L43) | Name of the project name. Used for project name instead of `name` variable. | string | | null | -| [group_iam](variables.tf#L49) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | -| [iam](variables.tf#L56) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L63) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L70) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [labels](variables.tf#L76) | Resource labels. | map(string) | | {} | -| [lien_reason](variables.tf#L83) | If non-empty, creates a project lien with this description. | string | | "" | -| [logging_exclusions](variables.tf#L89) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L96) | Logging sinks to create for this project. | map(object({…})) | | {} | -| [metric_scopes](variables.tf#L118) | List of projects that will act as metric scopes for this project. | list(string) | | [] | -| [oslogin](variables.tf#L130) | Enable OS Login. | bool | | false | -| [oslogin_admins](variables.tf#L136) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | -| [oslogin_users](variables.tf#L144) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | -| [parent](variables.tf#L151) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | -| [policy_boolean](variables.tf#L161) | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | -| [policy_list](variables.tf#L168) | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({…})) | | {} | -| [prefix](variables.tf#L180) | Prefix used to generate project id and name. | string | | null | -| [project_create](variables.tf#L186) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | -| [service_config](variables.tf#L192) | Configure service API activation. | object({…}) | | {…} | -| [service_encryption_key_ids](variables.tf#L204) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | -| [service_perimeter_bridges](variables.tf#L211) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | -| [service_perimeter_standard](variables.tf#L218) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | -| [services](variables.tf#L224) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | -| [shared_vpc_service_config](variables.tf#L239) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | null | -| [skip_delete](variables.tf#L249) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | -| [tag_bindings](variables.tf#L255) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | +| [group_iam](variables.tf#L55) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L62) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive](variables.tf#L69) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive_members](variables.tf#L76) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | +| [labels](variables.tf#L82) | Resource labels. | map(string) | | {} | +| [lien_reason](variables.tf#L89) | If non-empty, creates a project lien with this description. | string | | "" | +| [logging_exclusions](variables.tf#L95) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L102) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables.tf#L124) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [oslogin](variables.tf#L136) | Enable OS Login. | bool | | false | +| [oslogin_admins](variables.tf#L142) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | +| [oslogin_users](variables.tf#L150) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | +| [parent](variables.tf#L157) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [policy_boolean](variables.tf#L167) | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | +| [policy_list](variables.tf#L174) | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({…})) | | {} | +| [prefix](variables.tf#L186) | Prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L192) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [service_config](variables.tf#L198) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L210) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | +| [service_perimeter_bridges](variables.tf#L217) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | +| [service_perimeter_standard](variables.tf#L224) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | +| [services](variables.tf#L230) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L236) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L245) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | null | +| [skip_delete](variables.tf#L255) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | +| [tag_bindings](variables.tf#L261) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index 4bb73062..f6da4fbf 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -140,3 +140,11 @@ resource "google_kms_crypto_key_iam_member" "service_identity_cmek" { data.google_storage_project_service_account.gcs_sa, ] } + +resource "google_project_default_service_accounts" "default_service_accounts" { + count = upper(var.default_service_account) == "KEEP" ? 0 : 1 + action = upper(var.default_service_account) + project = local.project.project_id + restore_policy = "REVERT_AND_IGNORE_FAILURE" + depends_on = [google_project_service.project_services] +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 9268deb2..30eff53a 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -46,6 +46,12 @@ variable "descriptive_name" { default = null } +variable "default_service_account" { + description = "Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`." + default = "keep" + type = string +} + variable "group_iam" { description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." type = map(list(string))