-This [blueprint](./data-platform-foundations/) implements SQL Server Always On Availability Groups using Fabric modules. It builds a two node cluster with a fileshare witness instance in an existing VPC and adds the necessary firewalling. The actual setup process (apart from Active Directory operations) has been scripted, so that least amount of manual works needs to performed.
+This [blueprint](./sqlserver-alwayson/) implements SQL Server Always On Availability Groups using Fabric modules. It builds a two node cluster with a fileshare witness instance in an existing VPC and adds the necessary firewalling. The actual setup process (apart from Active Directory operations) has been scripted, so that least amount of manual works needs to performed.
+
+
+This [blueprint](./vertex-mlops/) implements the infrastructure required to have a fully functional MLOPs environment using Vertex AI: required GCP services activation, Vertex Workbench, GCS buckets to host Vertex AI and Cloud Build artifacts, Artifact Registry docker repository to host custom images, required service accounts, networking and Workload Identity Federation Provider for Github integration (optional).
+
+
+This [blueprint](./shielded-folder/) implements an opinionated folder configuration according to GCP best practices. Configurations implemented on the folder would be beneficial to host workloads inheriting constraints from the folder they belong to.
string | ✓ | |
-| [root_node](variables.tf#L45) | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | |
-| [location](variables.tf#L21) | The location where resources will be deployed. | string | | "europe" |
-| [project_kms_name](variables.tf#L27) | Name for the new KMS Project. | string | | "my-project-kms-001" |
-| [project_service_name](variables.tf#L33) | Name for the new Service Project. | string | | "my-project-service-001" |
-| [region](variables.tf#L39) | The region where resources will be deployed. | string | | "europe-west1" |
+| [prefix](variables.tf#L21) | Optional prefix used to generate resources names. | string | ✓ | |
+| [project_config](variables.tf#L27) | Provide 'billing_account_id' and 'parent' values if project creation is needed, uses existing 'projects_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | |
+| [location](variables.tf#L15) | The location where resources will be deployed. | string | | "europe" |
+| [region](variables.tf#L44) | The region where resources will be deployed. | string | | "europe-west1" |
| [vpc_ip_cidr_range](variables.tf#L50) | Ip range used in the subnet deployef in the Service Project. | string | | "10.0.0.0/20" |
| [vpc_name](variables.tf#L56) | Name of the VPC created in the Service Project. | string | | "local" |
| [vpc_subnet_name](variables.tf#L62) | Name of the subnet created in the Service Project. | string | | "subnet" |
diff --git a/blueprints/data-solutions/cmek-via-centralized-kms/main.tf b/blueprints/data-solutions/cmek-via-centralized-kms/main.tf
index fb7f9fdd..73f2b701 100644
--- a/blueprints/data-solutions/cmek-via-centralized-kms/main.tf
+++ b/blueprints/data-solutions/cmek-via-centralized-kms/main.tf
@@ -12,33 +12,61 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+locals {
+ # Needed when you create KMS keys and encrypted resources in the same terraform state but different projects.
+ kms_keys = {
+ gce = "projects/${module.project-kms.project_id}/locations/${var.region}/keyRings/${var.prefix}-${var.region}/cryptoKeys/key-gcs"
+ gcs = "projects/${module.project-kms.project_id}/locations/${var.region}/keyRings/${var.prefix}-${var.region}/cryptoKeys/key-gcs"
+ }
+}
+
###############################################################################
# Projects #
###############################################################################
module "project-service" {
source = "../../../modules/project"
- name = var.project_service_name
- parent = var.root_node
- billing_account = var.billing_account
+ name = var.project_config.project_ids.service
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
services = [
"compute.googleapis.com",
"servicenetworking.googleapis.com",
- "storage-component.googleapis.com"
+ "storage.googleapis.com",
+ "storage-component.googleapis.com",
+ ]
+ service_encryption_key_ids = {
+ compute = [
+ local.kms_keys.gce
+ ]
+ storage = [
+ local.kms_keys.gcs
+ ]
+ }
+ service_config = {
+ disable_on_destroy = false, disable_dependent_services = false
+ }
+ depends_on = [
+ module.kms
]
- oslogin = true
}
module "project-kms" {
source = "../../../modules/project"
- name = var.project_kms_name
- parent = var.root_node
- billing_account = var.billing_account
+ name = var.project_config.project_ids.encryption
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
services = [
"cloudkms.googleapis.com",
"servicenetworking.googleapis.com"
]
- oslogin = true
+ service_config = {
+ disable_on_destroy = false, disable_dependent_services = false
+ }
}
###############################################################################
@@ -48,11 +76,11 @@ module "project-kms" {
module "vpc" {
source = "../../../modules/net-vpc"
project_id = module.project-service.project_id
- name = var.vpc_name
+ name = "${var.prefix}-vpc"
subnets = [
{
- ip_cidr_range = var.vpc_ip_cidr_range
- name = var.vpc_subnet_name
+ ip_cidr_range = "10.0.0.0/20"
+ name = "${var.prefix}-${var.region}"
region = var.region
}
]
@@ -63,7 +91,7 @@ module "vpc-firewall" {
project_id = module.project-service.project_id
network = module.vpc.name
default_rules_config = {
- admin_ranges = [var.vpc_ip_cidr_range]
+ admin_ranges = ["10.0.0.0/20"]
}
}
@@ -75,22 +103,10 @@ module "kms" {
source = "../../../modules/kms"
project_id = module.project-kms.project_id
keyring = {
- name = "my-keyring",
- location = var.location
+ name = "${var.prefix}-${var.region}",
+ location = var.region
}
keys = { key-gce = null, key-gcs = null }
- key_iam = {
- key-gce = {
- "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
- "serviceAccount:${module.project-service.service_accounts.robots.compute}",
- ]
- },
- key-gcs = {
- "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
- "serviceAccount:${module.project-service.service_accounts.robots.storage}",
- ]
- }
- }
}
###############################################################################
@@ -101,10 +117,10 @@ module "vm_example" {
source = "../../../modules/compute-vm"
project_id = module.project-service.project_id
zone = "${var.region}-b"
- name = "kms-vm"
+ name = "${var.prefix}-vm"
network_interfaces = [{
network = module.vpc.self_link,
- subnetwork = module.vpc.subnet_self_links["${var.region}/subnet"],
+ subnetwork = module.vpc.subnet_self_links["${var.region}/${var.prefix}-${var.region}"],
nat = false,
addresses = null
}]
@@ -127,7 +143,7 @@ module "vm_example" {
encryption = {
encrypt_boot = true
disk_encryption_key_raw = null
- kms_key_self_link = module.kms.key_ids.key-gce
+ kms_key_self_link = local.kms_keys.gce
}
}
@@ -138,7 +154,9 @@ module "vm_example" {
module "kms-gcs" {
source = "../../../modules/gcs"
project_id = module.project-service.project_id
- prefix = "my-bucket-001"
- name = "kms-gcs"
- encryption_key = module.kms.keys.key-gcs.id
+ prefix = var.prefix
+ name = "${var.prefix}-bucket"
+ location = var.region
+ storage_class = "REGIONAL"
+ encryption_key = local.kms_keys.gcs
}
diff --git a/blueprints/data-solutions/cmek-via-centralized-kms/variables.tf b/blueprints/data-solutions/cmek-via-centralized-kms/variables.tf
index 737bde3d..5d35351c 100644
--- a/blueprints/data-solutions/cmek-via-centralized-kms/variables.tf
+++ b/blueprints/data-solutions/cmek-via-centralized-kms/variables.tf
@@ -12,28 +12,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-variable "billing_account" {
- description = "Billing account id used as default for new projects."
- type = string
-}
-
variable "location" {
description = "The location where resources will be deployed."
type = string
default = "europe"
}
-variable "project_kms_name" {
- description = "Name for the new KMS Project."
+variable "prefix" {
+ description = "Optional prefix used to generate resources names."
type = string
- default = "my-project-kms-001"
+ nullable = false
}
-variable "project_service_name" {
- description = "Name for the new Service Project."
- type = string
- default = "my-project-service-001"
+variable "project_config" {
+ description = "Provide 'billing_account_id' and 'parent' values if project creation is needed, uses existing 'projects_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
+ type = object({
+ billing_account_id = optional(string, null)
+ parent = optional(string, null)
+ project_ids = optional(object({
+ encryption = string
+ service = string
+ }), {
+ encryption = "encryption",
+ service = "service"
+ }
+ )
+ })
+ nullable = false
}
variable "region" {
@@ -42,11 +47,6 @@ variable "region" {
default = "europe-west1"
}
-variable "root_node" {
- description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id."
- type = string
-}
-
variable "vpc_ip_cidr_range" {
description = "Ip range used in the subnet deployef in the Service Project."
type = string
diff --git a/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf b/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf
+++ b/blueprints/data-solutions/cmek-via-centralized-kms/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/data-solutions/data-platform-foundations/03-composer.tf b/blueprints/data-solutions/data-platform-foundations/03-composer.tf
index 33a21408..f806f0e5 100644
--- a/blueprints/data-solutions/data-platform-foundations/03-composer.tf
+++ b/blueprints/data-solutions/data-platform-foundations/03-composer.tf
@@ -40,6 +40,7 @@ locals {
LOD_SA_DF = module.load-sa-df-0.email
ORC_PRJ = module.orch-project.project_id
ORC_GCS = module.orch-cs-0.url
+ ORC_GCS_TMP_DF = module.orch-cs-df-template.url
TRF_PRJ = module.transf-project.project_id
TRF_GCS_STAGING = module.transf-cs-df-0.url
TRF_NET_VPC = local.transf_vpc
diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
index 8e2d0725..a202afdd 100644
--- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
+++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
@@ -25,6 +25,11 @@ locals {
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)
+
+ # Note: This formatting is needed for output purposes since the fabric artifact registry
+ # module doesn't yet expose the docker usage path of a registry folder in the needed format.
+ orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
+ var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}
module "orch-project" {
@@ -44,6 +49,8 @@ module "orch-project" {
"roles/iam.serviceAccountUser",
"roles/storage.objectAdmin",
"roles/storage.admin",
+ "roles/artifactregistry.admin",
+ "roles/serviceusage.serviceUsageConsumer",
]
}
iam = {
@@ -65,7 +72,15 @@ module "orch-project" {
]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
+ module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
+ "serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
+ ]
+ "roles/artifactregistry.reader" = [
+ module.load-sa-df-0.iam_email,
+ ]
+ "roles/cloudbuild.serviceAgent" = [
+ module.orch-sa-df-build.iam_email,
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
@@ -81,6 +96,7 @@ module "orch-project" {
"compute.googleapis.com",
"container.googleapis.com",
"containerregistry.googleapis.com",
+ "artifactregistry.googleapis.com",
"dataflow.googleapis.com",
"orgpolicy.googleapis.com",
"pubsub.googleapis.com",
@@ -148,3 +164,46 @@ module "orch-nat" {
region = var.region
router_network = module.orch-vpc.0.name
}
+
+module "orch-artifact-reg" {
+ source = "../../../modules/artifact-registry"
+ project_id = module.orch-project.project_id
+ id = "${var.prefix}-app-images"
+ location = var.region
+ format = "DOCKER"
+ description = "Docker repository storing application images e.g. Dataflow, Cloud Run etc..."
+}
+
+module "orch-cs-df-template" {
+ source = "../../../modules/gcs"
+ project_id = module.orch-project.project_id
+ prefix = var.prefix
+ name = "orc-cs-df-template"
+ location = var.region
+ storage_class = "REGIONAL"
+ encryption_key = try(local.service_encryption_keys.storage, null)
+}
+
+module "orch-cs-build-staging" {
+ source = "../../../modules/gcs"
+ project_id = module.orch-project.project_id
+ prefix = var.prefix
+ name = "orc-cs-build-staging"
+ location = var.region
+ storage_class = "REGIONAL"
+ encryption_key = try(local.service_encryption_keys.storage, null)
+}
+
+module "orch-sa-df-build" {
+ source = "../../../modules/iam-service-account"
+ project_id = module.orch-project.project_id
+ prefix = var.prefix
+ name = "orc-sa-df-build"
+ display_name = "Data platform Dataflow build service account"
+ # Note values below should pertain to the system / group / users who are able to
+ # invoke the build via this service account
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
+ "roles/iam.serviceAccountUser" = [local.groups_iam.data-engineers]
+ }
+}
diff --git a/blueprints/data-solutions/data-platform-foundations/IAM.md b/blueprints/data-solutions/data-platform-foundations/IAM.md
index 5a1995da..dd898bd7 100644
--- a/blueprints/data-solutions/data-platform-foundations/IAM.md
+++ b/blueprints/data-solutions/data-platform-foundations/IAM.md
@@ -71,11 +71,13 @@ Legend: + additive, • conditional.
| members | roles |
|---|---|
-|gcp-data-engineers+|
-|load-df-0object({…}) | ✓ | |
+| [folder_config](variables.tf#L49) | Provide 'folder_create' values if folder creation is needed, uses existing 'folder_id' otherwise. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | |
+| [organization](variables.tf#L128) | Organization details. | object({…}) | ✓ | |
+| [prefix](variables.tf#L136) | Prefix used for resources that need unique names. | string | ✓ | |
+| [project_config](variables.tf#L141) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | |
+| [data_dir](variables.tf#L29) | Relative path for the folder storing configuration data. | string | | "data" |
+| [enable_features](variables.tf#L35) | Flag to enable features on the solution. | object({…}) | | {…} |
+| [groups](variables.tf#L65) | User groups. | object({…}) | | {} |
+| [kms_keys](variables.tf#L75) | KMS keys to create, keyed by name. | map(object({…})) | | {} |
+| [log_locations](variables.tf#L86) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} |
+| [log_sinks](variables.tf#L103) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} |
+| [vpc_sc_access_levels](variables.tf#L161) | VPC SC access level definitions. | map(object({…})) | | {} |
+| [vpc_sc_egress_policies](variables.tf#L190) | VPC SC egress policy defnitions. | map(object({…})) | | {} |
+| [vpc_sc_ingress_policies](variables.tf#L210) | VPC SC ingress policy defnitions. | map(object({…})) | | {} |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [folders](outputs.tf#L15) | Folders id. | |
+| [folders_sink_writer_identities](outputs.tf#L23) | Folders id. | |
+
+
diff --git a/blueprints/data-solutions/shielded-folder/data/firewall-policies/cidrs.yaml b/blueprints/data-solutions/shielded-folder/data/firewall-policies/cidrs.yaml
new file mode 100644
index 00000000..90dabfb6
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/data/firewall-policies/cidrs.yaml
@@ -0,0 +1,15 @@
+# skip boilerplate check
+
+healthchecks:
+ - 35.191.0.0/16
+ - 130.211.0.0/22
+ - 209.85.152.0/22
+ - 209.85.204.0/22
+
+rfc1918:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+
+onprem_probes:
+ - 10.255.255.254/32
\ No newline at end of file
diff --git a/blueprints/data-solutions/shielded-folder/data/firewall-policies/hierarchical-policy-rules.yaml b/blueprints/data-solutions/shielded-folder/data/firewall-policies/hierarchical-policy-rules.yaml
new file mode 100644
index 00000000..6a3b3133
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/data/firewall-policies/hierarchical-policy-rules.yaml
@@ -0,0 +1,50 @@
+# skip boilerplate check
+
+allow-admins:
+ description: Access from the admin subnet to all subnets
+ direction: INGRESS
+ action: allow
+ priority: 1000
+ ranges:
+ - $rfc1918
+ ports:
+ all: []
+ target_resources: null
+ enable_logging: false
+
+allow-healthchecks:
+ description: Enable HTTP and HTTPS healthchecks
+ direction: INGRESS
+ action: allow
+ priority: 1001
+ ranges:
+ - $healthchecks
+ ports:
+ tcp: ["80", "443"]
+ target_resources: null
+ enable_logging: false
+
+allow-ssh-from-iap:
+ description: Enable SSH from IAP
+ direction: INGRESS
+ action: allow
+ priority: 1002
+ ranges:
+ - 35.235.240.0/20
+ ports:
+ tcp: ["22"]
+ target_resources: null
+ enable_logging: false
+
+allow-icmp:
+ description: Enable ICMP
+ direction: INGRESS
+ action: allow
+ priority: 1003
+ ranges:
+ - 0.0.0.0/0
+ ports:
+ icmp: []
+ target_resources: null
+ enable_logging: false
+
\ No newline at end of file
diff --git a/fast/stages/01-resman/data/org-policies/compute.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/compute.yaml
similarity index 100%
rename from fast/stages/01-resman/data/org-policies/compute.yaml
rename to blueprints/data-solutions/shielded-folder/data/org-policies/compute.yaml
diff --git a/fast/stages/01-resman/data/org-policies/iam.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/iam.yaml
similarity index 100%
rename from fast/stages/01-resman/data/org-policies/iam.yaml
rename to blueprints/data-solutions/shielded-folder/data/org-policies/iam.yaml
diff --git a/fast/stages/01-resman/data/org-policies/serverless.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/serverless.yaml
similarity index 100%
rename from fast/stages/01-resman/data/org-policies/serverless.yaml
rename to blueprints/data-solutions/shielded-folder/data/org-policies/serverless.yaml
diff --git a/fast/stages/01-resman/data/org-policies/sql.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/sql.yaml
similarity index 100%
rename from fast/stages/01-resman/data/org-policies/sql.yaml
rename to blueprints/data-solutions/shielded-folder/data/org-policies/sql.yaml
diff --git a/fast/stages/01-resman/data/org-policies/storage.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/storage.yaml
similarity index 100%
rename from fast/stages/01-resman/data/org-policies/storage.yaml
rename to blueprints/data-solutions/shielded-folder/data/org-policies/storage.yaml
diff --git a/blueprints/data-solutions/shielded-folder/data/vpc-sc/accessible-services.yaml b/blueprints/data-solutions/shielded-folder/data/vpc-sc/accessible-services.yaml
new file mode 100644
index 00000000..2107d2ff
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/data/vpc-sc/accessible-services.yaml
@@ -0,0 +1,119 @@
+# skip boilerplate check
+
+- accessapproval.googleapis.com
+- adsdatahub.googleapis.com
+- aiplatform.googleapis.com
+- alloydb.googleapis.com
+- alpha-documentai.googleapis.com
+- analyticshub.googleapis.com
+- apigee.googleapis.com
+- apigeeconnect.googleapis.com
+- artifactregistry.googleapis.com
+- assuredworkloads.googleapis.com
+- automl.googleapis.com
+- baremetalsolution.googleapis.com
+- batch.googleapis.com
+- beyondcorp.googleapis.com
+- bigquery.googleapis.com
+- bigquerydatapolicy.googleapis.com
+- bigquerydatatransfer.googleapis.com
+- bigquerymigration.googleapis.com
+- bigqueryreservation.googleapis.com
+- bigtable.googleapis.com
+- binaryauthorization.googleapis.com
+- cloudasset.googleapis.com
+- cloudbuild.googleapis.com
+- clouddebugger.googleapis.com
+- clouderrorreporting.googleapis.com
+- cloudfunctions.googleapis.com
+- cloudkms.googleapis.com
+- cloudprofiler.googleapis.com
+- cloudresourcemanager.googleapis.com
+- cloudsearch.googleapis.com
+- cloudtrace.googleapis.com
+- composer.googleapis.com
+- compute.googleapis.com
+- connectgateway.googleapis.com
+- contactcenterinsights.googleapis.com
+- container.googleapis.com
+- containeranalysis.googleapis.com
+- containerfilesystem.googleapis.com
+- containerregistry.googleapis.com
+- containerthreatdetection.googleapis.com
+- contentwarehouse.googleapis.com
+- datacatalog.googleapis.com
+- dataflow.googleapis.com
+- datafusion.googleapis.com
+- datalineage.googleapis.com
+- datamigration.googleapis.com
+- datapipelines.googleapis.com
+- dataplex.googleapis.com
+- dataproc.googleapis.com
+- datastream.googleapis.com
+- dialogflow.googleapis.com
+- dlp.googleapis.com
+- dns.googleapis.com
+- documentai.googleapis.com
+- domains.googleapis.com
+- essentialcontacts.googleapis.com
+- eventarc.googleapis.com
+- file.googleapis.com
+- firebaseappcheck.googleapis.com
+- firebaserules.googleapis.com
+- firestore.googleapis.com
+- gameservices.googleapis.com
+- gkebackup.googleapis.com
+- gkeconnect.googleapis.com
+- gkehub.googleapis.com
+- gkemulticloud.googleapis.com
+- healthcare.googleapis.com
+- iam.googleapis.com
+- iamcredentials.googleapis.com
+- iaptunnel.googleapis.com
+- ids.googleapis.com
+- integrations.googleapis.com
+- language.googleapis.com
+- lifesciences.googleapis.com
+- logging.googleapis.com
+- managedidentities.googleapis.com
+- memcache.googleapis.com
+- meshca.googleapis.com
+- metastore.googleapis.com
+- ml.googleapis.com
+- monitoring.googleapis.com
+- networkconnectivity.googleapis.com
+- networkmanagement.googleapis.com
+- networksecurity.googleapis.com
+- networkservices.googleapis.com
+- notebooks.googleapis.com
+- opsconfigmonitoring.googleapis.com
+- osconfig.googleapis.com
+- oslogin.googleapis.com
+- policytroubleshooter.googleapis.com
+- privateca.googleapis.com
+- pubsub.googleapis.com
+- pubsublite.googleapis.com
+- recaptchaenterprise.googleapis.com
+- recommender.googleapis.com
+- redis.googleapis.com
+- retail.googleapis.com
+- run.googleapis.com
+- secretmanager.googleapis.com
+- servicecontrol.googleapis.com
+- servicedirectory.googleapis.com
+- spanner.googleapis.com
+- speakerid.googleapis.com
+- speech.googleapis.com
+- sqladmin.googleapis.com
+- storage.googleapis.com
+- storagetransfer.googleapis.com
+- texttospeech.googleapis.com
+- tpu.googleapis.com
+- trafficdirector.googleapis.com
+- transcoder.googleapis.com
+- translate.googleapis.com
+- videointelligence.googleapis.com
+- vision.googleapis.com
+- visionai.googleapis.com
+- vpcaccess.googleapis.com
+- workstations.googleapis.com
\ No newline at end of file
diff --git a/blueprints/data-solutions/shielded-folder/data/vpc-sc/restricted-services.yaml b/blueprints/data-solutions/shielded-folder/data/vpc-sc/restricted-services.yaml
new file mode 100644
index 00000000..2107d2ff
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/data/vpc-sc/restricted-services.yaml
@@ -0,0 +1,119 @@
+# skip boilerplate check
+
+- accessapproval.googleapis.com
+- adsdatahub.googleapis.com
+- aiplatform.googleapis.com
+- alloydb.googleapis.com
+- alpha-documentai.googleapis.com
+- analyticshub.googleapis.com
+- apigee.googleapis.com
+- apigeeconnect.googleapis.com
+- artifactregistry.googleapis.com
+- assuredworkloads.googleapis.com
+- automl.googleapis.com
+- baremetalsolution.googleapis.com
+- batch.googleapis.com
+- beyondcorp.googleapis.com
+- bigquery.googleapis.com
+- bigquerydatapolicy.googleapis.com
+- bigquerydatatransfer.googleapis.com
+- bigquerymigration.googleapis.com
+- bigqueryreservation.googleapis.com
+- bigtable.googleapis.com
+- binaryauthorization.googleapis.com
+- cloudasset.googleapis.com
+- cloudbuild.googleapis.com
+- clouddebugger.googleapis.com
+- clouderrorreporting.googleapis.com
+- cloudfunctions.googleapis.com
+- cloudkms.googleapis.com
+- cloudprofiler.googleapis.com
+- cloudresourcemanager.googleapis.com
+- cloudsearch.googleapis.com
+- cloudtrace.googleapis.com
+- composer.googleapis.com
+- compute.googleapis.com
+- connectgateway.googleapis.com
+- contactcenterinsights.googleapis.com
+- container.googleapis.com
+- containeranalysis.googleapis.com
+- containerfilesystem.googleapis.com
+- containerregistry.googleapis.com
+- containerthreatdetection.googleapis.com
+- contentwarehouse.googleapis.com
+- datacatalog.googleapis.com
+- dataflow.googleapis.com
+- datafusion.googleapis.com
+- datalineage.googleapis.com
+- datamigration.googleapis.com
+- datapipelines.googleapis.com
+- dataplex.googleapis.com
+- dataproc.googleapis.com
+- datastream.googleapis.com
+- dialogflow.googleapis.com
+- dlp.googleapis.com
+- dns.googleapis.com
+- documentai.googleapis.com
+- domains.googleapis.com
+- essentialcontacts.googleapis.com
+- eventarc.googleapis.com
+- file.googleapis.com
+- firebaseappcheck.googleapis.com
+- firebaserules.googleapis.com
+- firestore.googleapis.com
+- gameservices.googleapis.com
+- gkebackup.googleapis.com
+- gkeconnect.googleapis.com
+- gkehub.googleapis.com
+- gkemulticloud.googleapis.com
+- healthcare.googleapis.com
+- iam.googleapis.com
+- iamcredentials.googleapis.com
+- iaptunnel.googleapis.com
+- ids.googleapis.com
+- integrations.googleapis.com
+- language.googleapis.com
+- lifesciences.googleapis.com
+- logging.googleapis.com
+- managedidentities.googleapis.com
+- memcache.googleapis.com
+- meshca.googleapis.com
+- metastore.googleapis.com
+- ml.googleapis.com
+- monitoring.googleapis.com
+- networkconnectivity.googleapis.com
+- networkmanagement.googleapis.com
+- networksecurity.googleapis.com
+- networkservices.googleapis.com
+- notebooks.googleapis.com
+- opsconfigmonitoring.googleapis.com
+- osconfig.googleapis.com
+- oslogin.googleapis.com
+- policytroubleshooter.googleapis.com
+- privateca.googleapis.com
+- pubsub.googleapis.com
+- pubsublite.googleapis.com
+- recaptchaenterprise.googleapis.com
+- recommender.googleapis.com
+- redis.googleapis.com
+- retail.googleapis.com
+- run.googleapis.com
+- secretmanager.googleapis.com
+- servicecontrol.googleapis.com
+- servicedirectory.googleapis.com
+- spanner.googleapis.com
+- speakerid.googleapis.com
+- speech.googleapis.com
+- sqladmin.googleapis.com
+- storage.googleapis.com
+- storagetransfer.googleapis.com
+- texttospeech.googleapis.com
+- tpu.googleapis.com
+- trafficdirector.googleapis.com
+- transcoder.googleapis.com
+- translate.googleapis.com
+- videointelligence.googleapis.com
+- vision.googleapis.com
+- visionai.googleapis.com
+- vpcaccess.googleapis.com
+- workstations.googleapis.com
\ No newline at end of file
diff --git a/blueprints/data-solutions/shielded-folder/images/overview_diagram.png b/blueprints/data-solutions/shielded-folder/images/overview_diagram.png
new file mode 100644
index 00000000..abc8d56e
Binary files /dev/null and b/blueprints/data-solutions/shielded-folder/images/overview_diagram.png differ
diff --git a/blueprints/data-solutions/shielded-folder/kms.tf b/blueprints/data-solutions/shielded-folder/kms.tf
new file mode 100644
index 00000000..7a3b42b5
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/kms.tf
@@ -0,0 +1,102 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Security project, Cloud KMS and Secret Manager resources.
+
+locals {
+ kms_locations = distinct(flatten([
+ for k, v in var.kms_keys : v.locations
+ ]))
+ kms_locations_keys = {
+ for loc in local.kms_locations : loc => {
+ for k, v in var.kms_keys : k => v if contains(v.locations, loc)
+ }
+ }
+
+ kms_log_locations = distinct(flatten([
+ for k, v in local.kms_log_sink_keys : compact(v.locations)
+ ]))
+
+ # Log sink keys
+ kms_log_sink_keys = {
+ "storage" = {
+ labels = {}
+ locations = [var.log_locations.storage]
+ rotation_period = "7776000s"
+ }
+ "bq" = {
+ labels = {}
+ locations = [var.log_locations.bq]
+ rotation_period = "7776000s"
+ }
+ "pubsub" = {
+ labels = {}
+ locations = [var.log_locations.pubsub]
+ rotation_period = "7776000s"
+ }
+ }
+ kms_log_locations_keys = {
+ for loc in local.kms_log_locations : loc => {
+ for k, v in local.kms_log_sink_keys : k => v if contains(v.locations, loc)
+ }
+ }
+}
+
+module "sec-project" {
+ count = var.enable_features.encryption ? 1 : 0
+ source = "../../../modules/project"
+ name = var.project_config.project_ids["sec-core"]
+ parent = module.folder.id
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null && var.enable_features.encryption
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ group_iam = {
+ (local.groups.workload-security) = [
+ "roles/editor"
+ ]
+ }
+ services = [
+ "cloudkms.googleapis.com",
+ "secretmanager.googleapis.com",
+ "stackdriver.googleapis.com"
+ ]
+}
+
+module "sec-kms" {
+ for_each = var.enable_features.encryption ? toset(local.kms_locations) : toset([])
+ source = "../../../modules/kms"
+ project_id = module.sec-project[0].project_id
+ keyring = {
+ location = each.key
+ name = "${each.key}"
+ }
+ # rename to `key_iam` to switch to authoritative bindings
+ key_iam_additive = {
+ for k, v in local.kms_locations_keys[each.key] : k => v.iam
+ }
+ keys = local.kms_locations_keys[each.key]
+}
+
+module "log-kms" {
+ for_each = var.enable_features.encryption ? toset(local.kms_log_locations) : toset([])
+ source = "../../../modules/kms"
+ project_id = module.sec-project[0].project_id
+ keyring = {
+ location = each.key
+ name = "${each.key}"
+ }
+ keys = local.kms_log_locations_keys[each.key]
+}
diff --git a/blueprints/data-solutions/shielded-folder/log-export.tf b/blueprints/data-solutions/shielded-folder/log-export.tf
new file mode 100644
index 00000000..9eff32d2
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/log-export.tf
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Audit log project and sink.
+
+locals {
+ gcs_storage_class = (
+ length(split("-", var.log_locations.storage)) < 2
+ ? "MULTI_REGIONAL"
+ : "REGIONAL"
+ )
+ log_types = toset([for k, v in var.log_sinks : v.type])
+
+ _log_keys = var.enable_features.encryption ? {
+ bq = var.enable_features.log_sink ? ["projects/${module.sec-project.0.project_id}/locations/${var.log_locations.bq}/keyRings/${var.log_locations.bq}/cryptoKeys/bq"] : null
+ pubsub = var.enable_features.log_sink ? ["projects/${module.sec-project.0.project_id}/locations/${var.log_locations.pubsub}/keyRings/${var.log_locations.pubsub}/cryptoKeys/pubsub"] : null
+ storage = var.enable_features.log_sink ? ["projects/${module.sec-project.0.project_id}/locations/${var.log_locations.storage}/keyRings/${var.log_locations.storage}/cryptoKeys/storage"] : null
+ } : {}
+
+ log_keys = {
+ for service, key in local._log_keys : service => key if key != null
+ }
+}
+
+module "log-export-project" {
+ count = var.enable_features.log_sink ? 1 : 0
+ source = "../../../modules/project"
+ name = var.project_config.project_ids["audit-logs"]
+ parent = module.folder.id
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ group_iam = {
+ (local.groups.workload-security) = [
+ "roles/editor"
+ ]
+ }
+ iam = {
+ # "roles/owner" = [module.automation-tf-bootstrap-sa.iam_email]
+ }
+ services = [
+ "bigquery.googleapis.com",
+ "pubsub.googleapis.com",
+ "storage.googleapis.com",
+ "stackdriver.googleapis.com"
+ ]
+ service_encryption_key_ids = var.enable_features.encryption ? local.log_keys : {}
+
+ depends_on = [
+ module.log-kms
+ ]
+}
+
+# one log export per type, with conditionals to skip those not needed
+
+module "log-export-dataset" {
+ source = "../../../modules/bigquery-dataset"
+ count = var.enable_features.log_sink && contains(local.log_types, "bigquery") ? 1 : 0
+ project_id = module.log-export-project[0].project_id
+ id = "${var.prefix}_audit_export"
+ friendly_name = "Audit logs export."
+ location = replace(var.log_locations.bq, "europe", "EU")
+ encryption_key = var.enable_features.encryption ? module.log-kms[var.log_locations.bq].keys["bq"].id : false
+}
+
+module "log-export-gcs" {
+ source = "../../../modules/gcs"
+ count = var.enable_features.log_sink && contains(local.log_types, "storage") ? 1 : 0
+ project_id = module.log-export-project[0].project_id
+ name = "audit-logs"
+ prefix = var.prefix
+ location = replace(var.log_locations.storage, "europe", "EU")
+ storage_class = local.gcs_storage_class
+ encryption_key = var.enable_features.encryption ? module.log-kms[var.log_locations.storage].keys["storage"].id : null
+}
+
+module "log-export-logbucket" {
+ source = "../../../modules/logging-bucket"
+ for_each = var.enable_features.log_sink ? toset([for k, v in var.log_sinks : k if v.type == "logging"]) : []
+ parent_type = "project"
+ parent = module.log-export-project[0].project_id
+ id = "audit-logs-${each.key}"
+ location = var.log_locations.logging
+ #TODO check if logging bucket support encryption.
+}
+
+module "log-export-pubsub" {
+ source = "../../../modules/pubsub"
+ for_each = toset([for k, v in var.log_sinks : k if v.type == "pubsub" && var.enable_features.log_sink])
+ project_id = module.log-export-project[0].project_id
+ name = "audit-logs-${each.key}"
+ regions = [var.log_locations.pubsub]
+ kms_key = var.enable_features.encryption ? module.log-kms[var.log_locations.pubsub].keys["pubsub"].id : null
+}
diff --git a/blueprints/data-solutions/shielded-folder/main.tf b/blueprints/data-solutions/shielded-folder/main.tf
new file mode 100644
index 00000000..52fa0db2
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/main.tf
@@ -0,0 +1,141 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Folder resources.
+
+locals {
+ # Create Log sink ingress policies
+ _sink_ingress_policies = var.enable_features.log_sink ? {
+ log_sink = {
+ from = {
+ access_levels = ["*"]
+ identities = values(module.folder.sink_writer_identities)
+ }
+ to = {
+ resources = ["projects/${module.log-export-project.0.number}"]
+ operations = [{ service_name = "*" }]
+ } }
+ } : null
+
+ _vpc_sc_vpc_accessible_services = var.data_dir != null ? yamldecode(
+ file("${var.data_dir}/vpc-sc/restricted-services.yaml")
+ ) : null
+ _vpc_sc_restricted_services = var.data_dir != null ? yamldecode(
+ file("${var.data_dir}/vpc-sc/restricted-services.yaml")
+ ) : null
+
+ access_policy_create = var.access_policy_config.access_policy_create != null ? {
+ parent = "organizations/${var.organization.id}"
+ title = "shielded-folder"
+ scopes = [module.folder.id]
+ } : null
+
+ groups = {
+ for k, v in var.groups : k => "${v}@${var.organization.domain}"
+ }
+ groups_iam = {
+ for k, v in local.groups : k => "group:${v}"
+ }
+ group_iam = {
+ (local.groups.workload-engineers) = [
+ "roles/editor",
+ "roles/iam.serviceAccountTokenCreator"
+ ]
+ }
+
+ vpc_sc_resources = [
+ for k, v in data.google_projects.folder-projects.projects : format("projects/%s", v.number)
+ ]
+
+ log_sink_destinations = var.enable_features.log_sink ? merge(
+ # use the same dataset for all sinks with `bigquery` as destination
+ { for k, v in var.log_sinks : k => module.log-export-dataset.0 if v.type == "bigquery" },
+ # use the same gcs bucket for all sinks with `storage` as destination
+ { for k, v in var.log_sinks : k => module.log-export-gcs.0 if v.type == "storage" },
+ # use separate pubsub topics and logging buckets for sinks with
+ # destination `pubsub` and `logging`
+ module.log-export-pubsub,
+ module.log-export-logbucket
+ ) : null
+}
+
+module "folder" {
+ source = "../../../modules/folder"
+ folder_create = var.folder_config.folder_create != null
+ parent = try(var.folder_config.folder_create.parent, null)
+ name = try(var.folder_config.folder_create.display_name, null)
+ id = var.folder_config.folder_create != null ? null : var.folder_config.folder_id
+ group_iam = local.group_iam
+ org_policies_data_path = var.data_dir != null ? "${var.data_dir}/org-policies" : null
+ firewall_policy_factory = var.data_dir != null ? {
+ cidr_file = "${var.data_dir}/firewall-policies/cidrs.yaml"
+ policy_name = "${var.prefix}-fw-policy"
+ rules_file = "${var.data_dir}/firewall-policies/hierarchical-policy-rules.yaml"
+ } : null
+ logging_sinks = var.enable_features.log_sink ? {
+ for name, attrs in var.log_sinks : name => {
+ bq_partitioned_table = attrs.type == "bigquery"
+ destination = local.log_sink_destinations[name].id
+ filter = attrs.filter
+ type = attrs.type
+ }
+ } : null
+}
+
+module "folder-workload" {
+ source = "../../../modules/folder"
+ parent = module.folder.id
+ name = "${var.prefix}-workload"
+}
+
+
+#TODO VPCSC: Access levels
+data "google_projects" "folder-projects" {
+ filter = "parent.id:${split("/", module.folder.id)[1]}"
+
+ depends_on = [
+ module.sec-project,
+ module.log-export-project
+ ]
+}
+
+module "vpc-sc" {
+ count = var.enable_features.vpc_sc ? 1 : 0
+ source = "../../../modules/vpc-sc"
+ access_policy = try(var.access_policy_config.policy_name, null)
+ access_policy_create = local.access_policy_create
+ access_levels = var.vpc_sc_access_levels
+ egress_policies = var.vpc_sc_egress_policies
+ ingress_policies = merge(var.vpc_sc_ingress_policies, local._sink_ingress_policies)
+ service_perimeters_regular = {
+ shielded = {
+ # Move `spec` definition to `status` and comment `use_explicit_dry_run_spec` variable to enforce VPC-SC configuration
+ # Before enforing configuration check logs and create Access Level, Ingress/Egress policy as needed
+
+ status = null
+ spec = {
+ access_levels = keys(var.vpc_sc_access_levels)
+ resources = local.vpc_sc_resources
+ restricted_services = local._vpc_sc_restricted_services
+ egress_policies = keys(var.vpc_sc_egress_policies)
+ ingress_policies = keys(merge(var.vpc_sc_ingress_policies, local._sink_ingress_policies))
+ vpc_accessible_services = {
+ allowed_services = local._vpc_sc_vpc_accessible_services
+ enable_restriction = true
+ }
+ }
+ use_explicit_dry_run_spec = true
+ }
+ }
+}
diff --git a/blueprints/data-solutions/shielded-folder/outputs.tf b/blueprints/data-solutions/shielded-folder/outputs.tf
new file mode 100644
index 00000000..e1107fc6
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/outputs.tf
@@ -0,0 +1,30 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+output "folders" {
+ description = "Folders id."
+ value = {
+ shielded-folder = module.folder.id
+ workload-folder = module.folder-workload.id
+ }
+}
+
+output "folders_sink_writer_identities" {
+ description = "Folders id."
+ value = {
+ shielded-folder = module.folder.sink_writer_identities
+ workload-folder = module.folder-workload.sink_writer_identities
+ }
+}
+
diff --git a/blueprints/data-solutions/shielded-folder/variables.tf b/blueprints/data-solutions/shielded-folder/variables.tf
new file mode 100644
index 00000000..a9ecbb24
--- /dev/null
+++ b/blueprints/data-solutions/shielded-folder/variables.tf
@@ -0,0 +1,229 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Variables definition.
+
+variable "access_policy_config" {
+ description = "Provide 'access_policy_create' values if a folder scoped Access Policy creation is needed, uses existing 'policy_name' otherwise. Parent is in 'organizations/123456' format. Policy will be created scoped to the folder."
+ type = object({
+ policy_name = optional(string, null)
+ access_policy_create = optional(object({
+ parent = string
+ title = string
+ }), null)
+ })
+ nullable = false
+}
+
+variable "data_dir" {
+ description = "Relative path for the folder storing configuration data."
+ type = string
+ default = "data"
+}
+
+variable "enable_features" {
+ description = "Flag to enable features on the solution."
+ type = object({
+ encryption = optional(bool, false)
+ log_sink = optional(bool, true)
+ vpc_sc = optional(bool, true)
+ })
+ default = {
+ encryption = false
+ log_sink = true
+ vpc_sc = true
+ }
+}
+
+variable "folder_config" {
+ description = "Provide 'folder_create' values if folder creation is needed, uses existing 'folder_id' otherwise. Parent is in 'folders/nnn' or 'organizations/nnn' format."
+ type = object({
+ folder_id = optional(string, null)
+ folder_create = optional(object({
+ display_name = string
+ parent = string
+ }), null)
+ })
+ validation {
+ condition = var.folder_config.folder_id != null || var.folder_config.folder_create != null
+ error_message = "At least one attribute should be set."
+ }
+ nullable = false
+}
+
+variable "groups" {
+ description = "User groups."
+ type = object({
+ workload-engineers = optional(string, "gcp-data-engineers")
+ workload-security = optional(string, "gcp-data-security")
+ })
+ default = {}
+ nullable = false
+}
+
+variable "kms_keys" {
+ description = "KMS keys to create, keyed by name."
+ type = map(object({
+ iam = optional(map(list(string)), {})
+ labels = optional(map(string), {})
+ locations = optional(list(string), ["global", "europe", "europe-west1"])
+ rotation_period = optional(string, "7776000s")
+ }))
+ default = {}
+}
+
+variable "log_locations" {
+ description = "Optional locations for GCS, BigQuery, and logging buckets created here."
+ type = object({
+ bq = optional(string, "europe")
+ storage = optional(string, "europe")
+ logging = optional(string, "global")
+ pubsub = optional(string, "global")
+ })
+ default = {
+ bq = "europe"
+ storage = "europe"
+ logging = "global"
+ pubsub = null
+ }
+ nullable = false
+}
+
+variable "log_sinks" {
+ description = "Org-level log sinks, in name => {type, filter} format."
+ type = map(object({
+ filter = string
+ type = string
+ }))
+ default = {
+ audit-logs = {
+ filter = "logName:\"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName:\"/logs/cloudaudit.googleapis.com%2Fsystem_event\""
+ type = "bigquery"
+ }
+ vpc-sc = {
+ filter = "protoPayload.metadata.@type=\"type.googleapis.com/google.cloud.audit.VpcServiceControlAuditMetadata\""
+ type = "bigquery"
+ }
+ }
+ validation {
+ condition = alltrue([
+ for k, v in var.log_sinks :
+ contains(["bigquery", "logging", "pubsub", "storage"], v.type)
+ ])
+ error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'."
+ }
+}
+
+variable "organization" {
+ description = "Organization details."
+ type = object({
+ domain = string
+ id = string
+ })
+}
+
+variable "prefix" {
+ description = "Prefix used for resources that need unique names."
+ type = string
+}
+
+variable "project_config" {
+ description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
+ type = object({
+ billing_account_id = optional(string, null)
+ project_ids = optional(object({
+ sec-core = string
+ audit-logs = string
+ }), {
+ sec-core = "sec-core"
+ audit-logs = "audit-logs"
+ }
+ )
+ })
+ nullable = false
+ validation {
+ condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null
+ error_message = "At least one attribute should be set."
+ }
+}
+
+variable "vpc_sc_access_levels" {
+ description = "VPC SC access level definitions."
+ type = map(object({
+ combining_function = optional(string)
+ conditions = optional(list(object({
+ device_policy = optional(object({
+ allowed_device_management_levels = optional(list(string))
+ allowed_encryption_statuses = optional(list(string))
+ require_admin_approval = bool
+ require_corp_owned = bool
+ require_screen_lock = optional(bool)
+ os_constraints = optional(list(object({
+ os_type = string
+ minimum_version = optional(string)
+ require_verified_chrome_os = optional(bool)
+ })))
+ }))
+ ip_subnetworks = optional(list(string), [])
+ members = optional(list(string), [])
+ negate = optional(bool)
+ regions = optional(list(string), [])
+ required_access_levels = optional(list(string), [])
+ })), [])
+ description = optional(string)
+ }))
+ default = {}
+ nullable = false
+}
+
+variable "vpc_sc_egress_policies" {
+ description = "VPC SC egress policy defnitions."
+ type = map(object({
+ from = object({
+ identity_type = optional(string, "ANY_IDENTITY")
+ identities = optional(list(string))
+ })
+ to = object({
+ operations = optional(list(object({
+ method_selectors = optional(list(string))
+ service_name = string
+ })), [])
+ resources = optional(list(string))
+ resource_type_external = optional(bool, false)
+ })
+ }))
+ default = {}
+ nullable = false
+}
+
+variable "vpc_sc_ingress_policies" {
+ description = "VPC SC ingress policy defnitions."
+ type = map(object({
+ from = object({
+ access_levels = optional(list(string), [])
+ identity_type = optional(string)
+ identities = optional(list(string))
+ resources = optional(list(string), [])
+ })
+ to = object({
+ operations = optional(list(object({
+ method_selectors = optional(list(string))
+ service_name = string
+ })), [])
+ resources = optional(list(string))
+ })
+ }))
+ default = {}
+ nullable = false
+}
diff --git a/blueprints/data-solutions/vertex-mlops/README.md b/blueprints/data-solutions/vertex-mlops/README.md
new file mode 100644
index 00000000..d9f85fd8
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/README.md
@@ -0,0 +1,79 @@
+# MLOps with Vertex AI
+
+## Introduction
+This example implements the infrastructure required to deploy an end-to-end [MLOps process](https://services.google.com/fh/files/misc/practitioners_guide_to_mlops_whitepaper.pdf) using [Vertex AI](https://cloud.google.com/vertex-ai) platform.
+
+## GCP resources
+The blueprint will deploy all the required resources to have a fully functional MLOPs environment containing:
+- Vertex Workbench (for the experimentation environment)
+- GCP Project (optional) to host all the resources
+- Isolated VPC network and a subnet to be used by Vertex and Dataflow. Alternatively, an external Shared VPC can be configured using the `network_config`variable.
+- Firewall rule to allow the internal subnet communication required by Dataflow
+- Cloud NAT required to reach the internet from the different computing resources (Vertex and Dataflow)
+- GCS buckets to host Vertex AI and Cloud Build Artifacts. By default the buckets will be regional and should match the Vertex AI region for the different resources (i.e. Vertex Managed Dataset) and processes (i.e. Vertex trainining)
+- BigQuery Dataset where the training data will be stored. This is optional, since the training data could be already hosted in an existing BigQuery dataset.
+- Artifact Registry Docker repository to host the custom images.
+- Service account (`mlops-[env]@`) with the minimum permissions required by Vertex AI and Dataflow (if this service is used inside of the Vertex AI Pipeline).
+- Service account (`github@`) to be used by Workload Identity Federation, to federate Github identity (Optional).
+- Secret to store the Github SSH key to get access the CICD code repo.
+
+
+
+## Pre-requirements
+
+### User groups
+
+Assign roles relying on User groups is a way to decouple the final set of permissions from the stage where entities and resources are created, and their IAM bindings defined. You can configure the group names through the `groups` variable. These groups should be created before launching Terraform.
+
+We use the following groups to control access to resources:
+
+- *Data Scientits* (gcp-ml-ds@string | ✓ | |
+| [bucket_name](variables.tf#L18) | GCS bucket name to store the Vertex AI artifacts. | string | | null |
+| [dataset_name](variables.tf#L24) | BigQuery Dataset to store the training data. | string | | null |
+| [groups](variables.tf#L30) | Name of the groups (name@domain.org) to apply opinionated IAM permissions. | object({…}) | | {…} |
+| [identity_pool_claims](variables.tf#L45) | Claims to be used by Workload Identity Federation (i.e.: attribute.repository/ORGANIZATION/REPO). If a not null value is provided, then google_iam_workload_identity_pool resource will be created. | string | | null |
+| [labels](variables.tf#L51) | Labels to be assigned at project level. | map(string) | | {} |
+| [location](variables.tf#L57) | Location used for multi-regional resources. | string | | "eu" |
+| [network_config](variables.tf#L63) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null |
+| [notebooks](variables.tf#L73) | Vertex AI workbenchs to be deployed. | map(object({…})) | | {} |
+| [prefix](variables.tf#L86) | Prefix used for the project id. | string | | null |
+| [project_create](variables.tf#L92) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null |
+| [project_services](variables.tf#L106) | List of core services enabled on all projects. | list(string) | | […] |
+| [region](variables.tf#L126) | Region used for regional resources. | string | | "europe-west4" |
+| [repo_name](variables.tf#L132) | Cloud Source Repository name. null to avoid to create it. | string | | null |
+| [sa_mlops_name](variables.tf#L138) | Name for the MLOPs Service Account. | string | | "sa-mlops" |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [github](outputs.tf#L33) | Github Configuration. | |
+| [notebook](outputs.tf#L39) | Vertex AI managed notebook details. | |
+| [project](outputs.tf#L44) | The project resource as return by the `project` module. | |
+| [project_id](outputs.tf#L49) | Project ID. | |
+
+
+# TODO
+- Add support for User Managed Notebooks, SA permission option and non default SA for Single User mode.
+- Improve default naming for local VPC and Cloud NAT
\ No newline at end of file
diff --git a/blueprints/data-solutions/vertex-mlops/ci-cd.tf b/blueprints/data-solutions/vertex-mlops/ci-cd.tf
new file mode 100644
index 00000000..d73eacc8
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/ci-cd.tf
@@ -0,0 +1,74 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_iam_workload_identity_pool" "github_pool" {
+ count = var.identity_pool_claims == null ? 0 : 1
+ project = module.project.project_id
+ workload_identity_pool_id = "gh-pool"
+ display_name = "Github Actions Identity Pool"
+ description = "Identity pool for Github Actions"
+}
+
+resource "google_iam_workload_identity_pool_provider" "github_provider" {
+ count = var.identity_pool_claims == null ? 0 : 1
+ project = module.project.project_id
+ workload_identity_pool_id = google_iam_workload_identity_pool.github_pool[0].workload_identity_pool_id
+ workload_identity_pool_provider_id = "gh-provider"
+ display_name = "Github Actions provider"
+ description = "OIDC provider for Github Actions"
+ attribute_mapping = {
+ "google.subject" = "assertion.sub"
+ "attribute.repository" = "assertion.repository"
+ }
+ oidc {
+ issuer_uri = "https://token.actions.githubusercontent.com"
+ }
+}
+
+module "artifact_registry" {
+ source = "../../../modules/artifact-registry"
+ id = "docker-repo"
+ project_id = module.project.project_id
+ location = var.region
+ format = "DOCKER"
+ # iam = {
+ # "roles/artifactregistry.admin" = ["group:cicd@example.com"]
+ # }
+}
+
+module "service-account-github" {
+ source = "../../../modules/iam-service-account"
+ name = "sa-github"
+ project_id = module.project.project_id
+ iam = var.identity_pool_claims == null ? {} : { "roles/iam.workloadIdentityUser" = ["principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github_pool[0].name}/${var.identity_pool_claims}"] }
+}
+
+# NOTE: Secret manager module at the moment does not support CMEK
+module "secret-manager" {
+ project_id = module.project.project_id
+ source = "../../../modules/secret-manager"
+ secrets = {
+ github-key = [var.region]
+ }
+ iam = {
+ github-key = {
+ "roles/secretmanager.secretAccessor" = [
+ "serviceAccount:${module.project.service_accounts.robots.cloudbuild}",
+ module.service-account-mlops.iam_email
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/blueprints/data-solutions/vertex-mlops/images/mlops_projects.png b/blueprints/data-solutions/vertex-mlops/images/mlops_projects.png
new file mode 100644
index 00000000..24017bc9
Binary files /dev/null and b/blueprints/data-solutions/vertex-mlops/images/mlops_projects.png differ
diff --git a/blueprints/data-solutions/vertex-mlops/main.tf b/blueprints/data-solutions/vertex-mlops/main.tf
new file mode 100644
index 00000000..5f7fbc0c
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/main.tf
@@ -0,0 +1,278 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+locals {
+ group_iam = merge(
+ var.groups.gcp-ml-viewer == null ? {} : {
+ (var.groups.gcp-ml-viewer) = [
+ "roles/aiplatform.viewer",
+ "roles/artifactregistry.reader",
+ "roles/dataflow.viewer",
+ "roles/logging.viewer",
+ "roles/storage.objectViewer"
+ ]
+ },
+ var.groups.gcp-ml-ds == null ? {} : {
+ (var.groups.gcp-ml-ds) = [
+ "roles/aiplatform.admin",
+ "roles/artifactregistry.admin",
+ "roles/bigquery.dataEditor",
+ "roles/bigquery.jobUser",
+ "roles/bigquery.user",
+ "roles/cloudbuild.builds.editor",
+ "roles/cloudfunctions.developer",
+ "roles/dataflow.developer",
+ "roles/dataflow.worker",
+ "roles/iam.serviceAccountUser",
+ "roles/logging.logWriter",
+ "roles/logging.viewer",
+ "roles/notebooks.admin",
+ "roles/pubsub.editor",
+ "roles/serviceusage.serviceUsageConsumer",
+ "roles/storage.admin"
+ ]
+ },
+ var.groups.gcp-ml-eng == null ? {} : {
+ (var.groups.gcp-ml-eng) = [
+ "roles/aiplatform.admin",
+ "roles/artifactregistry.admin",
+ "roles/bigquery.dataEditor",
+ "roles/bigquery.jobUser",
+ "roles/bigquery.user",
+ "roles/dataflow.developer",
+ "roles/dataflow.worker",
+ "roles/iam.serviceAccountUser",
+ "roles/logging.logWriter",
+ "roles/logging.viewer",
+ "roles/serviceusage.serviceUsageConsumer",
+ "roles/storage.admin"
+ ]
+ }
+ )
+
+ service_encryption_keys = var.service_encryption_keys
+ shared_vpc_project = try(var.network_config.host_project, null)
+
+ subnet = (
+ local.use_shared_vpc
+ ? var.network_config.subnet_self_link
+ : values(module.vpc-local.0.subnet_self_links)[0]
+ )
+ vpc = (
+ local.use_shared_vpc
+ ? var.network_config.network_self_link
+ : module.vpc-local.0.self_link
+ )
+ use_shared_vpc = var.network_config != null
+
+ shared_vpc_bindings = {
+ "roles/compute.networkUser" = [
+ "robot-df", "notebooks"
+ ]
+ }
+
+ shared_vpc_role_members = {
+ robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
+ notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
+ }
+
+ # reassemble in a format suitable for for_each
+ shared_vpc_bindings_map = {
+ for binding in flatten([
+ for role, members in local.shared_vpc_bindings : [
+ for member in members : { role = role, member = member }
+ ]
+ ]) : "${binding.role}-${binding.member}" => binding
+ }
+}
+
+module "gcs-bucket" {
+ count = var.bucket_name == null ? 0 : 1
+ source = "../../../modules/gcs"
+ project_id = module.project.project_id
+ name = var.bucket_name
+ prefix = var.prefix
+ location = var.region
+ storage_class = "REGIONAL"
+ versioning = false
+ encryption_key = try(local.service_encryption_keys.storage, null)
+}
+
+# Default bucket for Cloud Build to prevent error: "'us' violates constraint ‘constraints/gcp.resourceLocations’"
+# https://stackoverflow.com/questions/53206667/cloud-build-fails-with-resource-location-constraint
+module "gcs-bucket-cloudbuild" {
+ source = "../../../modules/gcs"
+ project_id = module.project.project_id
+ name = "${var.project_id}_cloudbuild"
+ prefix = var.prefix
+ location = var.region
+ storage_class = "REGIONAL"
+ versioning = false
+ encryption_key = try(local.service_encryption_keys.storage, null)
+}
+
+module "bq-dataset" {
+ count = var.dataset_name == null ? 0 : 1
+ source = "../../../modules/bigquery-dataset"
+ project_id = module.project.project_id
+ id = var.dataset_name
+ location = var.region
+ encryption_key = try(local.service_encryption_keys.bq, null)
+}
+
+module "vpc-local" {
+ count = local.use_shared_vpc ? 0 : 1
+ source = "../../../modules/net-vpc"
+ project_id = module.project.project_id
+ name = "default"
+ subnets = [
+ {
+ "name" : "default",
+ "region" : "${var.region}",
+ "ip_cidr_range" : "10.4.0.0/24",
+ "secondary_ip_range" : null
+ }
+ ]
+ psa_config = {
+ ranges = {
+ "vertex" : "10.13.0.0/18"
+ }
+ routes = null
+ }
+}
+
+module "firewall" {
+ count = local.use_shared_vpc ? 0 : 1
+ source = "../../../modules/net-vpc-firewall"
+ project_id = module.project.project_id
+ network = module.vpc-local[0].name
+ default_rules_config = {
+ disabled = true
+ }
+ ingress_rules = {
+ dataflow-ingress = {
+ description = "Dataflow service."
+ direction = "INGRESS"
+ action = "allow"
+ sources = ["dataflow"]
+ targets = ["dataflow"]
+ ranges = []
+ use_service_accounts = false
+ rules = [{ protocol = "tcp", ports = ["12345-12346"] }]
+ extra_attributes = {}
+ }
+ }
+
+}
+
+module "cloudnat" {
+ count = local.use_shared_vpc ? 0 : 1
+ source = "../../../modules/net-cloudnat"
+ project_id = module.project.project_id
+ region = var.region
+ name = "default"
+ router_network = module.vpc-local[0].self_link
+}
+
+module "project" {
+ source = "../../../modules/project"
+ name = var.project_id
+ parent = try(var.project_create.parent, null)
+ billing_account = try(var.project_create.billing_account_id, null)
+ project_create = var.project_create != null
+ prefix = var.prefix
+ group_iam = local.group_iam
+ iam = {
+ "roles/aiplatform.user" = [module.service-account-mlops.iam_email]
+ "roles/artifactregistry.reader" = [module.service-account-mlops.iam_email]
+ "roles/artifactregistry.writer" = [module.service-account-github.iam_email]
+ "roles/bigquery.dataEditor" = [module.service-account-mlops.iam_email]
+ "roles/bigquery.jobUser" = [module.service-account-mlops.iam_email]
+ "roles/bigquery.user" = [module.service-account-mlops.iam_email]
+ "roles/cloudbuild.builds.editor" = [
+ module.service-account-mlops.iam_email,
+ module.service-account-github.iam_email
+ ]
+
+ "roles/cloudfunctions.invoker" = [module.service-account-mlops.iam_email]
+ "roles/dataflow.developer" = [module.service-account-mlops.iam_email]
+ "roles/dataflow.worker" = [module.service-account-mlops.iam_email]
+ "roles/iam.serviceAccountUser" = [
+ module.service-account-mlops.iam_email,
+ "serviceAccount:${module.project.service_accounts.robots.cloudbuild}"
+ ]
+ "roles/monitoring.metricWriter" = [module.service-account-mlops.iam_email]
+ "roles/run.invoker" = [module.service-account-mlops.iam_email]
+ "roles/serviceusage.serviceUsageConsumer" = [
+ module.service-account-mlops.iam_email,
+ module.service-account-github.iam_email
+ ]
+ "roles/storage.admin" = [
+ module.service-account-mlops.iam_email,
+ module.service-account-github.iam_email
+ ]
+ }
+ labels = var.labels
+
+ org_policies = {
+ # Example of applying a project wide policy
+ # "constraints/compute.requireOsLogin" = {
+ # enforce = false
+ # }
+ }
+
+ service_encryption_key_ids = {
+ bq = [try(local.service_encryption_keys.bq, null)]
+ compute = [try(local.service_encryption_keys.compute, null)]
+ cloudbuild = [try(local.service_encryption_keys.storage, null)]
+ notebooks = [try(local.service_encryption_keys.compute, null)]
+ storage = [try(local.service_encryption_keys.storage, null)]
+ }
+ services = var.project_services
+
+
+ shared_vpc_service_config = local.shared_vpc_project == null ? null : {
+ attach = true
+ host_project = local.shared_vpc_project
+ }
+
+}
+
+module "service-account-mlops" {
+ source = "../../../modules/iam-service-account"
+ name = var.sa_mlops_name
+ project_id = module.project.project_id
+ iam = {
+ "roles/iam.serviceAccountUser" = [module.service-account-github.iam_email]
+ }
+}
+
+resource "google_project_iam_member" "shared_vpc" {
+ count = local.use_shared_vpc ? 1 : 0
+ project = var.network_config.host_project
+ role = "roles/compute.networkUser"
+ member = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
+}
+
+
+resource "google_sourcerepo_repository" "code-repo" {
+ count = var.repo_name == null ? 0 : 1
+ name = var.repo_name
+ project = module.project.project_id
+}
+
+
diff --git a/blueprints/data-solutions/vertex-mlops/notebooks.tf b/blueprints/data-solutions/vertex-mlops/notebooks.tf
new file mode 100644
index 00000000..09d3e5a8
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/notebooks.tf
@@ -0,0 +1,60 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_notebooks_runtime" "runtime" {
+ for_each = var.notebooks
+ name = each.key
+
+ project = module.project.project_id
+ location = var.notebooks[each.key].region
+ access_config {
+ access_type = "SINGLE_USER"
+ runtime_owner = var.notebooks[each.key].owner
+ }
+ software_config {
+ enable_health_monitoring = true
+ idle_shutdown = var.notebooks[each.key].idle_shutdown
+ idle_shutdown_timeout = 1800
+ }
+ virtual_machine {
+ virtual_machine_config {
+ machine_type = "n1-standard-4"
+ network = local.vpc
+ subnet = local.subnet
+ internal_ip_only = var.notebooks[each.key].internal_ip_only
+ dynamic "encryption_config" {
+ for_each = try(local.service_encryption_keys.compute, null) == null ? [] : [1]
+ content {
+ kms_key = local.service_encryption_keys.compute
+ }
+ }
+ metadata = {
+ notebook-disable-nbconvert = "false"
+ notebook-disable-downloads = "false"
+ notebook-disable-terminal = "false"
+ #notebook-disable-root = "true"
+ #notebook-upgrade-schedule = "48 4 * * MON"
+ }
+ data_disk {
+ initialize_params {
+ disk_size_gb = "100"
+ disk_type = "PD_STANDARD"
+ }
+ }
+ }
+ }
+}
+
diff --git a/blueprints/data-solutions/vertex-mlops/outputs.tf b/blueprints/data-solutions/vertex-mlops/outputs.tf
new file mode 100644
index 00000000..9cb390d6
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/outputs.tf
@@ -0,0 +1,52 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# TODO(): proper outputs
+
+
+locals {
+ docker_split = try(split("/", module.artifact_registry.id), null)
+ docker_repo = try("${local.docker_split[3]}-docker.pkg.dev/${local.docker_split[1]}/${local.docker_split[5]}", null)
+ gh_config = {
+ WORKLOAD_ID_PROVIDER = try(google_iam_workload_identity_pool_provider.github_provider[0].name, null)
+ SERVICE_ACCOUNT = try(module.service-account-github.email, null)
+ PROJECT_ID = module.project.project_id
+ DOCKER_REPO = local.docker_repo
+ SA_MLOPS = module.service-account-mlops.email
+ SUBNETWORK = local.subnet
+ }
+}
+
+output "github" {
+
+ description = "Github Configuration."
+ value = local.gh_config
+}
+
+output "notebook" {
+ description = "Vertex AI managed notebook details."
+ value = { for k, v in resource.google_notebooks_runtime.runtime : k => v.id }
+}
+
+output "project" {
+ description = "The project resource as return by the `project` module."
+ value = module.project
+}
+
+output "project_id" {
+ description = "Project ID."
+ value = module.project.project_id
+}
diff --git a/blueprints/data-solutions/vertex-mlops/terraform.tfvars.sample b/blueprints/data-solutions/vertex-mlops/terraform.tfvars.sample
new file mode 100644
index 00000000..097bac3a
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/terraform.tfvars.sample
@@ -0,0 +1,20 @@
+bucket_name = "creditcards-dev"
+dataset_name = "creditcards"
+identity_pool_claims = "attribute.repository/ORGANIZATION/REPO"
+labels = {
+ "env" : "dev",
+ "team" : "ml"
+}
+notebooks = {
+ "myworkbench" : {
+ "owner" : "user@example.com",
+ "region" : "europe-west4",
+ "subnet" : "default",
+ }
+}
+prefix = "pref"
+project_id = "creditcards-dev"
+project_create = {
+ billing_account_id = "000000-123456-123456"
+ parent = "folders/111111111111"
+}
diff --git a/blueprints/data-solutions/vertex-mlops/variables.tf b/blueprints/data-solutions/vertex-mlops/variables.tf
new file mode 100644
index 00000000..f3f6efad
--- /dev/null
+++ b/blueprints/data-solutions/vertex-mlops/variables.tf
@@ -0,0 +1,152 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+variable "bucket_name" {
+ description = "GCS bucket name to store the Vertex AI artifacts."
+ type = string
+ default = null
+}
+
+variable "dataset_name" {
+ description = "BigQuery Dataset to store the training data."
+ type = string
+ default = null
+}
+
+variable "groups" {
+ description = "Name of the groups (name@domain.org) to apply opinionated IAM permissions."
+ type = object({
+ gcp-ml-ds = string
+ gcp-ml-eng = string
+ gcp-ml-viewer = string
+ })
+ default = {
+ gcp-ml-ds = null
+ gcp-ml-eng = null
+ gcp-ml-viewer = null
+ }
+ nullable = false
+}
+
+variable "identity_pool_claims" {
+ description = "Claims to be used by Workload Identity Federation (i.e.: attribute.repository/ORGANIZATION/REPO). If a not null value is provided, then google_iam_workload_identity_pool resource will be created."
+ type = string
+ default = null
+}
+
+variable "labels" {
+ description = "Labels to be assigned at project level."
+ type = map(string)
+ default = {}
+}
+
+variable "location" {
+ description = "Location used for multi-regional resources."
+ type = string
+ default = "eu"
+}
+
+variable "network_config" {
+ description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
+ type = object({
+ host_project = string
+ network_self_link = string
+ subnet_self_link = string
+ })
+ default = null
+}
+
+variable "notebooks" {
+ description = "Vertex AI workbenchs to be deployed."
+ type = map(object({
+ owner = string
+ region = string
+ subnet = string
+ internal_ip_only = optional(bool, false)
+ idle_shutdown = optional(bool)
+ }))
+ default = {}
+ nullable = false
+}
+
+variable "prefix" {
+ description = "Prefix used for the project id."
+ type = string
+ default = null
+}
+
+variable "project_create" {
+ description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
+ type = object({
+ billing_account_id = string
+ parent = string
+ })
+ default = null
+}
+
+variable "project_id" {
+ description = "Project id, references existing project if `project_create` is null."
+ type = string
+}
+
+variable "project_services" {
+ description = "List of core services enabled on all projects."
+ type = list(string)
+ default = [
+ "aiplatform.googleapis.com",
+ "artifactregistry.googleapis.com",
+ "bigquery.googleapis.com",
+ "cloudbuild.googleapis.com",
+ "compute.googleapis.com",
+ "datacatalog.googleapis.com",
+ "dataflow.googleapis.com",
+ "iam.googleapis.com",
+ "monitoring.googleapis.com",
+ "notebooks.googleapis.com",
+ "secretmanager.googleapis.com",
+ "servicenetworking.googleapis.com",
+ "serviceusage.googleapis.com"
+ ]
+}
+
+variable "region" {
+ description = "Region used for regional resources."
+ type = string
+ default = "europe-west4"
+}
+
+variable "repo_name" {
+ description = "Cloud Source Repository name. null to avoid to create it."
+ type = string
+ default = null
+}
+
+variable "sa_mlops_name" {
+ description = "Name for the MLOPs Service Account."
+ type = string
+ default = "sa-mlops"
+}
+
+variable "service_encryption_keys" { # service encription key
+ description = "Cloud KMS to use to encrypt different services. Key location should match service region."
+ type = object({
+ bq = string
+ compute = string
+ storage = string
+ })
+ default = null
+}
\ No newline at end of file
diff --git a/blueprints/factories/bigquery-factory/README.md b/blueprints/factories/bigquery-factory/README.md
index 05cabffb..2cba6e01 100644
--- a/blueprints/factories/bigquery-factory/README.md
+++ b/blueprints/factories/bigquery-factory/README.md
@@ -1,36 +1,18 @@
# Google Cloud BQ Factory
-This module allows creation and management of BigQuery datasets and views as well as tables by defining them in well formatted `yaml` files.
+This module allows creation and management of BigQuery datasets tables and views by defining them in well-formatted YAML files. YAML abstraction for BQ can simplify users onboarding and also makes creation of tables easier compared to HCL.
-Yaml abstraction for BQ can simplify users onboarding and also makes creation of tables easier compared to HCL.
+This factory is based on the [BQ dataset module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/bigquery-dataset) which currently only supports tables and views. As soon as external table and materialized view support is added, this factory will be enhanced accordingly.
-Subfolders distinguish between views and tables and ensures easier navigation for users.
-
-This factory is based on the [BQ dataset module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/bigquery-dataset) which currently only supports tables and views. As soon as external table and materialized view support is added, factory will be enhanced accordingly.
-
-You can create as many files as you like, the code will loop through it and create the required variables in order to execute everything accordingly.
+You can create as many files as you like, the code will loop through it and create everything accordingly.
## Example
### Terraform code
-```hcl
-module "bq" {
- source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric/modules/bigquery-dataset"
-
- for_each = local.output
- project_id = var.project_id
- id = each.key
- views = try(each.value.views, null)
- tables = try(each.value.tables, null)
-}
-# tftest skip
-```
-
-### Configuration Structure
-
+In this section we show how to create tables and views from a file structure simlar to the one shown below.
```bash
-base_folder
+bigquery
│
├── tables
│ ├── table_a.yaml
@@ -40,32 +22,43 @@ base_folder
│ ├── view_b.yaml
```
-## YAML structure and definition formatting
-
-### Tables
-
-Table definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following:
+First we create the table definition in `bigquery/tables/countries.yaml`.
```yaml
-
-dataset: # required name of the dataset the table is to be placed in
-table: # required descriptive name of the table
-schema: # required schema in JSON FORMAT Example: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}]
-labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"}
-use_legacy_sql: boolean # not required, defaults to false
-deletion_protection: boolean # not required, defaults to false
+# tftest-file id=table path=bigquery/tables/countries.yaml
+dataset: my_dataset
+table: countries
+deletion_protection: true
+labels:
+ env: prod
+schema:
+ - name: country
+ type: STRING
+ - name: population
+ type: INT64
```
-### Views
-View definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following:
+And a view in `bigquery/views/population.yaml`.
```yaml
-dataset: # required, name of the dataset the view is to be placed in
-view: # required, descriptive name of the view
-query: # required, SQL Query for the view in quotes
-labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"}
-use_legacy_sql: bool # not required, defaults to false
-deletion_protection: bool # not required, defaults to false
+# tftest-file id=view path=bigquery/views/population.yaml
+dataset: my_dataset
+view: department
+query: SELECT SUM(population) from my_dataset.countries
+labels:
+ env: prod
+```
+
+With this file structure, we can use the factory as follows:
+
+```hcl
+module "bq" {
+ source = "./fabric/blueprints/factories/bigquery-factory"
+ project_id = var.project_id
+ tables_path = "bigquery/tables"
+ views_path = "bigquery/views"
+}
+# tftest modules=2 resources=3 files=table,view inventory=simple.yaml
```
@@ -74,8 +67,8 @@ deletion_protection: bool # not required, defaults to false
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [project_id](variables.tf#L17) | Project ID. | string | ✓ | |
-| [tables_dir](variables.tf#L22) | Relative path for the folder storing table data. | string | ✓ | |
-| [views_dir](variables.tf#L27) | Relative path for the folder storing view data. | string | ✓ | |
+| [tables_path](variables.tf#L22) | Relative path for the folder storing table data. | string | ✓ | |
+| [views_path](variables.tf#L27) | Relative path for the folder storing view data. | string | ✓ | |
## TODO
diff --git a/blueprints/factories/bigquery-factory/main.tf b/blueprints/factories/bigquery-factory/main.tf
index 5995ea19..8c26f747 100644
--- a/blueprints/factories/bigquery-factory/main.tf
+++ b/blueprints/factories/bigquery-factory/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,17 +16,22 @@
locals {
views = {
- for f in fileset("${var.views_dir}", "**/*.yaml") :
- trimsuffix(f, ".yaml") => yamldecode(file("${var.views_dir}/${f}"))
+ for f in fileset(var.views_path, "**/*.yaml") :
+ trimsuffix(f, ".yaml") => yamldecode(file("${var.views_path}/${f}"))
}
tables = {
- for f in fileset("${var.tables_dir}", "**/*.yaml") :
- trimsuffix(f, ".yaml") => yamldecode(file("${var.tables_dir}/${f}"))
+ for f in fileset(var.tables_path, "**/*.yaml") :
+ trimsuffix(f, ".yaml") => yamldecode(file("${var.tables_path}/${f}"))
}
- output = {
- for dataset in distinct([for v in values(merge(local.views, local.tables)) : v.dataset]) :
+ all_datasets = distinct(concat(
+ [for x in values(local.tables) : x.dataset],
+ [for x in values(local.views) : x.dataset]
+ ))
+
+ datasets = {
+ for dataset in local.all_datasets :
dataset => {
"views" = {
for k, v in local.views :
@@ -57,9 +62,8 @@ locals {
}
module "bq" {
- source = "../../../modules/bigquery-dataset"
-
- for_each = local.output
+ source = "../../../modules/bigquery-dataset"
+ for_each = local.datasets
project_id = var.project_id
id = each.key
views = try(each.value.views, null)
diff --git a/blueprints/factories/bigquery-factory/variables.tf b/blueprints/factories/bigquery-factory/variables.tf
index 774ec86e..57025f62 100644
--- a/blueprints/factories/bigquery-factory/variables.tf
+++ b/blueprints/factories/bigquery-factory/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,12 +19,12 @@ variable "project_id" {
type = string
}
-variable "tables_dir" {
+variable "tables_path" {
description = "Relative path for the folder storing table data."
type = string
}
-variable "views_dir" {
+variable "views_path" {
description = "Relative path for the folder storing view data."
type = string
}
diff --git a/blueprints/factories/net-vpc-firewall-yaml/versions.tf b/blueprints/factories/net-vpc-firewall-yaml/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/factories/net-vpc-firewall-yaml/versions.tf
+++ b/blueprints/factories/net-vpc-firewall-yaml/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/factories/project-factory/README.md b/blueprints/factories/project-factory/README.md
index 32a1f8f0..2b8c3874 100644
--- a/blueprints/factories/project-factory/README.md
+++ b/blueprints/factories/project-factory/README.md
@@ -59,6 +59,7 @@ module "projects" {
for_each = local.projects
defaults = local.defaults
project_id = each.key
+ descriptive_name = try(each.value.descriptive_name, null)
billing_account_id = try(each.value.billing_account_id, null)
billing_alert = try(each.value.billing_alert, null)
dns_zones = try(each.value.dns_zones, [])
@@ -222,28 +223,29 @@ vpc:
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | |
-| [prefix](variables.tf#L151) | Prefix used for resource names. | string | ✓ | |
-| [project_id](variables.tf#L160) | Project id. | string | ✓ | |
+| [prefix](variables.tf#L157) | Prefix used for resource names. | string | ✓ | |
+| [project_id](variables.tf#L166) | Project id. | string | ✓ | |
| [billing_alert](variables.tf#L22) | Billing alert configuration. | object({…}) | | null |
| [defaults](variables.tf#L35) | Project factory default values. | object({…}) | | null |
-| [dns_zones](variables.tf#L57) | DNS private zones to create as child of var.defaults.environment_dns_zone. | list(string) | | [] |
-| [essential_contacts](variables.tf#L63) | Email contacts to be used for billing and GCP notifications. | list(string) | | [] |
-| [folder_id](variables.tf#L69) | Folder ID for the folder where the project will be created. | string | | null |
-| [group_iam](variables.tf#L75) | Custom IAM settings in group => [role] format. | map(list(string)) | | {} |
-| [group_iam_additive](variables.tf#L81) | Custom additive IAM settings in group => [role] format. | map(list(string)) | | {} |
-| [iam](variables.tf#L87) | Custom IAM settings in role => [principal] format. | map(list(string)) | | {} |
-| [iam_additive](variables.tf#L93) | Custom additive IAM settings in role => [principal] format. | map(list(string)) | | {} |
-| [kms_service_agents](variables.tf#L99) | KMS IAM configuration in as service => [key]. | map(list(string)) | | {} |
-| [labels](variables.tf#L105) | Labels to be assigned at project level. | map(string) | | {} |
-| [org_policies](variables.tf#L111) | Org-policy overrides at project level. | map(object({…})) | | {} |
-| [service_accounts](variables.tf#L165) | Service accounts to be created, and roles assigned them on the project. | map(list(string)) | | {} |
-| [service_accounts_additive](variables.tf#L171) | Service accounts to be created, and roles assigned them on the project additively. | map(list(string)) | | {} |
-| [service_accounts_iam](variables.tf#L177) | IAM bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string))) | | {} |
-| [service_accounts_iam_additive](variables.tf#L184) | IAM additive bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string))) | | {} |
-| [service_identities_iam](variables.tf#L191) | Custom IAM settings for service identities in service => [role] format. | map(list(string)) | | {} |
-| [service_identities_iam_additive](variables.tf#L198) | Custom additive IAM settings for service identities in service => [role] format. | map(list(string)) | | {} |
-| [services](variables.tf#L205) | Services to be enabled for the project. | list(string) | | [] |
-| [vpc](variables.tf#L212) | VPC configuration for the project. | object({…}) | | null |
+| [descriptive_name](variables.tf#L57) | Name of the project name. Used for project name instead of `name` variable. | string | | null |
+| [dns_zones](variables.tf#L63) | DNS private zones to create as child of var.defaults.environment_dns_zone. | list(string) | | [] |
+| [essential_contacts](variables.tf#L69) | Email contacts to be used for billing and GCP notifications. | list(string) | | [] |
+| [folder_id](variables.tf#L75) | Folder ID for the folder where the project will be created. | string | | null |
+| [group_iam](variables.tf#L81) | Custom IAM settings in group => [role] format. | map(list(string)) | | {} |
+| [group_iam_additive](variables.tf#L87) | Custom additive IAM settings in group => [role] format. | map(list(string)) | | {} |
+| [iam](variables.tf#L93) | Custom IAM settings in role => [principal] format. | map(list(string)) | | {} |
+| [iam_additive](variables.tf#L99) | Custom additive IAM settings in role => [principal] format. | map(list(string)) | | {} |
+| [kms_service_agents](variables.tf#L105) | KMS IAM configuration in as service => [key]. | map(list(string)) | | {} |
+| [labels](variables.tf#L111) | Labels to be assigned at project level. | map(string) | | {} |
+| [org_policies](variables.tf#L117) | Org-policy overrides at project level. | map(object({…})) | | {} |
+| [service_accounts](variables.tf#L171) | Service accounts to be created, and roles assigned them on the project. | map(list(string)) | | {} |
+| [service_accounts_additive](variables.tf#L177) | Service accounts to be created, and roles assigned them on the project additively. | map(list(string)) | | {} |
+| [service_accounts_iam](variables.tf#L183) | IAM bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string))) | | {} |
+| [service_accounts_iam_additive](variables.tf#L190) | IAM additive bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string))) | | {} |
+| [service_identities_iam](variables.tf#L197) | Custom IAM settings for service identities in service => [role] format. | map(list(string)) | | {} |
+| [service_identities_iam_additive](variables.tf#L204) | Custom additive IAM settings for service identities in service => [role] format. | map(list(string)) | | {} |
+| [services](variables.tf#L211) | Services to be enabled for the project. | list(string) | | [] |
+| [vpc](variables.tf#L218) | VPC configuration for the project. | object({…}) | | null |
## Outputs
diff --git a/blueprints/factories/project-factory/main.tf b/blueprints/factories/project-factory/main.tf
index f6b2a797..518d5a69 100644
--- a/blueprints/factories/project-factory/main.tf
+++ b/blueprints/factories/project-factory/main.tf
@@ -180,6 +180,7 @@ module "project" {
source = "../../../modules/project"
billing_account = local.billing_account_id
name = var.project_id
+ descriptive_name = var.descriptive_name
prefix = var.prefix
contacts = { for c in local.essential_contacts : c => ["ALL"] }
iam = local.iam
diff --git a/blueprints/factories/project-factory/variables.tf b/blueprints/factories/project-factory/variables.tf
index 0ece0f04..3aa3fa36 100644
--- a/blueprints/factories/project-factory/variables.tf
+++ b/blueprints/factories/project-factory/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -54,6 +54,12 @@ variable "defaults" {
default = null
}
+variable "descriptive_name" {
+ description = "Name of the project name. Used for project name instead of `name` variable."
+ type = string
+ default = null
+}
+
variable "dns_zones" {
description = "DNS private zones to create as child of var.defaults.environment_dns_zone."
type = list(string)
diff --git a/blueprints/gke/multitenant-fleet/README.md b/blueprints/gke/multitenant-fleet/README.md
index 1e09afaa..cadcf410 100644
--- a/blueprints/gke/multitenant-fleet/README.md
+++ b/blueprints/gke/multitenant-fleet/README.md
@@ -4,7 +4,7 @@ This blueprint presents an opinionated architecture to handle multiple homogeneo
The pattern used in this design is useful, for blueprint, in cases where multiple clusters host/support the same workloads, such as in the case of a multi-regional deployment. Furthermore, combined with Anthos Config Sync and proper RBAC, this architecture can be used to host multiple tenants (e.g. teams, applications) sharing the clusters.
-This blueprint is used as part of the [FAST GKE stage](../../../fast/stages/03-gke-multitenant/) but it can also be used independently if desired.
+This blueprint is used as part of the [FAST GKE stage](../../../fast/stages/3-gke-multitenant/) but it can also be used independently if desired.
diff --git a/blueprints/networking/__need_fixing/nginx-reverse-proxy-cluster/versions.tf b/blueprints/networking/__need_fixing/nginx-reverse-proxy-cluster/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/__need_fixing/nginx-reverse-proxy-cluster/versions.tf
+++ b/blueprints/networking/__need_fixing/nginx-reverse-proxy-cluster/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/__need_fixing/onprem-google-access-dns/versions.tf b/blueprints/networking/__need_fixing/onprem-google-access-dns/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/__need_fixing/onprem-google-access-dns/versions.tf
+++ b/blueprints/networking/__need_fixing/onprem-google-access-dns/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/decentralized-firewall/versions.tf b/blueprints/networking/decentralized-firewall/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/decentralized-firewall/versions.tf
+++ b/blueprints/networking/decentralized-firewall/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/filtering-proxy-psc/versions.tf b/blueprints/networking/filtering-proxy-psc/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/filtering-proxy-psc/versions.tf
+++ b/blueprints/networking/filtering-proxy-psc/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/filtering-proxy/versions.tf b/blueprints/networking/filtering-proxy/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/filtering-proxy/versions.tf
+++ b/blueprints/networking/filtering-proxy/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/hub-and-spoke-peering/versions.tf b/blueprints/networking/hub-and-spoke-peering/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/hub-and-spoke-peering/versions.tf
+++ b/blueprints/networking/hub-and-spoke-peering/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/hub-and-spoke-vpn/README.md b/blueprints/networking/hub-and-spoke-vpn/README.md
index 5f596142..bdf877c7 100644
--- a/blueprints/networking/hub-and-spoke-vpn/README.md
+++ b/blueprints/networking/hub-and-spoke-vpn/README.md
@@ -7,7 +7,7 @@ A few additional features are also shown:
- [custom BGP advertisements](https://cloud.google.com/router/docs/how-to/advertising-overview) to implement transitivity between spokes
- [VPC Global Routing](https://cloud.google.com/network-connectivity/docs/router/how-to/configuring-routing-mode) to leverage a regional set of VPN gateways in different regions as next hops (used here for illustrative/study purpose, not usually done in real life)
-The blueprint has been purposefully kept simple to show how to use and wire the VPC and VPN-HA modules together, and so that it can be used as a basis for experimentation. For a more complex scenario that better reflects real-life usage, including [Shared VPC](https://cloud.google.com/vpc/docs/shared-vpc) and [DNS cross-project binding](https://cloud.google.com/dns/docs/zones/cross-project-binding) please refer to the [FAST network stage](../../../fast/stages/02-networking-vpn/).
+The blueprint has been purposefully kept simple to show how to use and wire the VPC and VPN-HA modules together, and so that it can be used as a basis for experimentation. For a more complex scenario that better reflects real-life usage, including [Shared VPC](https://cloud.google.com/vpc/docs/shared-vpc) and [DNS cross-project binding](https://cloud.google.com/dns/docs/zones/cross-project-binding) please refer to the [FAST network stage](../../../fast/stages/2-networking-b-vpn/).
This is the high level diagram of this blueprint:
diff --git a/blueprints/networking/hub-and-spoke-vpn/versions.tf b/blueprints/networking/hub-and-spoke-vpn/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/hub-and-spoke-vpn/versions.tf
+++ b/blueprints/networking/hub-and-spoke-vpn/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/ilb-next-hop/versions.tf b/blueprints/networking/ilb-next-hop/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/ilb-next-hop/versions.tf
+++ b/blueprints/networking/ilb-next-hop/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/private-cloud-function-from-onprem/versions.tf b/blueprints/networking/private-cloud-function-from-onprem/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/private-cloud-function-from-onprem/versions.tf
+++ b/blueprints/networking/private-cloud-function-from-onprem/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/networking/shared-vpc-gke/versions.tf b/blueprints/networking/shared-vpc-gke/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/networking/shared-vpc-gke/versions.tf
+++ b/blueprints/networking/shared-vpc-gke/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/blueprints/third-party-solutions/openshift/tf/versions.tf b/blueprints/third-party-solutions/openshift/tf/versions.tf
index 4900174a..08492c6f 100644
--- a/blueprints/third-party-solutions/openshift/tf/versions.tf
+++ b/blueprints/third-party-solutions/openshift/tf/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/default-versions.tf b/default-versions.tf
index 4900174a..08492c6f 100644
--- a/default-versions.tf
+++ b/default-versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/diagram.svg b/diagram.svg
new file mode 100644
index 00000000..689adf24
--- /dev/null
+++ b/diagram.svg
@@ -0,0 +1,293 @@
+
\ No newline at end of file
diff --git a/fast/README.md b/fast/README.md
index e35a4837..7459c870 100644
--- a/fast/README.md
+++ b/fast/README.md
@@ -12,7 +12,7 @@ Fabric FAST was initially conceived to help enterprises quickly set up a GCP org
### Contracts and stages
-FAST uses the concept of stages, which individually perform precise tasks but, taken together, build a functional, ready-to-use GCP organization. More importantly, stages are modeled around the security boundaries that typically appear in mature organizations. This arrangement allows delegating ownership of each stage to the team responsible for the types of resources it manages. For example, as its name suggests, the networking stage sets up all the networking elements and is usually the responsibility of a dedicated networking team within the organization.
+FAST uses the concept of stages, which individually perform precise tasks but taken together build a functional, ready-to-use GCP organization. More importantly, stages are modeled around the security boundaries that typically appear in mature organizations. This arrangement allows delegating ownership of each stage to the team responsible for the types of resources it manages. For example, as its name suggests, the networking stage sets up all the networking elements and is usually the responsibility of a dedicated networking team within the organization.
From the perspective of FAST's overall design, stages also work as contacts or interfaces, defining a set of pre-requisites and inputs required to perform their designed task and generating outputs needed by other stages lower in the chain. The diagram below shows the relationships between stages.
@@ -20,7 +20,7 @@ From the perspective of FAST's overall design, stages also work as contacts or i
+ 
+
+ 
+
github_actions_secret · github_repository · github_repository_deploy_key · github_repository_file · tls_private_key |
+| [outputs.tf](./outputs.tf) | Module outputs. | |
+| [providers.tf](./providers.tf) | Provider configuration. | |
+| [variables.tf](./variables.tf) | Module variables. | |
+
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---:|:---:|:---:|
+| [organization](variables.tf#L50) | GitHub organization. | string | ✓ | |
+| [commmit_config](variables.tf#L17) | Configure commit metadata. | object({…}) | | {} |
+| [modules_config](variables.tf#L28) | Configure access to repository module via key, and replacement for modules sources in stage repositories. | object({…}) | | null |
+| [repositories](variables.tf#L55) | Repositories to create. | map(object({…})) | | {} |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [clone](outputs.tf#L17) | Clone repository commands. | |
+
+
diff --git a/fast/extras/00-cicd-github/cicd-versions.tf b/fast/extras/0-cicd-github/cicd-versions.tf
similarity index 96%
rename from fast/extras/00-cicd-github/cicd-versions.tf
rename to fast/extras/0-cicd-github/cicd-versions.tf
index 09f544cb..830f1e48 100644
--- a/fast/extras/00-cicd-github/cicd-versions.tf
+++ b/fast/extras/0-cicd-github/cicd-versions.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/fast/extras/00-cicd-github/github_token.png b/fast/extras/0-cicd-github/github_token.png
similarity index 100%
rename from fast/extras/00-cicd-github/github_token.png
rename to fast/extras/0-cicd-github/github_token.png
diff --git a/fast/extras/00-cicd-github/main.tf b/fast/extras/0-cicd-github/main.tf
similarity index 72%
rename from fast/extras/00-cicd-github/main.tf
rename to fast/extras/0-cicd-github/main.tf
index ac6028c1..d91ab970 100644
--- a/fast/extras/00-cicd-github/main.tf
+++ b/fast/extras/0-cicd-github/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,9 +15,6 @@
*/
locals {
- _modules_repository = [
- for k, v in var.repositories : local.repositories[k] if v.has_modules
- ]
_repository_files = flatten([
for k, v in var.repositories : [
for f in concat(
@@ -30,12 +27,12 @@ locals {
}
] if v.populate_from != null
])
- modules_ref = var.modules_ref == null ? "" : "?ref=${var.modules_ref}"
- modules_repository = (
- length(local._modules_repository) > 0
- ? local._modules_repository.0
- : null
+ modules_ref = (
+ try(var.modules_config.source_ref, null) == null
+ ? ""
+ : "?ref=${var.modules_config.source_ref}"
)
+ modules_repo = try(var.modules_config.repository_name, null)
repositories = {
for k, v in var.repositories :
k => v.create_options == null ? k : github_repository.default[k].name
@@ -56,6 +53,15 @@ locals {
name = "templates/providers.tf.tpl"
}
if v.populate_from != null
+ },
+ {
+ for k, v in var.repositories :
+ "${k}/templates/workflow-github.yaml" => {
+ repository = k
+ file = "../../assets/templates/workflow-github.yaml"
+ name = "templates/workflow-github.yaml"
+ }
+ if v.populate_from != null
}
)
}
@@ -96,41 +102,49 @@ resource "github_repository" "default" {
}
resource "tls_private_key" "default" {
- count = local.modules_repository != null ? 1 : 0
algorithm = "ED25519"
}
resource "github_repository_deploy_key" "default" {
- count = local.modules_repository == null ? 0 : 1
+ count = (
+ try(var.modules_config.key_config.create_key, null) == true ? 1 : 0
+ )
title = "Modules repository access"
- repository = local.modules_repository
- key = tls_private_key.default.0.public_key_openssh
- read_only = true
+ repository = local.modules_repo
+ key = (
+ try(var.modules_config.key_config.keypair_path, null) == null
+ ? tls_private_key.default.public_key_openssh
+ : file(pathexpand("${var.modules_config.key_config.keypair_path}.pub"))
+ )
+ read_only = true
}
resource "github_actions_secret" "default" {
- for_each = local.modules_repository == null ? {} : {
- for k, v in local.repositories :
- k => v if k != local.modules_repository
- }
- repository = each.key
- secret_name = "CICD_MODULES_KEY"
- plaintext_value = tls_private_key.default.0.private_key_openssh
+ for_each = (
+ try(var.modules_config.key_config.create_secrets, null) == true
+ ? local.repositories
+ : {}
+ )
+ repository = each.key
+ secret_name = "CICD_MODULES_KEY"
+ plaintext_value = (
+ try(var.modules_config.key_config.keypair_path, null) == null
+ ? tls_private_key.default.private_key_openssh
+ : file(pathexpand("${var.modules_config.key_config.keypair_path}"))
+ )
}
resource "github_repository_file" "default" {
- for_each = (
- local.modules_repository == null ? {} : local.repository_files
- )
+ for_each = local.modules_repo == null ? {} : local.repository_files
repository = local.repositories[each.value.repository]
branch = "main"
file = each.value.name
content = (
- endswith(each.value.name, ".tf") && local.modules_repository != null
+ endswith(each.value.name, ".tf") && local.modules_repo != null
? replace(
file(each.value.file),
"/source\\s*=\\s*\"../../../modules/([^/\"]+)\"/",
- "source = \"git@github.com:${var.organization}/${local.modules_repository}.git//$1${local.modules_ref}\"" # "
+ "source = \"git@github.com:${local.modules_repo}.git//$1${local.modules_ref}\"" # "
)
: file(each.value.file)
)
diff --git a/fast/extras/00-cicd-github/outputs.tf b/fast/extras/0-cicd-github/outputs.tf
similarity index 96%
rename from fast/extras/00-cicd-github/outputs.tf
rename to fast/extras/0-cicd-github/outputs.tf
index cb580e1f..61b5ffbc 100644
--- a/fast/extras/00-cicd-github/outputs.tf
+++ b/fast/extras/0-cicd-github/outputs.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/fast/extras/00-cicd-github/providers.tf b/fast/extras/0-cicd-github/providers.tf
similarity index 95%
rename from fast/extras/00-cicd-github/providers.tf
rename to fast/extras/0-cicd-github/providers.tf
index 29be30ae..a7ccb32d 100644
--- a/fast/extras/00-cicd-github/providers.tf
+++ b/fast/extras/0-cicd-github/providers.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/fast/extras/00-cicd-github/variables.tf b/fast/extras/0-cicd-github/variables.tf
similarity index 75%
rename from fast/extras/00-cicd-github/variables.tf
rename to fast/extras/0-cicd-github/variables.tf
index 0d9cb7fd..8e5d0832 100644
--- a/fast/extras/00-cicd-github/variables.tf
+++ b/fast/extras/0-cicd-github/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,10 +25,26 @@ variable "commmit_config" {
nullable = false
}
-variable "modules_ref" {
- description = "Optional git ref used in module sources."
- type = string
- default = null
+variable "modules_config" {
+ description = "Configure access to repository module via key, and replacement for modules sources in stage repositories."
+ type = object({
+ repository_name = string
+ source_ref = optional(string)
+ key_config = optional(object({
+ create_key = optional(bool, false)
+ create_secrets = optional(bool, false)
+ keypair_path = optional(string)
+ }), {})
+ })
+ default = null
+ validation {
+ condition = (
+ var.modules_config == null
+ ||
+ try(var.modules_config.repository_name, null) != null
+ )
+ error_message = "Modules configuration requires a modules repository name."
+ }
}
variable "organization" {
@@ -63,7 +79,6 @@ variable "repositories" {
}), {})
visibility = optional(string, "private")
}))
- has_modules = optional(bool, false)
populate_from = optional(string)
}))
default = {}
diff --git a/fast/extras/00-cicd-github/README.md b/fast/extras/00-cicd-github/README.md
deleted file mode 100644
index acf249bc..00000000
--- a/fast/extras/00-cicd-github/README.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# FAST GitHub repository management
-
-This small extra stage allows creation and management of GitHub repositories used to host FAST stage code, including initial population of files and rewriting of module sources.
-
-This stage is designed for quick repository creation in a GitHub organization, and is not suited for medium or long-term repository management especially if you enable initial population of files.
-
-## Initial population caveats
-
-Initial file population of repositories is controlled via the `populate_from` attribute, and needs a bit of care:
-
-- never run this stage with the same variables used for population once the repository starts being used, as **Terraform will manage file state and revert any changes at each apply**, which is probably not what you want.
-- initial population of the modules repository is discouraged, as the number of resulting files Terraform needs to manage is very close to the GitHub hourly limit for their API, it's much easier to populate modules via regular git commands
-
-The scenario for which this stage has been designed is one-shot creation and/or population of stage repositories, running it multiple times with different variables and Terraform states if incremental creation is needed for subsequent FAST stages (e.g. GKE, data platform, etc.).
-
-Once initial population is done, you need to manually push to the repository
-
-- the `.tfvars` file with custom variable values for your stages
-- the workflow configuration file generated by FAST stages
-
-## GitHub provider credentials
-
-A [GitHub token](https://github.com/settings/tokens) is needed to authenticate against their API. The token needs organization-level permissions, like shown in this screenshot:
-
-
-
-
github_actions_secret · github_repository · github_repository_deploy_key · github_repository_file · tls_private_key |
-| [outputs.tf](./outputs.tf) | Module outputs. | |
-| [providers.tf](./providers.tf) | Provider configuration. | |
-| [variables.tf](./variables.tf) | Module variables. | |
-
-## Variables
-
-| name | description | type | required | default |
-|---|---|:---:|:---:|:---:|
-| [organization](variables.tf#L34) | GitHub organization. | string | ✓ | |
-| [commmit_config](variables.tf#L17) | Configure commit metadata. | object({…}) | | {} |
-| [modules_ref](variables.tf#L28) | Optional git ref used in module sources. | string | | null |
-| [repositories](variables.tf#L39) | Repositories to create. | map(object({…})) | | {} |
-
-## Outputs
-
-| name | description | sensitive |
-|---|---|:---:|
-| [clone](outputs.tf#L17) | Clone repository commands. | |
-
-
diff --git a/fast/extras/README.md b/fast/extras/README.md
index 121fa4b0..9213224c 100644
--- a/fast/extras/README.md
+++ b/fast/extras/README.md
@@ -2,4 +2,4 @@
This folder contains additional helper stages for FAST, which can be used to simplify specific operational tasks:
-- [GitHub repository management](./00-cicd-github/)
+- [GitHub repository management](./0-cicd-github/)
diff --git a/fast/stage-links.sh b/fast/stage-links.sh
new file mode 100755
index 00000000..79d1973f
--- /dev/null
+++ b/fast/stage-links.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -eq 0 ]; then
+ echo "Error: no folder or GCS bucket specified. Use -h or --help for usage."
+ exit 1
+fi
+
+if [[ "$1" == "-h" || "$1" == "--help" ]]; then
+ cat <+ additive, • conditional.
+
+## Organization [org_id #0]
+
+| members | roles |
+|---|---|
+|tn0-admins+•+|
+|tn0-gke-dev-0+•|
+|tn0-gke-prod-0+•|
+|tn0-networking-0+•|
+|tn0-pf-dev-0+•|
+|tn0-pf-prod-0+•|
+|tn0-resman-0+•|
+|tn0-sandbox-0+•|
+|tn0-security-0+•|
+|tn0-teams-0+•|
+
+## Folder test tenant 0 [#1]
+
+| members | roles |
+|---|---|
+|tn0-admins+|
+
+## Project tn0-audit-logs-0
+
+| members | roles |
+|---|---|
+|f260055713332-284719+•|
+|prod-resman-0+|
+|prod-resman-0iam-service-account | google_organization_iam_member |
+| [automation.tf](./automation.tf) | Tenant automation project and resources. | gcs · iam-service-account · project | |
+| [billing.tf](./billing.tf) | Billing roles for standalone billing accounts. | | google_billing_account_iam_member |
+| [cicd.tf](./cicd.tf) | Workload Identity Federation configurations for CI/CD. | iam-service-account · source-repository | |
+| [identity-providers.tf](./identity-providers.tf) | Workload Identity Federation provider definitions. | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider |
+| [log-export.tf](./log-export.tf) | Audit log project and sink. | bigquery-dataset · gcs · logging-bucket · project · pubsub | |
+| [main.tf](./main.tf) | Module-level locals and resources. | folder | |
+| [organization.tf](./organization.tf) | Organization tag and conditional IAM grant. | organization | google_organization_iam_member · google_tags_tag_value_iam_member |
+| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file |
+| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object |
+| [outputs.tf](./outputs.tf) | Module outputs. | | |
+| [variables.tf](./variables.tf) | Module variables. | | |
+
+## Variables
+
+| name | description | type | required | default | producer |
+|---|---|:---:|:---:|:---:|:---:|
+| [automation](variables.tf#L20) | Automation resources created by the organization-level bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L38) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [organization](variables.tf#L194) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L210) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [tag_keys](variables.tf#L233) | Organization tag keys. | object({…}) | ✓ | | 1-resman |
+| [tag_names](variables.tf#L244) | Customized names for resource management tags. | object({…}) | ✓ | | 1-resman |
+| [tag_values](variables.tf#L255) | Organization resource management tag values. | map(string) | ✓ | | 1-resman |
+| [tenant_config](variables.tf#L262) | Tenant configuration. Short name must be 4 characters or less. | object({…}) | ✓ | | |
+| [cicd_repositories](variables.tf#L51) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | |
+| [custom_roles](variables.tf#L97) | Custom roles defined at the organization level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [fast_features](variables.tf#L107) | Selective control for top-level FAST features. | object({…}) | | {} | 0-bootstrap |
+| [federated_identity_providers](variables.tf#L121) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | |
+| [group_iam](variables.tf#L135) | Tenant-level custom group IAM settings in group => [roles] format. | map(list(string)) | | {} | |
+| [iam](variables.tf#L141) | Tenant-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | |
+| [iam_additive](variables.tf#L147) | Tenant-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | |
+| [locations](variables.tf#L153) | Optional locations for GCS, BigQuery, and logging buckets created here. These are the defaults set at the organization level, and can be overridden via the tenant config variable. | object({…}) | | {…} | 0-bootstrap |
+| [log_sinks](variables.tf#L173) | Tenant-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | |
+| [outputs_location](variables.tf#L204) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | |
+| [project_parent_ids](variables.tf#L220) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the tenant folder as parent. | object({…}) | | {…} | |
+| [test_principal](variables.tf#L302) | Used when testing to bypass the data source returning the current identity. | string | | null | |
+
+## Outputs
+
+| name | description | sensitive | consumers |
+|---|---|:---:|---|
+| [cicd_workflows](outputs.tf#L102) | CI/CD workflows for tenant bootstrap and resource management stages. | ✓ | |
+| [federated_identity](outputs.tf#L108) | Workload Identity Federation pool and providers. | | |
+| [provider](outputs.tf#L118) | Terraform provider file for tenant resource management stage. | ✓ | stage-01 |
+| [tenant_resources](outputs.tf#L125) | Tenant-level resources. | | |
+| [tfvars](outputs.tf#L136) | Terraform variable files for the following tenant stages. | ✓ | |
+
+
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf b/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf
new file mode 100644
index 00000000..cae14093
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf
@@ -0,0 +1,135 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Tenant automation stage 2 and 3 service accounts.
+
+locals {
+ branch_sas = {
+ dp-dev = {
+ condition = join(" && ", [
+ "resource.matchTag('${local.tag_keys.context}', 'data')",
+ "resource.matchTag('${local.tag_keys.environment}', 'development')"
+ ])
+ description = "data platform dev"
+ flag = "data_platform"
+ }
+ dp-prod = {
+ condition = join(" && ", [
+ "resource.matchTag('${local.tag_keys.context}', 'data')",
+ "resource.matchTag('${local.tag_keys.environment}', 'production')"
+ ])
+ description = "data platform prod"
+ flag = "data_platform"
+ }
+ gke-dev = {
+ condition = join(" && ", [
+ "resource.matchTag('${local.tag_keys.context}', 'gke')",
+ "resource.matchTag('${local.tag_keys.environment}', 'development')"
+ ])
+ description = "GKE dev"
+ flag = "gke"
+ }
+ gke-prod = {
+ condition = join(" && ", [
+ "resource.matchTag('${local.tag_keys.context}', 'gke')",
+ "resource.matchTag('${local.tag_keys.environment}', 'production')"
+ ])
+ description = "GKE prod"
+ flag = "gke"
+ }
+ networking = {
+ condition = "resource.matchTag('${local.tag_keys.context}', 'networking')"
+ description = "networking"
+ flag = "-"
+ }
+ pf-dev = {
+ condition = "resource.matchTag('${local.tag_keys.environment}', 'development')"
+ description = "project factory dev"
+ flag = "project_factory"
+ }
+ pf-prod = {
+ condition = "resource.matchTag('${local.tag_keys.environment}', 'production')"
+ description = "project factory prod"
+ flag = "project_factory"
+ }
+ sandbox = {
+ condition = "resource.matchTag('${local.tag_keys.context}', 'sandbox')"
+ description = "sandbox"
+ flag = "sandbox"
+ }
+ security = {
+ condition = "resource.matchTag('${local.tag_keys.context}', 'security')"
+ description = "security"
+ flag = "-"
+ }
+ teams = {
+ condition = "resource.matchTag('${local.tag_keys.context}', 'teams')"
+ description = "teams"
+ flag = "teams"
+ }
+ }
+}
+
+module "automation-tf-resman-sa-stage2-3" {
+ source = "../../../modules/iam-service-account"
+ for_each = {
+ for k, v in local.branch_sas :
+ k => v if lookup(local.fast_features, v.flag, true)
+ }
+ project_id = module.automation-project.project_id
+ name = "${each.key}-0"
+ display_name = "Terraform ${each.value.description} service account."
+ prefix = local.prefix
+ iam_billing_roles = !var.billing_account.is_org_level ? {
+ (var.billing_account.id) = [
+ "roles/billing.user", "roles/billing.costsManager"
+ ]
+ } : {}
+ iam_organization_roles = var.billing_account.is_org_level ? {
+ (var.organization.id) = [
+ "roles/billing.user", "roles/billing.costsManager"
+ ]
+ } : {}
+}
+
+# assign org policy admin with a tag-based condition to stage 2 and 3 SAs
+
+resource "google_organization_iam_member" "org_policy_admin_stage2_3" {
+ for_each = {
+ for k, v in module.automation-tf-resman-sa-stage2-3 : k => v.iam_email
+ }
+ org_id = var.organization.id
+ role = "roles/orgpolicy.policyAdmin"
+ member = each.value
+ condition {
+ title = "org_policy_tag_${var.tenant_config.short_name}_${each.key}_scoped"
+ description = join("", [
+ "Org policy tag scoped grant for tenant ${var.tenant_config.short_name} ",
+ local.branch_sas[each.key].description
+ ])
+ expression = join(" && ", [
+ local.iam_tenant_condition, local.branch_sas[each.key].condition
+ ])
+ }
+}
+
+# assign custom tenant network admin role to networking SA
+
+resource "google_organization_iam_member" "tenant_network_admin" {
+ org_id = var.organization.id
+ role = var.custom_roles.tenant_network_admin
+ member = module.automation-tf-resman-sa-stage2-3["networking"].iam_email
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/automation.tf b/fast/stages-multitenant/0-bootstrap-tenant/automation.tf
new file mode 100644
index 00000000..9684e7ca
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/automation.tf
@@ -0,0 +1,141 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Tenant automation project and resources.
+
+module "automation-project" {
+ source = "../../../modules/project"
+ billing_account = var.billing_account.id
+ name = "iac-core-0"
+ parent = coalesce(
+ var.project_parent_ids.automation,
+ module.tenant-folder.id
+ )
+ prefix = local.prefix
+ # human (groups) IAM bindings
+ group_iam = {
+ (local.groups.gcp-admins) = [
+ "roles/iam.serviceAccountAdmin",
+ "roles/iam.serviceAccountTokenCreator",
+ ]
+ (local.groups.gcp-admins) = [
+ "roles/iam.serviceAccountTokenCreator",
+ "roles/iam.workloadIdentityPoolAdmin"
+ ]
+ }
+ # machine (service accounts) IAM bindings
+ iam = {
+ "roles/owner" = [
+ module.automation-tf-resman-sa.iam_email,
+ "serviceAccount:${local.resman_sa}"
+ ]
+ "roles/cloudbuild.builds.editor" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/iam.serviceAccountAdmin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/iam.workloadIdentityPoolAdmin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/source.admin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/storage.admin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ }
+ services = [
+ "accesscontextmanager.googleapis.com",
+ "bigquery.googleapis.com",
+ "bigqueryreservation.googleapis.com",
+ "bigquerystorage.googleapis.com",
+ "billingbudgets.googleapis.com",
+ "cloudbilling.googleapis.com",
+ "cloudbuild.googleapis.com",
+ "cloudkms.googleapis.com",
+ "cloudresourcemanager.googleapis.com",
+ "container.googleapis.com",
+ "compute.googleapis.com",
+ "container.googleapis.com",
+ "essentialcontacts.googleapis.com",
+ "iam.googleapis.com",
+ "iamcredentials.googleapis.com",
+ "orgpolicy.googleapis.com",
+ "pubsub.googleapis.com",
+ "servicenetworking.googleapis.com",
+ "serviceusage.googleapis.com",
+ "sourcerepo.googleapis.com",
+ "stackdriver.googleapis.com",
+ "storage-component.googleapis.com",
+ "storage.googleapis.com",
+ "sts.googleapis.com"
+ ]
+}
+
+# output files bucket
+
+module "automation-tf-output-gcs" {
+ source = "../../../modules/gcs"
+ project_id = module.automation-project.project_id
+ name = "iac-core-outputs-0"
+ prefix = local.prefix
+ location = local.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+}
+
+# resource management stage bucket and service account
+
+module "automation-tf-resman-gcs" {
+ source = "../../../modules/gcs"
+ project_id = module.automation-project.project_id
+ name = "iac-core-resman-0"
+ prefix = local.prefix
+ location = local.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [module.automation-tf-resman-sa.iam_email]
+ }
+}
+
+module "automation-tf-resman-sa" {
+ source = "../../../modules/iam-service-account"
+ project_id = module.automation-project.project_id
+ name = "resman-0"
+ display_name = "Terraform stage 1 resman service account."
+ prefix = local.prefix
+ # allow SA used by CI/CD workflow to impersonate this SA
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.automation-tf-cicd-sa-resman["0"].iam_email, null)
+ ])
+ }
+ iam_billing_roles = !var.billing_account.is_org_level ? {
+ (var.billing_account.id) = [
+ "roles/billing.admin", "roles/billing.costsManager"
+ ]
+ } : {}
+ iam_organization_roles = var.billing_account.is_org_level ? {
+ (var.organization.id) = [
+ "roles/billing.admin", "roles/billing.costsManager"
+ ]
+ } : {}
+ iam_storage_roles = {
+ (module.automation-tf-output-gcs.name) = ["roles/storage.admin"]
+ }
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/billing.tf b/fast/stages-multitenant/0-bootstrap-tenant/billing.tf
new file mode 100644
index 00000000..77c26b91
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/billing.tf
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Billing roles for standalone billing accounts.
+
+# service account billing roles are in the SA module in automation.tf
+
+resource "google_billing_account_iam_member" "billing_ext_admin" {
+ for_each = toset(var.billing_account.is_org_level ? [] : [
+ "group:${local.groups.gcp-admins}",
+ module.automation-tf-resman-sa.iam_email
+ ])
+ billing_account_id = var.billing_account.id
+ role = "roles/billing.admin"
+ member = each.key
+}
+
+resource "google_billing_account_iam_member" "billing_ext_cost_manager" {
+ for_each = toset(var.billing_account.is_org_level ? [] : [
+ "group:${local.groups.gcp-admins}",
+ module.automation-tf-resman-sa.iam_email
+ ])
+ billing_account_id = var.billing_account.id
+ role = "roles/billing.costsManager"
+ member = each.key
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf b/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf
new file mode 100644
index 00000000..a25215af
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf
@@ -0,0 +1,223 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Workload Identity Federation configurations for CI/CD.
+
+locals {
+ _file_prefix = "tenants/${var.tenant_config.short_name}"
+ # derive identity pool names from identity providers for easy reference
+ cicd_identity_pools = {
+ for k, v in local.cicd_identity_providers :
+ k => split("/providers/", v.name)[0]
+ }
+ # merge org-level and tenant-level identity providers
+ cicd_identity_providers = merge(
+ var.automation.federated_identity_providers,
+ {
+ for k, v in google_iam_workload_identity_pool_provider.default :
+ k => {
+ issuer = local.identity_providers[k].issuer
+ issuer_uri = local.identity_providers[k].issuer_uri
+ name = v.name
+ principal_tpl = local.identity_providers[k].principal_tpl
+ principalset_tpl = local.identity_providers[k].principalset_tpl
+ }
+ })
+ # filter CI/CD repositories to only keep valid ones
+ cicd_repositories = {
+ for k, v in coalesce(var.cicd_repositories, {}) : k => v
+ if(
+ v != null
+ &&
+ (
+ try(v.type, null) == "sourcerepo"
+ ||
+ contains(
+ keys(local.cicd_identity_providers),
+ coalesce(try(v.identity_provider, null), ":")
+ )
+ )
+ &&
+ fileexists(
+ format("${path.module}/templates/workflow-%s.yaml", try(v.type, ""))
+ )
+ )
+ }
+}
+
+# tenant bootstrap runs in the org scope and uses top-level automation project
+
+module "automation-tf-cicd-repo-bootstrap" {
+ source = "../../../modules/source-repository"
+ for_each = {
+ for k, v in local.cicd_repositories : 0 => v
+ if k == "bootstrap" && try(v.type, null) == "sourcerepo"
+ }
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = [
+ local.resman_sa
+ ]
+ "roles/source.reader" = [
+ module.automation-tf-cicd-sa-bootstrap["0"].iam_email
+ ]
+ }
+ triggers = {
+ "fast-${var.tenant_config.short_name}-0-bootstrap" = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
+ service_account = module.automation-tf-cicd-sa-bootstrap["0"].id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+}
+
+module "automation-tf-cicd-sa-bootstrap" {
+ source = "../../../modules/iam-service-account"
+ for_each = {
+ for k, v in local.cicd_repositories : 0 => v
+ if k == "bootstrap" && try(v.type, null) != null
+ }
+ project_id = var.automation.project_id
+ name = "bootstrap-1"
+ display_name = "Terraform CI/CD ${var.tenant_config.short_name} bootstrap."
+ prefix = local.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {}
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
+
+module "automation-tf-org-resman-sa" {
+ source = "../../../modules/iam-service-account"
+ for_each = {
+ for k, v in local.cicd_repositories : 0 => v
+ if k == "bootstrap" && try(v.type, null) != null
+ }
+ project_id = var.automation.project_id
+ name = local.resman_sa
+ service_account_create = false
+ iam_additive = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.automation-tf-cicd-sa-bootstrap["0"].iam_email, null)
+ ])
+ }
+}
+
+# tenant resman runs in the tenant scope and uses its own automation project
+
+module "automation-tf-cicd-repo-resman" {
+ source = "../../../modules/source-repository"
+ for_each = {
+ for k, v in local.cicd_repositories : 0 => v
+ if k == "resman" && try(v.type, null) == "sourcerepo"
+ }
+ project_id = module.automation-project.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/source.reader" = [
+ module.automation-tf-cicd-sa-resman["0"].iam_email
+ ]
+ }
+ triggers = {
+ fast-1-resman = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
+ service_account = module.automation-tf-cicd-sa-resman["0"].id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+}
+
+module "automation-tf-cicd-sa-resman" {
+ source = "../../../modules/iam-service-account"
+ for_each = {
+ for k, v in local.cicd_repositories : 0 => v
+ if k == "resman" && try(v.type, null) != null
+ }
+ project_id = module.automation-project.project_id
+ name = "resman-1"
+ display_name = "Terraform CI/CD resman."
+ prefix = local.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {}
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (module.automation-project.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (module.automation-tf-output-gcs.name) = ["roles/storage.objectViewer"]
+ }
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/diagram.svg b/fast/stages-multitenant/0-bootstrap-tenant/diagram.svg
new file mode 100644
index 00000000..4090c7b0
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/diagram.svg
@@ -0,0 +1,597 @@
+
+
\ No newline at end of file
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf b/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf
new file mode 100644
index 00000000..3f8499b7
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Workload Identity Federation provider definitions.
+
+locals {
+ identity_providers = {
+ for k, v in var.federated_identity_providers : k => merge(
+ v,
+ lookup(local.identity_providers_defs, v.issuer, {})
+ )
+ }
+ identity_providers_defs = {
+ # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect
+ github = {
+ attribute_mapping = {
+ "google.subject" = "assertion.sub"
+ "attribute.sub" = "assertion.sub"
+ "attribute.actor" = "assertion.actor"
+ "attribute.repository" = "assertion.repository"
+ "attribute.repository_owner" = "assertion.repository_owner"
+ "attribute.ref" = "assertion.ref"
+ }
+ issuer_uri = "https://token.actions.githubusercontent.com"
+ principal_tpl = "principal://iam.googleapis.com/%s/subject/repo:%s:ref:refs/heads/%s"
+ principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
+ }
+ # https://docs.gitlab.com/ee/ci/cloud_services/index.html#how-it-works
+ gitlab = {
+ attribute_mapping = {
+ "google.subject" = "assertion.sub"
+ "attribute.sub" = "assertion.sub"
+ "attribute.environment" = "assertion.environment"
+ "attribute.environment_protected" = "assertion.environment_protected"
+ "attribute.namespace_id" = "assertion.namespace_id"
+ "attribute.namespace_path" = "assertion.namespace_path"
+ "attribute.pipeline_id" = "assertion.pipeline_id"
+ "attribute.pipeline_source" = "assertion.pipeline_source"
+ "attribute.project_id" = "assertion.project_id"
+ "attribute.project_path" = "assertion.project_path"
+ "attribute.repository" = "assertion.project_path"
+ "attribute.ref" = "assertion.ref"
+ "attribute.ref_protected" = "assertion.ref_protected"
+ "attribute.ref_type" = "assertion.ref_type"
+ }
+ allowed_audiences = ["https://gitlab.com"]
+ issuer_uri = "https://gitlab.com"
+ principal_tpl = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s"
+ principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s"
+ }
+ }
+}
+
+resource "google_iam_workload_identity_pool" "default" {
+ provider = google-beta
+ count = length(local.identity_providers) > 0 ? 1 : 0
+ project = module.automation-project.project_id
+ workload_identity_pool_id = "${var.prefix}-bootstrap"
+}
+
+resource "google_iam_workload_identity_pool_provider" "default" {
+ provider = google-beta
+ for_each = local.identity_providers
+ project = module.automation-project.project_id
+ workload_identity_pool_id = (
+ google_iam_workload_identity_pool.default.0.workload_identity_pool_id
+ )
+ workload_identity_pool_provider_id = "${var.prefix}-bootstrap-${each.key}"
+ attribute_condition = each.value.attribute_condition
+ attribute_mapping = each.value.attribute_mapping
+ oidc {
+ allowed_audiences = (
+ try(each.value.custom_settings.allowed_audiences, null) != null
+ ? each.value.custom_settings.allowed_audiences
+ : try(each.value.allowed_audiences, null)
+ )
+ issuer_uri = (
+ try(each.value.custom_settings.issuer_uri, null) != null
+ ? each.value.custom_settings.issuer_uri
+ : try(each.value.issuer_uri, null)
+ )
+ }
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/log-export.tf b/fast/stages-multitenant/0-bootstrap-tenant/log-export.tf
new file mode 100644
index 00000000..b0bf115a
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/log-export.tf
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Audit log project and sink.
+
+locals {
+ log_sink_destinations = merge(
+ # use the same dataset for all sinks with `bigquery` as destination
+ { for k, v in var.log_sinks : k => module.log-export-dataset.0 if v.type == "bigquery" },
+ # use the same gcs bucket for all sinks with `storage` as destination
+ { for k, v in var.log_sinks : k => module.log-export-gcs.0 if v.type == "storage" },
+ # use separate pubsub topics and logging buckets for sinks with
+ # destination `pubsub` and `logging`
+ module.log-export-pubsub,
+ module.log-export-logbucket
+ )
+ log_types = toset([for k, v in var.log_sinks : v.type])
+}
+
+module "log-export-project" {
+ source = "../../../modules/project"
+ billing_account = var.billing_account.id
+ name = "audit-logs-0"
+ parent = coalesce(
+ var.project_parent_ids.logging,
+ module.tenant-folder.id
+ )
+ prefix = local.prefix
+ iam = {
+ "roles/owner" = [
+ module.automation-tf-resman-sa.iam_email,
+ "serviceAccount:${local.resman_sa}"
+ ]
+ }
+ services = [
+ # "cloudresourcemanager.googleapis.com",
+ # "iam.googleapis.com",
+ # "serviceusage.googleapis.com",
+ "bigquery.googleapis.com",
+ "storage.googleapis.com",
+ "stackdriver.googleapis.com"
+ ]
+}
+
+# one log export per type, with conditionals to skip those not needed
+
+module "log-export-dataset" {
+ source = "../../../modules/bigquery-dataset"
+ count = contains(local.log_types, "bigquery") ? 1 : 0
+ project_id = module.log-export-project.project_id
+ id = "audit_export"
+ friendly_name = "Audit logs export."
+ location = var.locations.bq
+}
+
+module "log-export-gcs" {
+ source = "../../../modules/gcs"
+ count = contains(local.log_types, "storage") ? 1 : 0
+ project_id = module.log-export-project.project_id
+ name = "audit-logs-0"
+ prefix = local.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+}
+
+module "log-export-logbucket" {
+ source = "../../../modules/logging-bucket"
+ for_each = toset([for k, v in var.log_sinks : k if v.type == "logging"])
+ parent_type = "project"
+ parent = module.log-export-project.project_id
+ id = "audit-logs-${each.key}"
+ location = var.locations.logging
+}
+
+module "log-export-pubsub" {
+ source = "../../../modules/pubsub"
+ for_each = toset([for k, v in var.log_sinks : k if v.type == "pubsub"])
+ project_id = module.log-export-project.project_id
+ name = "audit-logs-${each.key}"
+ regions = var.locations.pubsub
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/main.tf b/fast/stages-multitenant/0-bootstrap-tenant/main.tf
new file mode 100644
index 00000000..3a150594
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/main.tf
@@ -0,0 +1,100 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+ gcs_storage_class = (
+ length(split("-", local.locations.gcs)) < 2
+ ? "MULTI_REGIONAL"
+ : "REGIONAL"
+ )
+ groups = {
+ for k, v in var.tenant_config.groups :
+ k => v == null ? null : "${v}@${var.organization.domain}"
+ }
+ fast_features = {
+ for k, v in var.tenant_config.fast_features :
+ k => v == null ? var.fast_features[k] : v
+ }
+ locations = {
+ for k, v in var.tenant_config.locations :
+ k => v == null || v == [] ? var.locations[k] : v
+ }
+ prefix = join("-", compact([var.prefix, var.tenant_config.short_name]))
+ resman_sa = (
+ var.test_principal == null
+ ? data.google_client_openid_userinfo.resman-sa.0.email
+ : var.test_principal
+ )
+}
+
+data "google_client_openid_userinfo" "resman-sa" {
+ count = var.test_principal == null ? 1 : 0
+}
+
+module "tenant-folder" {
+ source = "../../../modules/folder"
+ parent = "organizations/${var.organization.id}"
+ name = var.tenant_config.descriptive_name
+ logging_sinks = {
+ for name, attrs in var.log_sinks : name => {
+ bq_partitioned_table = attrs.type == "bigquery"
+ destination = local.log_sink_destinations[name].id
+ filter = attrs.filter
+ type = attrs.type
+ }
+ }
+ tag_bindings = {
+ tenant = try(
+ module.organization.tag_values["${var.tag_names.tenant}/${var.tenant_config.short_name}"].id,
+ null
+ )
+ }
+}
+
+module "tenant-folder-iam" {
+ source = "../../../modules/folder"
+ id = module.tenant-folder.id
+ folder_create = false
+ group_iam = merge(var.group_iam, {
+ (local.groups.gcp-admins) = [
+ "roles/logging.admin",
+ "roles/owner",
+ "roles/resourcemanager.folderAdmin",
+ "roles/resourcemanager.projectCreator",
+ "roles/compute.xpnAdmin"
+ ]
+ })
+ iam = merge(var.iam, {
+ "roles/compute.xpnAdmin" = [
+ module.automation-tf-resman-sa.iam_email,
+ module.automation-tf-resman-sa-stage2-3["networking"].iam_email
+ ]
+ "roles/logging.admin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/resourcemanager.folderAdmin" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/resourcemanager.projectCreator" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/owner" = [
+ module.automation-tf-resman-sa.iam_email
+ ]
+ })
+ iam_additive = var.iam_additive
+ depends_on = [module.automation-project]
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/organization.tf b/fast/stages-multitenant/0-bootstrap-tenant/organization.tf
new file mode 100644
index 00000000..46f8c0d4
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/organization.tf
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Organization tag and conditional IAM grant.
+
+locals {
+ iam_tenant_condition = "resource.matchTag('${local.tag_keys.tenant}', '${var.tenant_config.short_name}')"
+ tag_keys = {
+ for k, v in var.tag_names : k => "${var.organization.id}/${v}"
+ }
+}
+
+module "organization" {
+ source = "../../../modules/organization"
+ organization_id = "organizations/${var.organization.id}"
+ iam_additive = merge(
+ {
+ "roles/resourcemanager.organizationViewer" = [
+ "group:${local.groups.gcp-admins}"
+ ]
+ },
+ var.billing_account.is_org_level ? {
+ "roles/billing.admin" = [
+ "group:${local.groups.gcp-admins}",
+ module.automation-tf-resman-sa.iam_email
+ ]
+ "roles/billing.costsManager" = ["group:${local.groups.gcp-admins}"]
+ } : {}
+ )
+ tags = {
+ tenant = {
+ id = var.tag_keys.tenant
+ values = {
+ (var.tenant_config.short_name) = {}
+ }
+ }
+ }
+}
+
+resource "google_tags_tag_value_iam_member" "resman_tag_user" {
+ for_each = var.tag_values
+ tag_value = each.value
+ role = "roles/resourcemanager.tagUser"
+ member = module.automation-tf-resman-sa.iam_email
+}
+
+resource "google_tags_tag_value_iam_member" "admins_tag_viewer" {
+ for_each = var.tag_values
+ tag_value = each.value
+ role = "roles/resourcemanager.tagViewer"
+ member = "group:${local.groups.gcp-admins}"
+}
+
+# assign org policy admin with a tag-based condition to admin group and stage 1 SA
+
+resource "google_organization_iam_member" "org_policy_admin_stage0" {
+ for_each = toset([
+ "group:${local.groups.gcp-admins}",
+ module.automation-tf-resman-sa.iam_email
+ ])
+ org_id = var.organization.id
+ role = "roles/orgpolicy.policyAdmin"
+ member = each.key
+ condition {
+ title = "org_policy_tag_${var.tenant_config.short_name}_scoped"
+ description = "Org policy tag scoped grant for tenant ${var.tenant_config.short_name}."
+ expression = local.iam_tenant_condition
+ }
+}
+
+# tag-based condition for service accounts is in the automation-sa file
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/outputs-files.tf b/fast/stages-multitenant/0-bootstrap-tenant/outputs-files.tf
new file mode 100644
index 00000000..28bec327
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/outputs-files.tf
@@ -0,0 +1,46 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Output files persistence to local filesystem.
+
+locals {
+ outputs_root = join("/", [
+ try(pathexpand(var.outputs_location), ""),
+ "tenants",
+ var.tenant_config.short_name
+ ])
+}
+
+resource "local_file" "providers" {
+ count = var.outputs_location == null ? 0 : 1
+ file_permission = "0644"
+ filename = "${local.outputs_root}/providers/1-resman-tenant-providers.tf"
+ content = try(local.provider, null)
+}
+
+resource "local_file" "tfvars" {
+ count = var.outputs_location == null ? 0 : 1
+ file_permission = "0644"
+ filename = "${local.outputs_root}/tfvars/0-bootstrap-tenant.auto.tfvars.json"
+ content = jsonencode(local.tfvars)
+}
+
+resource "local_file" "workflows" {
+ for_each = var.outputs_location == null ? {} : local.cicd_workflows
+ file_permission = "0644"
+ filename = "${local.outputs_root}/workflows/${each.key}-${local.cicd_repositories[each.key].type}.yaml"
+ content = each.value
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/outputs-gcs.tf b/fast/stages-multitenant/0-bootstrap-tenant/outputs-gcs.tf
new file mode 100644
index 00000000..5196bad7
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/outputs-gcs.tf
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Output files persistence to automation GCS bucket.
+
+resource "google_storage_bucket_object" "providers" {
+ bucket = module.automation-tf-output-gcs.name
+ # provider suffix allows excluding via .gitignore when linked from stages
+ name = "providers/1-resman-tenant-providers.tf"
+ content = local.provider
+}
+
+resource "google_storage_bucket_object" "tfvars" {
+ bucket = module.automation-tf-output-gcs.name
+ name = "tfvars/0-bootstrap-tenant.auto.tfvars.json"
+ content = jsonencode(local.tfvars)
+}
+
+resource "google_storage_bucket_object" "workflows" {
+ for_each = local.cicd_workflows
+ bucket = (
+ each.key == "bootstrap"
+ ? var.automation.outputs_bucket
+ : module.automation-tf-output-gcs.name
+ )
+ name = "workflows/${each.key}-${local.cicd_repositories[each.key].type}.yaml"
+ content = each.value
+}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf b/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf
new file mode 100644
index 00000000..4f22ff63
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf
@@ -0,0 +1,140 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+ cicd_workflows = {
+ for k, v in local.cicd_repositories : k => templatefile(
+ "${path.module}/templates/workflow-${v.type}.yaml", (
+ k == "bootstrap"
+ ? {
+ identity_provider = try(
+ local.cicd_identity_providers[v["identity_provider"]].name, ""
+ )
+ outputs_bucket = var.automation.outputs_bucket
+ service_account = try(
+ module.automation-tf-cicd-sa-bootstrap["0"].email, ""
+ )
+ stage_name = k
+ tf_providers_file = ""
+ tf_var_files = [
+ "0-bootstrap.auto.tfvars.json",
+ "1-resman.auto.tfvars.json",
+ "globals.auto.tfvars.json"
+ ]
+ }
+ : {
+ identity_provider = try(
+ local.cicd_identity_providers[v["identity_provider"]].name, ""
+ )
+ outputs_bucket = module.automation-tf-output-gcs.name
+ service_account = try(
+ module.automation-tf-cicd-sa-resman["0"].email, ""
+ )
+ stage_name = k
+ tf_providers_file = (
+ "${local._file_prefix}/providers/1-resman-tenant-providers.tf"
+ )
+ tf_var_files = [
+ "${local._file_prefix}/tfvars/0-bootstrap-tenant.auto.tfvars.json"
+ ]
+ }
+ )
+ )
+ }
+ provider = templatefile(
+ "${path.module}/templates/providers.tf.tpl", {
+ bucket = module.automation-tf-resman-gcs.name
+ name = "resman"
+ sa = module.automation-tf-resman-sa.email
+ }
+ )
+ tfvars = {
+ automation = {
+ outputs_bucket = module.automation-tf-output-gcs.name
+ project_id = module.automation-project.project_id
+ project_number = module.automation-project.number
+ federated_identity_pools = compact([
+ try(google_iam_workload_identity_pool.default.0.name, null),
+ var.automation.federated_identity_pool,
+ ])
+ federated_identity_providers = local.cicd_identity_providers
+ service_accounts = merge(
+ { resman = module.automation-tf-resman-sa.email },
+ {
+ for k, v in local.branch_sas : k => try(
+ module.automation-tf-resman-sa-stage2-3[k].email, null
+ )
+ }
+ )
+ }
+ billing_account = var.billing_account
+ custom_roles = var.custom_roles
+ fast_features = local.fast_features
+ groups = var.tenant_config.groups
+ locations = local.locations
+ organization = var.organization
+ prefix = local.prefix
+ root_node = module.tenant-folder.id
+ short_name = var.tenant_config.short_name
+ tags = {
+ keys = var.tag_keys
+ names = var.tag_names
+ values = merge(var.tag_values, {
+ for k, v in module.organization.tag_values : k => v.id
+ })
+ }
+ }
+}
+
+output "cicd_workflows" {
+ description = "CI/CD workflows for tenant bootstrap and resource management stages."
+ sensitive = true
+ value = local.cicd_workflows
+}
+
+output "federated_identity" {
+ description = "Workload Identity Federation pool and providers."
+ value = {
+ pool = try(
+ google_iam_workload_identity_pool.default.0.name, null
+ )
+ providers = local.cicd_identity_providers
+ }
+}
+
+output "provider" {
+ # tfdoc:output:consumers stage-01
+ description = "Terraform provider file for tenant resource management stage."
+ sensitive = true
+ value = local.provider
+}
+
+output "tenant_resources" {
+ description = "Tenant-level resources."
+ value = {
+ bucket = module.automation-tf-resman-gcs.name
+ folder = module.tenant-folder.id
+ project_id = module.automation-project.project_id
+ project_number = module.automation-project.number
+ service_account = module.automation-tf-resman-sa.email
+ }
+}
+
+output "tfvars" {
+ description = "Terraform variable files for the following tenant stages."
+ sensitive = true
+ value = local.tfvars
+}
diff --git a/tests/blueprints/factories/bigquery_factory/fixture/main.tf b/fast/stages-multitenant/0-bootstrap-tenant/templates/providers.tf.tpl
similarity index 64%
rename from tests/blueprints/factories/bigquery_factory/fixture/main.tf
rename to fast/stages-multitenant/0-bootstrap-tenant/templates/providers.tf.tpl
index 75f4fc1c..e11a51b8 100644
--- a/tests/blueprints/factories/bigquery_factory/fixture/main.tf
+++ b/fast/stages-multitenant/0-bootstrap-tenant/templates/providers.tf.tpl
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,10 +14,17 @@
* limitations under the License.
*/
-module "bq" {
- source = "../../../../../blueprints/factories/bigquery-factory/"
-
- project_id = "test-project"
- views_dir = "./views"
- tables_dir = "./tables"
+terraform {
+ backend "gcs" {
+ bucket = "${bucket}"
+ impersonate_service_account = "${sa}"
+ }
}
+provider "google" {
+ impersonate_service_account = "${sa}"
+}
+provider "google-beta" {
+ impersonate_service_account = "${sa}"
+}
+
+# end provider.tf for ${name}
diff --git a/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-github.yaml b/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-github.yaml
new file mode 100644
index 00000000..364ee8f4
--- /dev/null
+++ b/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-github.yaml
@@ -0,0 +1,202 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "FAST ${stage_name} stage"
+
+on:
+ pull_request:
+ branches:
+ - main
+ types:
+ - closed
+ - opened
+ - synchronize
+
+env:
+ FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+ FAST_SERVICE_ACCOUNT: ${service_account}
+ FAST_WIF_PROVIDER: ${identity_provider}
+ SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+ %{~ if tf_providers_file != "" ~}
+ TF_PROVIDERS_FILE: ${tf_providers_file}
+ %{~ endif ~}
+ TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)}
+ TF_VERSION: 1.3.2
+
+jobs:
+ fast-pr:
+ permissions:
+ contents: read
+ id-token: write
+ issues: write
+ pull-requests: write
+ runs-on: ubuntu-latest
+ steps:
+ - id: checkout
+ name: Checkout repository
+ uses: actions/checkout@v3
+
+ # set up SSH key authentication to the modules repository
+ - id: ssh-config
+ name: Configure SSH authentication
+ run: |
+ ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
+ ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
+
+ # set up authentication via Workload identity Federation
+ - id: gcp-auth
+ name: Authenticate to Google Cloud
+ uses: google-github-actions/auth@v0
+ with:
+ workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+ service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+ access_token_lifetime: 3600s
+
+ - id: gcp-sdk
+ name: Set up Cloud SDK
+ uses: google-github-actions/setup-gcloud@v0
+ with:
+ install_components: alpha
+
+ # copy provider and tfvars files
+ - id: tf-config
+ name: Copy Terraform output files
+ run: |
+ %{~ if tf_providers_file != "" ~}
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+ %{~ endif ~}
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+ for f in $${{env.TF_VAR_FILES}}; do
+ ln -s "tfvars/$f" ./
+ done
+
+ - id: tf-setup
+ name: Set up Terraform
+ uses: hashicorp/setup-terraform@v2.0.3
+ with:
+ terraform_version: $${{ env.TF_VERSION }}
+
+ # run Terraform init/validate/plan
+ - id: tf-init
+ name: Terraform init
+ continue-on-error: true
+ run: |
+ terraform init -no-color
+
+ - id: tf-validate
+ name: Terraform validate
+ continue-on-error: true
+ run: terraform validate -no-color
+
+ - id: tf-plan
+ name: Terraform plan
+ continue-on-error: true
+ run: |
+ terraform plan -input=false -out ../plan.out -no-color
+
+ - id: tf-apply
+ if: github.event.pull_request.merged == true && success()
+ name: Terraform apply
+ continue-on-error: true
+ run: |
+ terraform apply -input=false -auto-approve -no-color ../plan.out
+
+ - id: pr-comment
+ name: Post comment to Pull Request
+ continue-on-error: true
+ uses: actions/github-script@v6
+ if: github.event_name == 'pull_request'
+ env:
+ PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+ with:
+ script: |
+ const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+
+ ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+
+ + additive, • conditional.
+
+## Folder development [#0]
+
+| members | roles |
+|---|---|
+|tn0-gke-dev-0++|
+|tn0-security-0+|
diff --git a/fast/stages-multitenant/1-resman-tenant/README.md b/fast/stages-multitenant/1-resman-tenant/README.md
new file mode 100644
index 00000000..ae42fc30
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/README.md
@@ -0,0 +1,184 @@
+# Tenant resource management
+
+This stage is run for a specific tenant after [tenant bootstrap](../0-bootstrap-tenant/) has successfully created initial resources for the tenant, which is then decoupled from the organization.
+
+It is logically equivalent and almost identical in code to the corresponding [organization resource management stage](../../stages/1-resman/), with a few notable differences:
+
+- the hierarchy is rooted in the tenant top-level folder instead of the organization
+- there's no management of tag values and keys since they organization-level resources (it could be implemented for tenant-specific tags if the need arises)
+- automation service accounts for subsequent stages are configured but not created here (tenant-level bootstrap creates them and assigns organization-level permissions)
+
+The stage runs with a dedicated service account for the tenant, which has no permissions at the organization level except for billing and organization policies, constrained by a condition on the tenant tag.
+
+The following diagram is a high level reference of what this stage manages, showing one hypothetical tenant (additional tenants require additional instances of this stage being deployed):
+
+```mermaid
+%%{init: {'theme':'base'}}%%
+classDiagram
+ Tenant_root~📁~ -- tn0_automation
+ Tenant_root~📁~ -- Networking~📁~
+ Tenant_root~📁~ -- Security~📁~
+ Tenant_root~📁~ -- Data_Platform~📁~
+ Data_Platform~📁~ -- DP_Dev~📁~
+ Data_Platform~📁~ -- DP_Prod~📁~
+ Tenant_root~📁~ -- GKE~📁~
+ GKE~📁~ -- GKE_Dev~📁~
+ GKE~📁~ -- GKE_Prod~📁~
+ Tenant_root~📁~ -- Teams~📁~
+ Teams~📁~ -- Team_0~📁~
+ Team_0~📁~ -- Team_0_Dev~📁~
+ Team_0~📁~ -- Team_0_Prod~📁~
+ Tenant_root~📁~ -- Sandbox~📁~
+ class Tenant_root~📁~ {
+ - IAM bindings()
+ - org policies()
+ }
+ class tn0_automation {
+ - GCS buckets
+ - IAM bindings()
+ }
+ class Data_Platform~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class DP_Dev~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class DP_Prod~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class GKE~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class GKE_Dev~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class GKE_Prod~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Networking~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Security~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Sandbox~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Teams~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Team_0~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Team_0_Dev~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+ class Team_0_Prod~📁~ {
+ - IAM bindings()
+ - tag bindings()
+ }
+```
+
+As most of the features of this stage follow the same design and configurations of the [organization-level resource management stage](../../stages/1-resman/), we will only focus on the tenant-specific configuration in this document.
+
+## How to run this stage
+
+As mentioned above this stage is decoupled from organization-level stages: it uses a service account and state bucket from the tenant-specific automation project, and its tfvars and provider files are also tenant-specific.
+
+The `stage-links.sh` script can be used to get the commands needed for the provider and output files, just set the variable for the tenant shortname (the same one specified in the tenant bootstrap stage) and pass a single argument with your FAST output files folder path, or GCS bucket URI:
+
+```bash
+TENANT=tn0 ../../stage-links.sh ~/fast-config
+```
+
+The script output can be copy/pasted to a terminal:
+
+```bash
+# copy and paste the following commands for '1-resman-tenant'
+
+ln -s ~/fast-config/tenants/tn0/providers/1-resman-tenant-providers.tf ./
+ln -s ~/fast-config/tenants/tn0/tfvars/0-bootstrap-tenant.auto.tfvars.json ./
+```
+
+Once that is done, stage-level configuration variables are the same as the corresponding organization-level stage.
+
+### Running the stage
+
+Once the configuration is done just go through the usual `init/apply` cycle. On successful apply, a tfvars file specific for this tenant and a set of provider files will be created.
+
+
+
+
+## Files
+
+| name | description | modules | resources |
+|---|---|---|---|
+| [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | folder · gcs · iam-service-account | |
+| [branch-gke.tf](./branch-gke.tf) | GKE multitenant stage resources. | folder · gcs · iam-service-account | |
+| [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | folder · gcs · iam-service-account | |
+| [branch-project-factory.tf](./branch-project-factory.tf) | Project factory stage resources. | gcs · iam-service-account | |
+| [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | folder · gcs | |
+| [branch-security.tf](./branch-security.tf) | Security stage resources. | folder · gcs · iam-service-account | |
+| [branch-teams.tf](./branch-teams.tf) | Team stage resources. | folder · gcs · iam-service-account | |
+| [cicd-data-platform.tf](./cicd-data-platform.tf) | CI/CD resources for the data platform branch. | iam-service-account · source-repository | |
+| [cicd-gke.tf](./cicd-gke.tf) | CI/CD resources for the data platform branch. | iam-service-account · source-repository | |
+| [cicd-networking.tf](./cicd-networking.tf) | CI/CD resources for the networking branch. | iam-service-account · source-repository | |
+| [cicd-project-factory.tf](./cicd-project-factory.tf) | CI/CD resources for the teams branch. | iam-service-account · source-repository | |
+| [cicd-security.tf](./cicd-security.tf) | CI/CD resources for the security branch. | iam-service-account · source-repository | |
+| [main.tf](./main.tf) | Module-level locals and resources. | | |
+| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file |
+| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object |
+| [outputs.tf](./outputs.tf) | Module outputs. | | |
+| [root_node.tf](./root_node.tf) | Tenant root folder configuration. | folder | |
+| [variables.tf](./variables.tf) | Module variables. | | |
+
+## Variables
+
+| name | description | type | required | default | producer |
+|---|---|:---:|:---:|:---:|:---:|
+| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L51) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [organization](variables.tf#L206) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L228) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [root_node](variables.tf#L239) | Root folder node for the tenant, in folders/nnnnnn format. | string | ✓ | | |
+| [short_name](variables.tf#L244) | Short name used to identify the tenant. | string | ✓ | | |
+| [tags](variables.tf#L249) | Resource management tags. | object({…}) | ✓ | | |
+| [cicd_repositories](variables.tf#L64) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | |
+| [custom_roles](variables.tf#L146) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [data_dir](variables.tf#L155) | Relative path for the folder storing configuration data. | string | | "data" | |
+| [fast_features](variables.tf#L161) | Selective control for top-level FAST features. | object({…}) | | {} | 0-0-bootstrap |
+| [groups](variables.tf#L175) | Group names to grant organization-level permissions. | object({…}) | | {} | 0-bootstrap |
+| [locations](variables.tf#L188) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | 0-bootstrap |
+| [organization_policy_data_path](variables.tf#L216) | Path for the data folder used by the organization policies factory. | string | | null | |
+| [outputs_location](variables.tf#L222) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | |
+| [team_folders](variables.tf#L267) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | |
+| [test_skip_data_sources](variables.tf#L277) | Used when testing to bypass data sources. | bool | | false | |
+
+## Outputs
+
+| name | description | sensitive | consumers |
+|---|---|:---:|---|
+| [cicd_repositories](outputs.tf#L189) | WIF configuration for CI/CD repositories. | | |
+| [dataplatform](outputs.tf#L203) | Data for the Data Platform stage. | | |
+| [gke_multitenant](outputs.tf#L219) | Data for the GKE multitenant stage. | | 03-gke-multitenant |
+| [networking](outputs.tf#L240) | Data for the networking stage. | | |
+| [project_factories](outputs.tf#L249) | Data for the project factories stage. | | |
+| [providers](outputs.tf#L264) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams |
+| [sandbox](outputs.tf#L271) | Data for the sandbox stage. | | xx-sandbox |
+| [security](outputs.tf#L285) | Data for the networking stage. | | 02-security |
+| [teams](outputs.tf#L295) | Data for the teams stage. | | |
+| [tfvars](outputs.tf#L307) | Terraform variable files for the following stages. | ✓ | |
+
+
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-data-platform.tf b/fast/stages-multitenant/1-resman-tenant/branch-data-platform.tf
new file mode 100644
index 00000000..3916d635
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-data-platform.tf
@@ -0,0 +1,133 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Data Platform stages resources.
+
+module "branch-dp-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.data_platform ? 1 : 0
+ parent = module.root-folder.id
+ name = "Data Platform"
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.context}/data"]
+ }
+}
+
+module "branch-dp-dev-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.data_platform ? 1 : 0
+ parent = module.branch-dp-folder.0.id
+ name = "Development"
+ group_iam = {}
+ iam = {
+ (local.custom_roles.service_project_network_admin) = [
+ local.automation_sas_iam.dp-dev
+ ]
+ # remove owner here and at project level if SA does not manage project resources
+ "roles/owner" = [local.automation_sas_iam.dp-dev]
+ "roles/logging.admin" = [local.automation_sas_iam.dp-dev]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.dp-dev]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.dp-dev]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.environment}/development"]
+ }
+}
+
+module "branch-dp-prod-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.data_platform ? 1 : 0
+ parent = module.branch-dp-folder.0.id
+ name = "Production"
+ group_iam = {}
+ iam = {
+ (local.custom_roles.service_project_network_admin) = [
+ local.automation_sas_iam.dp-prod
+ ]
+ # remove owner here and at project level if SA does not manage project resources
+ "roles/owner" = [local.automation_sas_iam.dp-prod]
+ "roles/logging.admin" = [local.automation_sas_iam.dp-prod]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.dp-prod]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.dp-prod]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.environment}/production"]
+ }
+}
+
+# automation service accounts and buckets
+
+module "branch-dp-dev-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.data_platform ? 1 : 0
+ project_id = var.automation.project_id
+ name = "dp-dev-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.branch-dp-dev-sa-cicd.0.iam_email, null)
+ ])
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-dp-prod-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.data_platform ? 1 : 0
+ project_id = var.automation.project_id
+ name = "dp-prod-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.branch-dp-prod-sa-cicd.0.iam_email, null)
+ ])
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-dp-dev-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.data_platform ? 1 : 0
+ project_id = var.automation.project_id
+ name = "dev-resman-dp-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.dp-dev]
+ }
+}
+
+module "branch-dp-prod-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.data_platform ? 1 : 0
+ project_id = var.automation.project_id
+ name = "prod-resman-dp-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.dp-prod]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-gke.tf b/fast/stages-multitenant/1-resman-tenant/branch-gke.tf
new file mode 100644
index 00000000..9ece810b
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-gke.tf
@@ -0,0 +1,133 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description GKE multitenant stage resources.
+
+module "branch-gke-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.gke ? 1 : 0
+ parent = module.root-folder.id
+ name = "GKE"
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.context}/gke"]
+ }
+}
+
+module "branch-gke-dev-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.gke ? 1 : 0
+ parent = module.branch-gke-folder.0.id
+ name = "Development"
+ iam = {
+ "roles/owner" = [local.automation_sas_iam.gke-dev]
+ "roles/logging.admin" = [local.automation_sas_iam.gke-dev]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.gke-dev]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.gke-dev]
+ "roles/compute.xpnAdmin" = [local.automation_sas_iam.gke-dev]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.environment}/development"]
+ }
+}
+
+module "branch-gke-prod-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.gke ? 1 : 0
+ parent = module.branch-gke-folder.0.id
+ name = "Production"
+ iam = {
+ "roles/owner" = [local.automation_sas_iam.gke-prod]
+ "roles/logging.admin" = [local.automation_sas_iam.gke-prod]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.gke-prod]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.gke-prod]
+ "roles/compute.xpnAdmin" = [local.automation_sas_iam.gke-prod]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.environment}/production"]
+ }
+}
+
+module "branch-gke-dev-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.gke ? 1 : 0
+ project_id = var.automation.project_id
+ name = "gke-dev-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = concat(
+ (
+ local.groups.gcp-devops == null
+ ? []
+ : ["group:${local.groups.gcp-devops}"]
+ ),
+ compact([
+ try(module.branch-gke-dev-sa-cicd.0.iam_email, null)
+ ])
+ )
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-gke-prod-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.gke ? 1 : 0
+ project_id = var.automation.project_id
+ name = "gke-prod-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = concat(
+ (
+ local.groups.gcp-devops == null
+ ? []
+ : ["group:${local.groups.gcp-devops}"]
+ ),
+ compact([
+ try(module.branch-gke-prod-sa-cicd.0.iam_email, null)
+ ])
+ )
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-gke-dev-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.gke ? 1 : 0
+ project_id = var.automation.project_id
+ name = "dev-resman-gke-0"
+ prefix = var.prefix
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.gke-dev]
+ }
+}
+
+module "branch-gke-prod-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.gke ? 1 : 0
+ project_id = var.automation.project_id
+ name = "prod-resman-gke-0"
+ prefix = var.prefix
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.gke-prod]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-networking.tf b/fast/stages-multitenant/1-resman-tenant/branch-networking.tf
new file mode 100644
index 00000000..85490baf
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-networking.tf
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Networking stage resources.
+
+module "branch-network-folder" {
+ source = "../../../modules/folder"
+ parent = module.root-folder.id
+ name = "Networking"
+ group_iam = local.groups.gcp-network-admins == null ? {} : {
+ (local.groups.gcp-network-admins) = [
+ # add any needed roles for resources/services not managed via Terraform,
+ # or replace editor with ~viewer if no broad resource management needed
+ # e.g.
+ # "roles/compute.networkAdmin",
+ # "roles/dns.admin",
+ # "roles/compute.securityAdmin",
+ "roles/editor",
+ ]
+ }
+ iam = {
+ "roles/logging.admin" = [local.automation_sas_iam.networking]
+ "roles/owner" = [local.automation_sas_iam.networking]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.networking]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.networking]
+ "roles/compute.xpnAdmin" = [local.automation_sas_iam.networking]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.context}/networking"]
+ }
+}
+
+module "branch-network-prod-folder" {
+ source = "../../../modules/folder"
+ parent = module.branch-network-folder.id
+ name = "Production"
+ iam = {
+ (local.custom_roles.service_project_network_admin) = concat(
+ local.branch_optional_sa_lists.dp-prod,
+ local.branch_optional_sa_lists.gke-prod,
+ local.branch_optional_sa_lists.pf-prod,
+ )
+ }
+ tag_bindings = {
+ environment = var.tags.values["${var.tags.names.environment}/production"]
+ }
+}
+
+module "branch-network-dev-folder" {
+ source = "../../../modules/folder"
+ parent = module.branch-network-folder.id
+ name = "Development"
+ iam = {
+ (local.custom_roles.service_project_network_admin) = concat(
+ local.branch_optional_sa_lists.dp-dev,
+ local.branch_optional_sa_lists.gke-dev,
+ local.branch_optional_sa_lists.pf-dev,
+ )
+ }
+ tag_bindings = {
+ environment = var.tags.values["${var.tags.names.environment}/development"]
+ }
+}
+
+# automation service account and bucket
+
+module "branch-network-sa" {
+ source = "../../../modules/iam-service-account"
+ project_id = var.automation.project_id
+ name = "networking-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.branch-network-sa-cicd.0.iam_email, null)
+ ])
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-network-gcs" {
+ source = "../../../modules/gcs"
+ project_id = var.automation.project_id
+ name = "prod-resman-net-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.networking]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-project-factory.tf b/fast/stages-multitenant/1-resman-tenant/branch-project-factory.tf
new file mode 100644
index 00000000..2fa64bbc
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-project-factory.tf
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Project factory stage resources.
+
+module "branch-pf-dev-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.project_factory ? 1 : 0
+ project_id = var.automation.project_id
+ name = "pf-dev-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.branch-pf-dev-sa-cicd.0.iam_email, null)
+ ])
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-pf-prod-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.project_factory ? 1 : 0
+ project_id = var.automation.project_id
+ name = "pf-prod-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.branch-pf-prod-sa-cicd.0.iam_email, null)
+ ])
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-pf-dev-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.project_factory ? 1 : 0
+ project_id = var.automation.project_id
+ name = "dev-resman-pf-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.pf-dev]
+ }
+}
+
+module "branch-pf-prod-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.project_factory ? 1 : 0
+ project_id = var.automation.project_id
+ name = "prod-resman-pf-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.pf-prod]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf b/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf
new file mode 100644
index 00000000..6f3d526c
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf
@@ -0,0 +1,51 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Sandbox stage resources.
+
+module "branch-sandbox-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.sandbox ? 1 : 0
+ parent = module.root-folder.id
+ name = "Sandbox"
+ iam = {
+ "roles/logging.admin" = [local.automation_sas_iam.sandbox]
+ "roles/owner" = [local.automation_sas_iam.sandbox]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.sandbox]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.sandbox]
+ }
+ org_policies = {
+ "constraints/sql.restrictPublicIp" = { enforce = false }
+ "constraints/compute.vmExternalIpAccess" = { allow = { all = true } }
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.context}/sandbox"]
+ }
+}
+
+module "branch-sandbox-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.sandbox ? 1 : 0
+ project_id = var.automation.project_id
+ name = "dev-resman-sbox-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.sandbox]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-security.tf b/fast/stages-multitenant/1-resman-tenant/branch-security.tf
new file mode 100644
index 00000000..d7253cce
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-security.tf
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Security stage resources.
+
+module "branch-security-folder" {
+ source = "../../../modules/folder"
+ parent = module.root-folder.id
+ name = "Security"
+ group_iam = local.groups.gcp-security-admins == null ? {} : {
+ (local.groups.gcp-security-admins) = [
+ # add any needed roles for resources/services not managed via Terraform,
+ # e.g.
+ # "roles/bigquery.admin",
+ # "roles/cloudasset.owner",
+ # "roles/cloudkms.admin",
+ # "roles/logging.admin",
+ # "roles/secretmanager.admin",
+ # "roles/storage.admin",
+ "roles/viewer"
+ ]
+ }
+ iam = {
+ "roles/logging.admin" = [local.automation_sas_iam.security]
+ "roles/owner" = [local.automation_sas_iam.security]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.security]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.security]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.context}/security"]
+ }
+}
+
+# automation service account and bucket
+
+module "branch-security-sa" {
+ source = "../../../modules/iam-service-account"
+ project_id = var.automation.project_id
+ name = "security-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = compact([
+ try(module.branch-security-sa-cicd.0.iam_email, null)
+ ])
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-security-gcs" {
+ source = "../../../modules/gcs"
+ project_id = var.automation.project_id
+ name = "prod-resman-sec-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [local.automation_sas_iam.security]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-teams.tf b/fast/stages-multitenant/1-resman-tenant/branch-teams.tf
new file mode 100644
index 00000000..57f22110
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/branch-teams.tf
@@ -0,0 +1,163 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Team stage resources.
+
+# TODO(ludo): add support for CI/CD
+
+############### top-level Teams branch and automation resources ###############
+
+module "branch-teams-folder" {
+ source = "../../../modules/folder"
+ count = var.fast_features.teams ? 1 : 0
+ parent = module.root-folder.id
+ name = "Teams"
+ iam = {
+ "roles/logging.admin" = [local.automation_sas_iam.teams]
+ "roles/owner" = [local.automation_sas_iam.teams]
+ "roles/resourcemanager.folderAdmin" = [local.automation_sas_iam.teams]
+ "roles/resourcemanager.projectCreator" = [local.automation_sas_iam.teams]
+ "roles/compute.xpnAdmin" = [local.automation_sas_iam.teams]
+ }
+ tag_bindings = {
+ context = var.tags.values["${var.tags.names.context}/teams"]
+ }
+}
+
+module "branch-teams-sa" {
+ source = "../../../modules/iam-service-account"
+ count = var.fast_features.teams ? 1 : 0
+ project_id = var.automation.project_id
+ name = "teams-0"
+ prefix = var.prefix
+ service_account_create = var.test_skip_data_sources
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.admin"]
+ }
+}
+
+module "branch-teams-gcs" {
+ source = "../../../modules/gcs"
+ count = var.fast_features.teams ? 1 : 0
+ project_id = var.automation.project_id
+ name = "prod-resman-teams-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [module.branch-teams-sa.0.iam_email]
+ }
+}
+
+################## per-team folders and automation resources ##################
+
+module "branch-teams-team-folder" {
+ source = "../../../modules/folder"
+ for_each = var.fast_features.teams ? coalesce(var.team_folders, {}) : {}
+ parent = module.branch-teams-folder.0.id
+ name = each.value.descriptive_name
+ iam = {
+ "roles/logging.admin" = [module.branch-teams-team-sa[each.key].iam_email]
+ "roles/owner" = [module.branch-teams-team-sa[each.key].iam_email]
+ "roles/resourcemanager.folderAdmin" = [module.branch-teams-team-sa[each.key].iam_email]
+ "roles/resourcemanager.projectCreator" = [module.branch-teams-team-sa[each.key].iam_email]
+ "roles/compute.xpnAdmin" = [module.branch-teams-team-sa[each.key].iam_email]
+ }
+ group_iam = each.value.group_iam == null ? {} : each.value.group_iam
+}
+
+module "branch-teams-team-sa" {
+ source = "../../../modules/iam-service-account"
+ for_each = var.fast_features.teams ? coalesce(var.team_folders, {}) : {}
+ project_id = var.automation.project_id
+ name = "prod-teams-${each.key}-0"
+ display_name = "Terraform team ${each.key} service account."
+ prefix = var.prefix
+ iam = {
+ "roles/iam.serviceAccountTokenCreator" = (
+ each.value.impersonation_groups == null
+ ? []
+ : [for g in each.value.impersonation_groups : "group:${g}"]
+ )
+ }
+}
+
+module "branch-teams-team-gcs" {
+ source = "../../../modules/gcs"
+ for_each = var.fast_features.teams ? coalesce(var.team_folders, {}) : {}
+ project_id = var.automation.project_id
+ name = "prod-teams-${each.key}-0"
+ prefix = var.prefix
+ location = var.locations.gcs
+ storage_class = local.gcs_storage_class
+ versioning = true
+ iam = {
+ "roles/storage.objectAdmin" = [module.branch-teams-team-sa[each.key].iam_email]
+ }
+}
+
+# per-team environment folders where project factory SAs can create projects
+
+module "branch-teams-team-dev-folder" {
+ source = "../../../modules/folder"
+ for_each = var.fast_features.teams ? coalesce(var.team_folders, {}) : {}
+ parent = module.branch-teams-team-folder[each.key].id
+ # naming: environment descriptive name
+ name = "Development"
+ # environment-wide human permissions on the whole teams environment
+ group_iam = {}
+ iam = {
+ (local.custom_roles.service_project_network_admin) = (
+ local.branch_optional_sa_lists.pf-dev
+ )
+ # remove owner here and at project level if SA does not manage project resources
+ "roles/owner" = local.branch_optional_sa_lists.pf-dev
+ "roles/logging.admin" = local.branch_optional_sa_lists.pf-dev
+ "roles/resourcemanager.folderAdmin" = local.branch_optional_sa_lists.pf-dev
+ "roles/resourcemanager.projectCreator" = local.branch_optional_sa_lists.pf-dev
+ }
+ tag_bindings = {
+ environment = try(
+ var.tags.values["${var.tags.names.environment}/development"], null
+ )
+ }
+}
+
+module "branch-teams-team-prod-folder" {
+ source = "../../../modules/folder"
+ for_each = var.fast_features.teams ? coalesce(var.team_folders, {}) : {}
+ parent = module.branch-teams-team-folder[each.key].id
+ # naming: environment descriptive name
+ name = "Production"
+ # environment-wide human permissions on the whole teams environment
+ group_iam = {}
+ iam = {
+ (local.custom_roles.service_project_network_admin) = (
+ local.branch_optional_sa_lists.pf-prod
+ )
+ # remove owner here and at project level if SA does not manage project resources
+ "roles/owner" = local.branch_optional_sa_lists.pf-prod
+ "roles/logging.admin" = local.branch_optional_sa_lists.pf-prod
+ "roles/resourcemanager.folderAdmin" = local.branch_optional_sa_lists.pf-prod
+ "roles/resourcemanager.projectCreator" = local.branch_optional_sa_lists.pf-prod
+ }
+ tag_bindings = {
+ environment = try(
+ var.tags.values["${var.tags.names.environment}/production"], null
+ )
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/cicd-data-platform.tf b/fast/stages-multitenant/1-resman-tenant/cicd-data-platform.tf
new file mode 100644
index 00000000..704f45d7
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/cicd-data-platform.tf
@@ -0,0 +1,173 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description CI/CD resources for the data platform branch.
+
+# source repositories
+
+module "branch-dp-dev-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.data_platform_dev.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.data_platform_dev }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = local.branch_optional_sa_lists.dp-dev
+ "roles/source.reader" = compact([
+ try(module.branch-dp-dev-sa-cicd.0.iam_email, "")
+ ])
+ }
+ triggers = {
+ fast-03-dp-dev = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = [
+ "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
+ ]
+ service_account = module.branch-dp-dev-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-dp-dev-sa-cicd]
+}
+
+module "branch-dp-prod-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.data_platform_prod.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.data_platform_prod }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = local.branch_optional_sa_lists.dp-prod
+ "roles/source.reader" = [module.branch-dp-prod-sa-cicd.0.iam_email]
+ }
+ triggers = {
+ fast-03-dp-prod = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = [
+ "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
+ ]
+ service_account = module.branch-dp-prod-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-dp-prod-sa-cicd]
+}
+
+# SAs used by CI/CD workflows to impersonate automation SAs
+
+module "branch-dp-dev-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.data_platform_dev.name, null) != null
+ ? { 0 = local.cicd_repositories.data_platform_dev }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "dev-resman-dp-1"
+ display_name = "Terraform CI/CD data platform development service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
+
+module "branch-dp-prod-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.data_platform_prod.name, null) != null
+ ? { 0 = local.cicd_repositories.data_platform_prod }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "prod-resman-dp-1"
+ display_name = "Terraform CI/CD data platform production service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/cicd-gke.tf b/fast/stages-multitenant/1-resman-tenant/cicd-gke.tf
new file mode 100644
index 00000000..dfd035a5
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/cicd-gke.tf
@@ -0,0 +1,175 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description CI/CD resources for the data platform branch.
+
+# source repositories
+
+module "branch-gke-dev-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.gke_dev.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.gke_dev }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = compact([
+ try(module.branch-gke-dev-sa.0.iam_email, "")
+ ])
+ "roles/source.reader" = compact([
+ try(module.branch-gke-dev-sa-cicd.0.iam_email, "")
+ ])
+ }
+ triggers = {
+ fast-03-gke-dev = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = [
+ "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
+ ]
+ service_account = module.branch-gke-dev-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-gke-dev-sa-cicd]
+}
+
+module "branch-gke-prod-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.gke_prod.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.gke_prod }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = [module.branch-gke-prod-sa.0.iam_email]
+ "roles/source.reader" = [module.branch-gke-prod-sa-cicd.0.iam_email]
+ }
+ triggers = {
+ fast-03-gke-prod = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = [
+ "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
+ ]
+ service_account = module.branch-gke-prod-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-gke-prod-sa-cicd]
+}
+
+# SAs used by CI/CD workflows to impersonate automation SAs
+
+module "branch-gke-dev-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.gke_dev.name, null) != null
+ ? { 0 = local.cicd_repositories.gke_dev }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "dev-resman-gke-1"
+ display_name = "Terraform CI/CD GKE development service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
+
+module "branch-gke-prod-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.gke_prod.name, null) != null
+ ? { 0 = local.cicd_repositories.gke_prod }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "prod-resman-gke-1"
+ display_name = "Terraform CI/CD GKE production service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/cicd-networking.tf b/fast/stages-multitenant/1-resman-tenant/cicd-networking.tf
new file mode 100644
index 00000000..dbaf587d
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/cicd-networking.tf
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description CI/CD resources for the networking branch.
+
+# source repository
+
+module "branch-network-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.networking.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.networking }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = [module.branch-network-sa.iam_email]
+ "roles/source.reader" = [module.branch-network-sa-cicd.0.iam_email]
+ }
+ triggers = {
+ fast-02-networking = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
+ service_account = module.branch-network-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-network-sa-cicd]
+}
+
+# SA used by CI/CD workflows to impersonate automation SAs
+
+module "branch-network-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.networking.name, null) != null
+ ? { 0 = local.cicd_repositories.networking }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "prod-resman-net-1"
+ display_name = "Terraform CI/CD stage 2 networking service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/cicd-project-factory.tf b/fast/stages-multitenant/1-resman-tenant/cicd-project-factory.tf
new file mode 100644
index 00000000..4c46d858
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/cicd-project-factory.tf
@@ -0,0 +1,191 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description CI/CD resources for the teams branch.
+
+# source repositories
+
+moved {
+ from = module.branch-teams-dev-pf-cicd-repo
+ to = module.branch-pf-dev-cicd-repo
+}
+
+module "branch-pf-dev-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.project_factory_dev.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.project_factory_dev }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = local.branch_optional_sa_lists.pf-dev
+ "roles/source.reader" = [module.branch-pf-dev-sa-cicd.0.iam_email]
+ }
+ triggers = {
+ fast-03-pf-dev = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = [
+ "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
+ ]
+ service_account = module.branch-pf-dev-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-pf-dev-sa-cicd]
+}
+
+moved {
+ from = module.branch-teams-prod-pf-cicd-repo
+ to = module.branch-pf-prod-cicd-repo
+}
+
+module "branch-pf-prod-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.project_factory_prod.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.project_factory_prod }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = local.branch_optional_sa_lists.pf-prod
+ "roles/source.reader" = [module.branch-pf-prod-sa-cicd.0.iam_email]
+ }
+ triggers = {
+ fast-03-pf-prod = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = [
+ "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
+ ]
+ service_account = module.branch-pf-prod-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-pf-prod-sa-cicd]
+}
+
+# SAs used by CI/CD workflows to impersonate automation SAs
+
+moved {
+ from = module.branch-teams-dev-pf-sa-cicd
+ to = module.branch-pf-dev-sa-cicd
+}
+
+module "branch-pf-dev-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.project_factory_dev.name, null) != null
+ ? { 0 = local.cicd_repositories.project_factory_dev }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "dev-pf-resman-pf-1"
+ display_name = "Terraform CI/CD project factory development service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
+
+moved {
+ from = module.branch-teams-prod-pf-sa-cicd
+ to = module.branch-pf-prod-sa-cicd
+}
+
+module "branch-pf-prod-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.project_factory_prod.name, null) != null
+ ? { 0 = local.cicd_repositories.project_factory_prod }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "prod-pf-resman-pf-1"
+ display_name = "Terraform CI/CD project factory production service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ var.automation.federated_identity_pool,
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ var.automation.federated_identity_pool,
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/cicd-security.tf b/fast/stages-multitenant/1-resman-tenant/cicd-security.tf
new file mode 100644
index 00000000..5cb1581c
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/cicd-security.tf
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description CI/CD resources for the security branch.
+
+# source repository
+
+module "branch-security-cicd-repo" {
+ source = "../../../modules/source-repository"
+ for_each = (
+ try(local.cicd_repositories.security.type, null) == "sourcerepo"
+ ? { 0 = local.cicd_repositories.security }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = each.value.name
+ iam = {
+ "roles/source.admin" = [module.branch-security-sa.iam_email]
+ "roles/source.reader" = [module.branch-security-sa-cicd.0.iam_email]
+ }
+ triggers = {
+ fast-02-security = {
+ filename = ".cloudbuild/workflow.yaml"
+ included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
+ service_account = module.branch-security-sa-cicd.0.id
+ substitutions = {}
+ template = {
+ project_id = null
+ branch_name = each.value.branch
+ repo_name = each.value.name
+ tag_name = null
+ }
+ }
+ }
+ depends_on = [module.branch-security-sa-cicd]
+}
+
+# SA used by CI/CD workflows to impersonate automation SAs
+
+module "branch-security-sa-cicd" {
+ source = "../../../modules/iam-service-account"
+ for_each = (
+ try(local.cicd_repositories.security.name, null) != null
+ ? { 0 = local.cicd_repositories.security }
+ : {}
+ )
+ project_id = var.automation.project_id
+ name = "prod-resman-sec-1"
+ display_name = "Terraform CI/CD stage 2 security service account."
+ prefix = var.prefix
+ iam = (
+ each.value.type == "sourcerepo"
+ # used directly from the cloud build trigger for source repos
+ ? {
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
+ }
+ # impersonated via workload identity federation for external repos
+ : {
+ "roles/iam.workloadIdentityUser" = [
+ each.value.branch == null
+ ? format(
+ local.cicd_identity_providers[each.value.identity_provider].principalset_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name
+ )
+ : format(
+ local.cicd_identity_providers[each.value.identity_provider].principal_tpl,
+ local.cicd_identity_pools[each.value.identity_provider],
+ each.value.name,
+ each.value.branch
+ )
+ ]
+ }
+ )
+ iam_project_roles = {
+ (var.automation.project_id) = ["roles/logging.logWriter"]
+ }
+ iam_storage_roles = {
+ (var.automation.outputs_bucket) = ["roles/storage.objectViewer"]
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml
new file mode 100644
index 00000000..0d27ac42
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml
@@ -0,0 +1,73 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+compute.disableGuestAttributesAccess:
+ enforce: true
+
+compute.requireOsLogin:
+ enforce: true
+
+compute.restrictLoadBalancerCreationForTypes:
+ allow:
+ values:
+ - in:INTERNAL
+
+compute.skipDefaultNetworkCreation:
+ enforce: true
+
+compute.vmExternalIpAccess:
+ deny:
+ all: true
+
+
+# compute.disableInternetNetworkEndpointGroup:
+# enforce: true
+
+# compute.disableNestedVirtualization:
+# enforce: true
+
+# compute.disableSerialPortAccess:
+# enforce: true
+
+# compute.restrictCloudNATUsage:
+# deny:
+# all: true
+
+# compute.restrictDedicatedInterconnectUsage:
+# deny:
+# all: true
+
+# compute.restrictPartnerInterconnectUsage:
+# deny:
+# all: true
+
+# compute.restrictProtocolForwardingCreationForTypes:
+# deny:
+# all: true
+
+# compute.restrictSharedVpcHostProjects:
+# deny:
+# all: true
+
+# compute.restrictSharedVpcSubnetworks:
+# deny:
+# all: true
+
+# compute.restrictVpcPeering:
+# deny:
+# all: true
+
+# compute.restrictVpnPeerIPs:
+# deny:
+# all: true
+
+# compute.restrictXpnProjectLienRemoval:
+# enforce: true
+
+# compute.setNewProjectDefaultToZonalDNSOnly:
+# enforce: true
+
+# compute.vmCanIpForward:
+# deny:
+# all: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml
new file mode 100644
index 00000000..4d83f827
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml
@@ -0,0 +1,12 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+iam.automaticIamGrantsForDefaultServiceAccounts:
+ enforce: true
+
+iam.disableServiceAccountKeyCreation:
+ enforce: true
+
+iam.disableServiceAccountKeyUpload:
+ enforce: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml
new file mode 100644
index 00000000..de62e6c7
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml
@@ -0,0 +1,26 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+run.allowedIngress:
+ allow:
+ values:
+ - is:internal
+
+# run.allowedVPCEgress:
+# allow:
+# values:
+# - is:private-ranges-only
+
+# cloudfunctions.allowedIngressSettings:
+# allow:
+# values:
+# - is:ALLOW_INTERNAL_ONLY
+
+# cloudfunctions.allowedVpcConnectorEgressSettings:
+# allow:
+# values:
+# - is:PRIVATE_RANGES_ONLY
+
+# cloudfunctions.requireVPCConnector:
+# enforce: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml
new file mode 100644
index 00000000..88b84d9d
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml
@@ -0,0 +1,9 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+sql.restrictAuthorizedNetworks:
+ enforce: true
+
+sql.restrictPublicIp:
+ enforce: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml
new file mode 100644
index 00000000..6c0a673f
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml
@@ -0,0 +1,6 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+storage.uniformBucketLevelAccess:
+ enforce: true
diff --git a/fast/stages/01-resman/diagram.png b/fast/stages-multitenant/1-resman-tenant/diagram.png
similarity index 100%
rename from fast/stages/01-resman/diagram.png
rename to fast/stages-multitenant/1-resman-tenant/diagram.png
diff --git a/fast/stages/01-resman/diagram.svg b/fast/stages-multitenant/1-resman-tenant/diagram.svg
similarity index 100%
rename from fast/stages/01-resman/diagram.svg
rename to fast/stages-multitenant/1-resman-tenant/diagram.svg
diff --git a/fast/stages-multitenant/1-resman-tenant/main.tf b/fast/stages-multitenant/1-resman-tenant/main.tf
new file mode 100644
index 00000000..76c04639
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/main.tf
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+ automation_resman_sa_iam = [
+ "serviceAccount:${var.automation.service_accounts.resman}"
+ ]
+ automation_sas_iam = {
+ for k, v in var.automation.service_accounts :
+ k => v == null ? null : "serviceAccount:${v}"
+ }
+ branch_optional_sa_lists = {
+ dp-dev = compact([local.automation_sas_iam.dp-dev])
+ dp-prod = compact([local.automation_sas_iam.dp-prod])
+ gke-dev = compact([local.automation_sas_iam.gke-dev])
+ gke-prod = compact([local.automation_sas_iam.gke-prod])
+ pf-dev = compact([local.automation_sas_iam.pf-dev])
+ pf-prod = compact([local.automation_sas_iam.pf-prod])
+ }
+ # derive identity pool names from identity providers for easy reference
+ cicd_identity_pools = {
+ for k, v in local.cicd_identity_providers :
+ k => split("/providers/", v.name)[0]
+ }
+ cicd_identity_providers = coalesce(
+ try(var.automation.federated_identity_providers, null), {}
+ )
+ cicd_repositories = {
+ for k, v in coalesce(var.cicd_repositories, {}) : k => v
+ if(
+ v != null &&
+ (
+ try(v.type, null) == "sourcerepo"
+ ||
+ contains(
+ keys(local.cicd_identity_providers),
+ coalesce(try(v.identity_provider, null), ":")
+ )
+ ) &&
+ fileexists("${path.module}/templates/workflow-${try(v.type, "")}.yaml")
+ )
+ }
+ cicd_workflow_var_files = {
+ stage_2 = [
+ "0-bootstrap-tenant.auto.tfvars.json",
+ ]
+ stage_3 = [
+ "0-bootstrap-tenant.auto.tfvars.json",
+ "2-networking.auto.tfvars.json",
+ "2-security.auto.tfvars.json"
+ ]
+ }
+ custom_roles = coalesce(var.custom_roles, {})
+ gcs_storage_class = (
+ length(split("-", var.locations.gcs)) < 2
+ ? "MULTI_REGIONAL"
+ : "REGIONAL"
+ )
+ groups = {
+ for k, v in var.groups :
+ k => v == null ? null : "${v}@${var.organization.domain}"
+ }
+ groups_iam = {
+ for k, v in local.groups : k => v != null ? "group:${v}" : null
+ }
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/outputs-files.tf b/fast/stages-multitenant/1-resman-tenant/outputs-files.tf
new file mode 100644
index 00000000..29d5ed46
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/outputs-files.tf
@@ -0,0 +1,46 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Output files persistence to local filesystem.
+
+locals {
+ outputs_root = join("/", [
+ try(pathexpand(var.outputs_location), ""),
+ "tenants",
+ var.short_name
+ ])
+}
+
+resource "local_file" "providers" {
+ for_each = var.outputs_location == null ? {} : local.providers
+ file_permission = "0644"
+ filename = "${local.outputs_root}/providers/${each.key}-providers.tf"
+ content = try(each.value, null)
+}
+
+resource "local_file" "tfvars" {
+ count = var.outputs_location == null ? 0 : 1
+ file_permission = "0644"
+ filename = "${local.outputs_root}/tfvars/1-resman.auto.tfvars.json"
+ content = jsonencode(local.tfvars)
+}
+
+resource "local_file" "workflows" {
+ for_each = var.outputs_location == null ? {} : local.cicd_workflows
+ file_permission = "0644"
+ filename = "${local.outputs_root}/workflows/${replace(each.key, "_", "-")}-workflow.yaml"
+ content = try(each.value, null)
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/outputs-gcs.tf b/fast/stages-multitenant/1-resman-tenant/outputs-gcs.tf
new file mode 100644
index 00000000..6b0fc89c
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/outputs-gcs.tf
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Output files persistence to automation GCS bucket.
+
+resource "google_storage_bucket_object" "providers" {
+ for_each = local.providers
+ bucket = var.automation.outputs_bucket
+ name = "providers/${each.key}-providers.tf"
+ content = each.value
+}
+
+resource "google_storage_bucket_object" "tfvars" {
+ bucket = var.automation.outputs_bucket
+ name = "tfvars/1-resman.auto.tfvars.json"
+ content = jsonencode(local.tfvars)
+}
+
+resource "google_storage_bucket_object" "workflows" {
+ for_each = local.cicd_workflows
+ bucket = var.automation.outputs_bucket
+ name = "workflows/${replace(each.key, "_", "-")}-workflow.yaml"
+ content = each.value
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/outputs.tf b/fast/stages-multitenant/1-resman-tenant/outputs.tf
new file mode 100644
index 00000000..592f995e
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/outputs.tf
@@ -0,0 +1,311 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+ _tpl_providers = "${path.module}/templates/providers.tf.tpl"
+ cicd_workflow_attrs = {
+ data_platform_dev = {
+ service_account = try(module.branch-dp-dev-sa-cicd.0.email, null)
+ tf_providers_file = "3-data-platform-dev-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_3
+ }
+ data_platform_prod = {
+ service_account = try(module.branch-dp-prod-sa-cicd.0.email, null)
+ tf_providers_file = "3-data-platform-prod-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_3
+ }
+ gke_dev = {
+ service_account = try(module.branch-gke-dev-sa-cicd.0.email, null)
+ tf_providers_file = "3-gke-dev-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_3
+ }
+ gke_prod = {
+ service_account = try(module.branch-gke-prod-sa-cicd.0.email, null)
+ tf_providers_file = "3-gke-prod-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_3
+ }
+ networking = {
+ service_account = try(module.branch-network-sa-cicd.0.email, null)
+ tf_providers_file = "2-networking-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_2
+ }
+ project_factory_dev = {
+ service_account = try(module.branch-pf-dev-sa-cicd.0.email, null)
+ tf_providers_file = "3-project-factory-dev-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_3
+ }
+ project_factory_prod = {
+ service_account = try(module.branch-pf-prod-sa-cicd.0.email, null)
+ tf_providers_file = "3-project-factory-prod-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_3
+ }
+ security = {
+ service_account = try(module.branch-security-sa-cicd.0.email, null)
+ tf_providers_file = "2-security-providers.tf"
+ tf_var_files = local.cicd_workflow_var_files.stage_2
+ }
+ }
+ cicd_workflows = {
+ for k, v in local.cicd_repositories : k => templatefile(
+ "${path.module}/templates/workflow-${v.type}.yaml",
+ merge(local.cicd_workflow_attrs[k], {
+ identity_provider = try(
+ local.cicd_identity_providers[v.identity_provider].name, null
+ )
+ outputs_bucket = var.automation.outputs_bucket
+ stage_name = k
+ })
+ )
+ }
+ folder_ids = merge(
+ {
+ data-platform-dev = try(module.branch-dp-dev-folder.0.id, null)
+ data-platform-prod = try(module.branch-dp-prod-folder.0.id, null)
+ gke-dev = try(module.branch-gke-dev-folder.0.id, null)
+ gke-prod = try(module.branch-gke-prod-folder.0.id, null)
+ networking = module.branch-network-folder.id
+ networking-dev = module.branch-network-dev-folder.id
+ networking-prod = module.branch-network-prod-folder.id
+ sandbox = try(module.branch-sandbox-folder.0.id, null)
+ security = module.branch-security-folder.id
+ teams = try(module.branch-teams-folder.0.id, null)
+ },
+ {
+ for k, v in module.branch-teams-team-folder :
+ "team-${k}" => v.id
+ },
+ {
+ for k, v in module.branch-teams-team-dev-folder :
+ "team-${k}-dev" => v.id
+ },
+ {
+ for k, v in module.branch-teams-team-prod-folder :
+ "team-${k}-prod" => v.id
+ }
+ )
+ providers = merge(
+ {
+ "2-networking" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-network-gcs.name
+ name = "networking"
+ sa = module.branch-network-sa.email
+ })
+ "2-security" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-security-gcs.name
+ name = "security"
+ sa = module.branch-security-sa.email
+ })
+ },
+ !var.fast_features.data_platform ? {} : {
+ "3-data-platform-dev" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-dp-dev-gcs.0.name
+ name = "dp-dev"
+ sa = module.branch-dp-dev-sa.0.email
+ })
+ "3-data-platform-prod" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-dp-prod-gcs.0.name
+ name = "dp-prod"
+ sa = module.branch-dp-prod-sa.0.email
+ })
+ },
+ !var.fast_features.gke ? {} : {
+ "3-gke-dev" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-gke-dev-gcs.0.name
+ name = "gke-dev"
+ sa = module.branch-gke-dev-sa.0.email
+ })
+ "3-gke-prod" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-gke-prod-gcs.0.name
+ name = "gke-prod"
+ sa = module.branch-gke-prod-sa.0.email
+ })
+ },
+ !var.fast_features.project_factory ? {} : {
+ "3-project-factory-dev" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-pf-dev-gcs.0.name
+ name = "team-dev"
+ sa = var.automation.service_accounts.pf-dev
+ })
+ "3-project-factory-prod" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-pf-prod-gcs.0.name
+ name = "team-prod"
+ sa = var.automation.service_accounts.pf-prod
+ })
+ },
+ !var.fast_features.sandbox ? {} : {
+ "9-sandbox" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-sandbox-gcs.0.name
+ name = "sandbox"
+ sa = var.automation.service_accounts.sandbox
+ })
+ },
+ !var.fast_features.teams ? {} : merge(
+ {
+ "3-teams" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-teams-gcs.0.name
+ name = "teams"
+ sa = module.branch-teams-sa.0.email
+ })
+ },
+ {
+ for k, v in module.branch-teams-team-sa :
+ "3-teams-${k}" => templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-teams-team-gcs[k].name
+ name = "teams"
+ sa = v.email
+ })
+ }
+ )
+ )
+ tfvars = {
+ folder_ids = local.folder_ids
+ }
+}
+
+output "cicd_repositories" {
+ description = "WIF configuration for CI/CD repositories."
+ value = {
+ for k, v in local.cicd_repositories : k => {
+ branch = v.branch
+ name = v.name
+ provider = try(
+ local.cicd_identity_providers[v.identity_provider].name, null
+ )
+ service_account = local.cicd_workflow_attrs[k].service_account
+ } if v != null
+ }
+}
+
+output "dataplatform" {
+ description = "Data for the Data Platform stage."
+ value = !var.fast_features.data_platform ? {} : {
+ dev = {
+ folder = module.branch-dp-dev-folder.0.id
+ gcs_bucket = module.branch-dp-dev-gcs.0.name
+ service_account = module.branch-dp-dev-sa.0.email
+ }
+ prod = {
+ folder = module.branch-dp-prod-folder.0.id
+ gcs_bucket = module.branch-dp-prod-gcs.0.name
+ service_account = module.branch-dp-prod-sa.0.email
+ }
+ }
+}
+
+output "gke_multitenant" {
+ # tfdoc:output:consumers 03-gke-multitenant
+ description = "Data for the GKE multitenant stage."
+ value = (
+ var.fast_features.gke
+ ? {
+ "dev" = {
+ folder = module.branch-gke-dev-folder.0.id
+ gcs_bucket = module.branch-gke-dev-gcs.0.name
+ service_account = module.branch-gke-dev-sa.0.email
+ }
+ "prod" = {
+ folder = module.branch-gke-prod-folder.0.id
+ gcs_bucket = module.branch-gke-prod-gcs.0.name
+ service_account = module.branch-gke-prod-sa.0.email
+ }
+ }
+ : {}
+ )
+}
+
+output "networking" {
+ description = "Data for the networking stage."
+ value = {
+ folder = module.branch-network-folder.id
+ gcs_bucket = module.branch-network-gcs.name
+ service_account = module.branch-network-sa.iam_email
+ }
+}
+
+output "project_factories" {
+ description = "Data for the project factories stage."
+ value = !var.fast_features.project_factory ? {} : {
+ dev = {
+ bucket = module.branch-pf-dev-gcs.0.name
+ sa = var.automation.service_accounts.pf-dev
+ }
+ prod = {
+ bucket = module.branch-pf-prod-gcs.0.name
+ sa = var.automation.service_accounts.pf-prod
+ }
+ }
+}
+
+# ready to use provider configurations for subsequent stages
+output "providers" {
+ # tfdoc:output:consumers 02-networking 02-security 03-dataplatform xx-sandbox xx-teams
+ description = "Terraform provider files for this stage and dependent stages."
+ sensitive = true
+ value = local.providers
+}
+
+output "sandbox" {
+ # tfdoc:output:consumers xx-sandbox
+ description = "Data for the sandbox stage."
+ value = (
+ var.fast_features.sandbox
+ ? {
+ folder = module.branch-sandbox-folder.0.id
+ gcs_bucket = module.branch-sandbox-gcs.0.name
+ service_account = var.automation.service_accounts.sandbox
+ }
+ : null
+ )
+}
+
+output "security" {
+ # tfdoc:output:consumers 02-security
+ description = "Data for the networking stage."
+ value = {
+ folder = module.branch-security-folder.id
+ gcs_bucket = module.branch-security-gcs.name
+ service_account = module.branch-security-sa.iam_email
+ }
+}
+
+output "teams" {
+ description = "Data for the teams stage."
+ value = {
+ for k, v in module.branch-teams-team-folder : k => {
+ folder = v.id
+ gcs_bucket = module.branch-teams-team-gcs[k].name
+ service_account = module.branch-teams-team-sa[k].email
+ }
+ }
+}
+
+# ready to use variable values for subsequent stages
+output "tfvars" {
+ description = "Terraform variable files for the following stages."
+ sensitive = true
+ value = local.tfvars
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/root_node.tf b/fast/stages-multitenant/1-resman-tenant/root_node.tf
new file mode 100644
index 00000000..5b83d2dd
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/root_node.tf
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Tenant root folder configuration.
+
+module "root-folder" {
+ source = "../../../modules/folder"
+ id = var.root_node
+ folder_create = var.test_skip_data_sources
+ # start test attributes
+ parent = (
+ var.test_skip_data_sources ? "organizations/${var.organization.id}" : null
+ )
+ name = var.test_skip_data_sources ? "Test" : null
+ # end test attributes
+ iam_additive = {
+ "roles/accesscontextmanager.policyAdmin" = [
+ local.automation_sas_iam.security
+ ]
+ "roles/compute.orgFirewallPolicyAdmin" = [
+ local.automation_sas_iam.networking
+ ]
+ "roles/compute.xpnAdmin" = [
+ local.automation_sas_iam.networking
+ ]
+ }
+ org_policies_data_path = var.organization_policy_data_path
+}
diff --git a/fast/stages-multitenant/1-resman-tenant/templates/providers.tf.tpl b/fast/stages-multitenant/1-resman-tenant/templates/providers.tf.tpl
new file mode 100644
index 00000000..993c78ca
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/templates/providers.tf.tpl
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+ backend "gcs" {
+ bucket = "${bucket}"
+ impersonate_service_account = "${sa}"
+ %{~ if backend_extra != null ~}
+ ${indent(4, backend_extra)}
+ %{~ endif ~}
+ }
+}
+provider "google" {
+ impersonate_service_account = "${sa}"
+}
+provider "google-beta" {
+ impersonate_service_account = "${sa}"
+}
+
+# end provider.tf for ${name}
diff --git a/fast/stages-multitenant/1-resman-tenant/templates/workflow-github.yaml b/fast/stages-multitenant/1-resman-tenant/templates/workflow-github.yaml
new file mode 100644
index 00000000..e9669909
--- /dev/null
+++ b/fast/stages-multitenant/1-resman-tenant/templates/workflow-github.yaml
@@ -0,0 +1,198 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "FAST ${stage_name} stage"
+
+on:
+ pull_request:
+ branches:
+ - main
+ types:
+ - closed
+ - opened
+ - synchronize
+
+env:
+ FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+ FAST_SERVICE_ACCOUNT: ${service_account}
+ FAST_WIF_PROVIDER: ${identity_provider}
+ SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+ TF_PROVIDERS_FILE: ${tf_providers_file}
+ TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)}
+ TF_VERSION: 1.3.2
+
+jobs:
+ fast-pr:
+ permissions:
+ contents: read
+ id-token: write
+ issues: write
+ pull-requests: write
+ runs-on: ubuntu-latest
+ steps:
+ - id: checkout
+ name: Checkout repository
+ uses: actions/checkout@v3
+
+ # set up SSH key authentication to the modules repository
+ - id: ssh-config
+ name: Configure SSH authentication
+ run: |
+ ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
+ ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
+
+ # set up authentication via Workload identity Federation
+ - id: gcp-auth
+ name: Authenticate to Google Cloud
+ uses: google-github-actions/auth@v0
+ with:
+ workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+ service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+ access_token_lifetime: 3600s
+
+ - id: gcp-sdk
+ name: Set up Cloud SDK
+ uses: google-github-actions/setup-gcloud@v0
+ with:
+ install_components: alpha
+
+ # copy provider and tfvars files
+ - id: tf-config
+ name: Copy Terraform output files
+ run: |
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+ for f in $${{env.TF_VAR_FILES}}; do
+ ln -s "tfvars/$f" ./
+ done
+
+ - id: tf-setup
+ name: Set up Terraform
+ uses: hashicorp/setup-terraform@v2.0.3
+ with:
+ terraform_version: $${{ env.TF_VERSION }}
+
+ # run Terraform init/validate/plan
+ - id: tf-init
+ name: Terraform init
+ continue-on-error: true
+ run: |
+ terraform init -no-color
+
+ - id: tf-validate
+ name: Terraform validate
+ continue-on-error: true
+ run: terraform validate -no-color
+
+ - id: tf-plan
+ name: Terraform plan
+ continue-on-error: true
+ run: |
+ terraform plan -input=false -out ../plan.out -no-color
+
+ - id: tf-apply
+ if: github.event.pull_request.merged == true && success()
+ name: Terraform apply
+ continue-on-error: true
+ run: |
+ terraform apply -input=false -auto-approve -no-color ../plan.out
+
+ - id: pr-comment
+ name: Post comment to Pull Request
+ continue-on-error: true
+ uses: actions/github-script@v6
+ if: github.event_name == 'pull_request'
+ env:
+ PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+ with:
+ script: |
+ const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+
+ ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+
+
+
+
gcs · iam-service-account · project | |
-| [billing.tf](./billing.tf) | Billing export project and dataset. | bigquery-dataset · organization · project | google_billing_account_iam_member · google_organization_iam_binding |
+| [billing.tf](./billing.tf) | Billing export project and dataset. | bigquery-dataset · project | google_billing_account_iam_member |
| [cicd.tf](./cicd.tf) | Workload Identity Federation configurations for CI/CD. | iam-service-account · source-repository | |
| [identity-providers.tf](./identity-providers.tf) | Workload Identity Federation provider definitions. | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider |
| [log-export.tf](./log-export.tf) | Audit log project and sink. | bigquery-dataset · gcs · logging-bucket · project · pubsub | |
@@ -468,35 +468,35 @@ The remaining configuration is manual, as it regards the repositories themselves
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | |
-| [organization](variables.tf#L202) | Organization details. | object({…}) | ✓ | | |
-| [prefix](variables.tf#L217) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | |
-| [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | string | | null | |
-| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | |
-| [custom_role_names](variables.tf#L83) | Names of custom roles defined at the org level. | object({…}) | | {…} | |
-| [fast_features](variables.tf#L95) | Selective control for top-level FAST features. | object({…}) | | {…} | |
-| [federated_identity_providers](variables.tf#L114) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | |
-| [groups](variables.tf#L128) | Group names to grant organization-level permissions. | map(string) | | {…} | |
-| [iam](variables.tf#L146) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | |
-| [iam_additive](variables.tf#L152) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | |
-| [locations](variables.tf#L158) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | |
-| [log_sinks](variables.tf#L177) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | |
-| [outputs_location](variables.tf#L211) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | |
-| [project_parent_ids](variables.tf#L227) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the organization as parent. | object({…}) | | {…} | |
+| [billing_account](variables.tf#L17) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | |
+| [organization](variables.tf#L196) | Organization details. | object({…}) | ✓ | | |
+| [prefix](variables.tf#L211) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | |
+| [bootstrap_user](variables.tf#L29) | Email of the nominal user running this stage for the first time. | string | | null | |
+| [cicd_repositories](variables.tf#L35) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | |
+| [custom_role_names](variables.tf#L81) | Names of custom roles defined at the org level. | object({…}) | | {…} | |
+| [fast_features](variables.tf#L95) | Selective control for top-level FAST features. | object({…}) | | {} | |
+| [federated_identity_providers](variables.tf#L108) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | |
+| [groups](variables.tf#L122) | Group names to grant organization-level permissions. | map(string) | | {…} | |
+| [iam](variables.tf#L140) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | |
+| [iam_additive](variables.tf#L146) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | |
+| [locations](variables.tf#L152) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | |
+| [log_sinks](variables.tf#L171) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | |
+| [outputs_location](variables.tf#L205) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | |
+| [project_parent_ids](variables.tf#L221) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the organization as parent. | object({…}) | | {…} | |
## Outputs
| name | description | sensitive | consumers |
|---|---|:---:|---|
-| [automation](outputs.tf#L89) | Automation resources. | | |
-| [billing_dataset](outputs.tf#L94) | BigQuery dataset prepared for billing export. | | |
-| [cicd_repositories](outputs.tf#L99) | CI/CD repository configurations. | | |
-| [custom_roles](outputs.tf#L111) | Organization-level custom roles. | | |
-| [federated_identity](outputs.tf#L116) | Workload Identity Federation pool and providers. | | |
-| [outputs_bucket](outputs.tf#L126) | GCS bucket where generated output files are stored. | | |
-| [project_ids](outputs.tf#L131) | Projects created by this stage. | | |
-| [providers](outputs.tf#L141) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 |
-| [service_accounts](outputs.tf#L148) | Automation service accounts created by this stage. | | |
-| [tfvars](outputs.tf#L158) | Terraform variable files for the following stages. | ✓ | |
+| [automation](outputs.tf#L86) | Automation resources. | | |
+| [billing_dataset](outputs.tf#L91) | BigQuery dataset prepared for billing export. | | |
+| [cicd_repositories](outputs.tf#L96) | CI/CD repository configurations. | | |
+| [custom_roles](outputs.tf#L108) | Organization-level custom roles. | | |
+| [federated_identity](outputs.tf#L113) | Workload Identity Federation pool and providers. | | |
+| [outputs_bucket](outputs.tf#L123) | GCS bucket where generated output files are stored. | | |
+| [project_ids](outputs.tf#L128) | Projects created by this stage. | | |
+| [providers](outputs.tf#L138) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 |
+| [service_accounts](outputs.tf#L145) | Automation service accounts created by this stage. | | |
+| [tfvars](outputs.tf#L154) | Terraform variable files for the following stages. | ✓ | |
diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/0-bootstrap/automation.tf
similarity index 82%
rename from fast/stages/00-bootstrap/automation.tf
rename to fast/stages/0-bootstrap/automation.tf
index 1475c811..90c14a81 100644
--- a/fast/stages/00-bootstrap/automation.tf
+++ b/fast/stages/0-bootstrap/automation.tf
@@ -127,39 +127,6 @@ module "automation-tf-bootstrap-sa" {
}
}
-# cicd stage's bucket and service account
-
-module "automation-tf-cicd-gcs" {
- source = "../../../modules/gcs"
- project_id = module.automation-project.project_id
- name = "iac-core-cicd-0"
- prefix = local.prefix
- location = var.locations.gcs
- storage_class = local.gcs_storage_class
- versioning = true
- iam = {
- "roles/storage.objectAdmin" = [module.automation-tf-cicd-provisioning-sa.iam_email]
- }
- depends_on = [module.organization]
-}
-
-module "automation-tf-cicd-provisioning-sa" {
- source = "../../../modules/iam-service-account"
- project_id = module.automation-project.project_id
- name = "cicd-0"
- display_name = "Terraform stage 1 CICD service account."
- prefix = local.prefix
- # allow SA used by CI/CD workflow to impersonate this SA
- iam = {
- "roles/iam.serviceAccountTokenCreator" = compact([
- try(module.automation-tf-cicd-sa["cicd"].iam_email, null)
- ])
- }
- iam_storage_roles = {
- (module.automation-tf-output-gcs.name) = ["roles/storage.admin"]
- }
-}
-
# resource hierarchy stage's bucket and service account
module "automation-tf-resman-gcs" {
@@ -183,7 +150,8 @@ module "automation-tf-resman-sa" {
display_name = "Terraform stage 1 resman service account."
prefix = local.prefix
# allow SA used by CI/CD workflow to impersonate this SA
- iam = {
+ # we use additive IAM to allow tenant CI/CD SAs to impersonate it
+ iam_additive = {
"roles/iam.serviceAccountTokenCreator" = compact([
try(module.automation-tf-cicd-sa["resman"].iam_email, null)
])
diff --git a/fast/stages/00-bootstrap/billing.tf b/fast/stages/0-bootstrap/billing.tf
similarity index 60%
rename from fast/stages/00-bootstrap/billing.tf
rename to fast/stages/0-bootstrap/billing.tf
index df10e8f0..aee033bd 100644
--- a/fast/stages/00-bootstrap/billing.tf
+++ b/fast/stages/0-bootstrap/billing.tf
@@ -30,7 +30,7 @@ locals {
module "billing-export-project" {
source = "../../../modules/project"
- count = local.billing_org ? 1 : 0
+ count = var.billing_account.is_org_level ? 1 : 0
billing_account = var.billing_account.id
name = "billing-exp-0"
parent = coalesce(
@@ -52,56 +52,18 @@ module "billing-export-project" {
module "billing-export-dataset" {
source = "../../../modules/bigquery-dataset"
- count = local.billing_org ? 1 : 0
+ count = var.billing_account.is_org_level ? 1 : 0
project_id = module.billing-export-project.0.project_id
id = "billing_export"
friendly_name = "Billing export."
location = var.locations.bq
}
-# billing account in a different org
-
-module "billing-organization-ext" {
- source = "../../../modules/organization"
- count = local.billing_org_ext ? 1 : 0
- organization_id = "organizations/${var.billing_account.organization_id}"
- iam_additive = {
- "roles/billing.admin" = local.billing_ext_admins
- }
-}
-
-
-resource "google_organization_iam_binding" "billing_org_ext_admin_delegated" {
- # refer to organization.tf for the explanation of how this binding works
- count = local.billing_org_ext ? 1 : 0
- org_id = var.billing_account.organization_id
- # if the billing org does not have our custom role, user the predefined one
- # role = "roles/resourcemanager.organizationAdmin"
- role = join("", [
- "organizations/${var.billing_account.organization_id}/",
- "roles/${var.custom_role_names.organization_iam_admin}"
- ])
- members = [module.automation-tf-resman-sa.iam_email]
- condition {
- title = "automation_sa_delegated_grants"
- description = "Automation service account delegated grants."
- expression = format(
- "api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly([%s])",
- join(",", formatlist("'%s'", [
- "roles/billing.costsManager",
- "roles/billing.user",
- ]
- ))
- )
- }
- depends_on = [module.billing-organization-ext]
-}
-
# standalone billing account
resource "google_billing_account_iam_member" "billing_ext_admin" {
for_each = toset(
- local.billing_ext ? local.billing_ext_admins : []
+ !var.billing_account.is_org_level ? local.billing_ext_admins : []
)
billing_account_id = var.billing_account.id
role = "roles/billing.admin"
@@ -110,7 +72,7 @@ resource "google_billing_account_iam_member" "billing_ext_admin" {
resource "google_billing_account_iam_member" "billing_ext_cost_manager" {
for_each = toset(
- local.billing_ext ? local.billing_ext_admins : []
+ !var.billing_account.is_org_level ? local.billing_ext_admins : []
)
billing_account_id = var.billing_account.id
role = "roles/billing.costsManager"
diff --git a/fast/stages/00-bootstrap/cicd.tf b/fast/stages/0-bootstrap/cicd.tf
similarity index 86%
rename from fast/stages/00-bootstrap/cicd.tf
rename to fast/stages/0-bootstrap/cicd.tf
index 7cdae41c..2b2a3df9 100644
--- a/fast/stages/00-bootstrap/cicd.tf
+++ b/fast/stages/0-bootstrap/cicd.tf
@@ -17,6 +17,16 @@
# tfdoc:file:description Workload Identity Federation configurations for CI/CD.
locals {
+ cicd_providers = {
+ for k, v in google_iam_workload_identity_pool_provider.default :
+ k => {
+ issuer = local.identity_providers[k].issuer
+ issuer_uri = local.identity_providers[k].issuer_uri
+ name = v.name
+ principal_tpl = local.identity_providers[k].principal_tpl
+ principalset_tpl = local.identity_providers[k].principalset_tpl
+ }
+ }
cicd_repositories = {
for k, v in coalesce(var.cicd_repositories, {}) : k => v
if(
@@ -32,18 +42,13 @@ locals {
)
}
cicd_workflow_providers = {
- bootstrap = "00-bootstrap-providers.tf"
- cicd = "00-cicd-providers.tf"
- resman = "01-resman-providers.tf"
+ bootstrap = "0-bootstrap-providers.tf"
+ resman = "1-resman-providers.tf"
}
cicd_workflow_var_files = {
bootstrap = []
- cicd = [
- "00-bootstrap.auto.tfvars.json",
- "globals.auto.tfvars.json"
- ]
resman = [
- "00-bootstrap.auto.tfvars.json",
+ "0-bootstrap.auto.tfvars.json",
"globals.auto.tfvars.json"
]
}
@@ -69,7 +74,7 @@ module "automation-tf-cicd-repo" {
]
}
triggers = {
- "fast-00-${each.key}" = {
+ "fast-0-${each.key}" = {
filename = ".cloudbuild/workflow.yaml"
included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
service_account = module.automation-tf-cicd-sa[each.key].id
diff --git a/fast/stages/00-bootstrap/diagram.png b/fast/stages/0-bootstrap/diagram.png
similarity index 100%
rename from fast/stages/00-bootstrap/diagram.png
rename to fast/stages/0-bootstrap/diagram.png
diff --git a/fast/stages/00-bootstrap/diagram.svg b/fast/stages/0-bootstrap/diagram.svg
similarity index 100%
rename from fast/stages/00-bootstrap/diagram.svg
rename to fast/stages/0-bootstrap/diagram.svg
diff --git a/fast/stages/00-bootstrap/groups.gif b/fast/stages/0-bootstrap/groups.gif
similarity index 100%
rename from fast/stages/00-bootstrap/groups.gif
rename to fast/stages/0-bootstrap/groups.gif
diff --git a/fast/stages/00-bootstrap/identity-providers.tf b/fast/stages/0-bootstrap/identity-providers.tf
similarity index 100%
rename from fast/stages/00-bootstrap/identity-providers.tf
rename to fast/stages/0-bootstrap/identity-providers.tf
diff --git a/fast/stages/00-bootstrap/log-export.tf b/fast/stages/0-bootstrap/log-export.tf
similarity index 82%
rename from fast/stages/00-bootstrap/log-export.tf
rename to fast/stages/0-bootstrap/log-export.tf
index 1c9f5a87..b9b5da42 100644
--- a/fast/stages/00-bootstrap/log-export.tf
+++ b/fast/stages/0-bootstrap/log-export.tf
@@ -17,6 +17,16 @@
# tfdoc:file:description Audit log project and sink.
locals {
+ log_sink_destinations = merge(
+ # use the same dataset for all sinks with `bigquery` as destination
+ { for k, v in var.log_sinks : k => module.log-export-dataset.0 if v.type == "bigquery" },
+ # use the same gcs bucket for all sinks with `storage` as destination
+ { for k, v in var.log_sinks : k => module.log-export-gcs.0 if v.type == "storage" },
+ # use separate pubsub topics and logging buckets for sinks with
+ # destination `pubsub` and `logging`
+ module.log-export-pubsub,
+ module.log-export-logbucket
+ )
log_types = toset([for k, v in var.log_sinks : v.type])
}
diff --git a/fast/stages/00-bootstrap/main.tf b/fast/stages/0-bootstrap/main.tf
similarity index 78%
rename from fast/stages/00-bootstrap/main.tf
rename to fast/stages/0-bootstrap/main.tf
index 839019f3..dba2ed08 100644
--- a/fast/stages/00-bootstrap/main.tf
+++ b/fast/stages/0-bootstrap/main.tf
@@ -28,10 +28,6 @@ locals {
for k, v in local.groups :
k => "group:${v}"
}
- # convenience flags that express where billing account resides
- billing_ext = var.billing_account.organization_id == null
- billing_org = var.billing_account.organization_id == var.organization.id
- billing_org_ext = !local.billing_ext && !local.billing_org
# naming: environment used in most resource names
prefix = join("-", compact([var.prefix, "prod"]))
}
diff --git a/fast/stages/00-bootstrap/organization.tf b/fast/stages/0-bootstrap/organization.tf
similarity index 89%
rename from fast/stages/00-bootstrap/organization.tf
rename to fast/stages/0-bootstrap/organization.tf
index 33b87820..154b37fe 100644
--- a/fast/stages/00-bootstrap/organization.tf
+++ b/fast/stages/0-bootstrap/organization.tf
@@ -24,7 +24,10 @@ locals {
"domain:${var.organization.domain}"
]
"roles/logging.admin" = concat(
- [module.automation-tf-bootstrap-sa.iam_email],
+ [
+ module.automation-tf-bootstrap-sa.iam_email,
+ module.automation-tf-resman-sa.iam_email
+ ],
local._iam_bootstrap_user
)
"roles/owner" = local._iam_bootstrap_user
@@ -35,12 +38,11 @@ locals {
[module.automation-tf-bootstrap-sa.iam_email],
local._iam_bootstrap_user
)
- # the following is useful if roles/browser is not desirable
- # "roles/resourcemanager.organizationViewer" = [
- # "domain:${var.organization.domain}"
- # ]
"roles/resourcemanager.projectCreator" = concat(
- [module.automation-tf-bootstrap-sa.iam_email],
+ [
+ module.automation-tf-bootstrap-sa.iam_email,
+ module.automation-tf-resman-sa.iam_email
+ ],
local._iam_bootstrap_user
)
"roles/resourcemanager.projectMover" = [
@@ -78,8 +80,12 @@ locals {
local.groups_iam.gcp-security-admins,
module.automation-tf-resman-sa.iam_email
]
+ # the following is useful if roles/browser is not desirable
+ # "roles/resourcemanager.organizationViewer" = [
+ # "domain:${var.organization.domain}"
+ # ]
},
- local.billing_org ? {
+ var.billing_account.is_org_level ? {
"roles/billing.admin" = [
local.groups_iam.gcp-billing-admins,
local.groups_iam.gcp-organization-admins,
@@ -115,16 +121,6 @@ locals {
iam_roles_additive = distinct(concat(
keys(local._iam_additive), keys(var.iam_additive)
))
- log_sink_destinations = merge(
- # use the same dataset for all sinks with `bigquery` as destination
- { for k, v in var.log_sinks : k => module.log-export-dataset.0 if v.type == "bigquery" },
- # use the same gcs bucket for all sinks with `storage` as destination
- { for k, v in var.log_sinks : k => module.log-export-gcs.0 if v.type == "storage" },
- # use separate pubsub topics and logging buckets for sinks with
- # destination `pubsub` and `logging`
- module.log-export-pubsub,
- module.log-export-logbucket
- )
}
module "organization" {
@@ -190,6 +186,9 @@ module "organization" {
"dns.networks.bindPrivateDNSZone",
"resourcemanager.projects.get",
]
+ (var.custom_role_names.tenant_network_admin) = [
+ "compute.globalOperations.get",
+ ]
}
logging_sinks = {
for name, attrs in var.log_sinks : name => {
@@ -220,8 +219,10 @@ resource "google_organization_iam_binding" "org_admin_delegated" {
"roles/compute.orgFirewallPolicyAdmin",
"roles/compute.xpnAdmin",
"roles/orgpolicy.policyAdmin",
+ "roles/resourcemanager.organizationViewer",
+ module.organization.custom_role_id[var.custom_role_names.tenant_network_admin]
],
- local.billing_org ? [
+ var.billing_account.is_org_level ? [
"roles/billing.admin",
"roles/billing.costsManager",
"roles/billing.user",
diff --git a/fast/stages/00-bootstrap/outputs-files.tf b/fast/stages/0-bootstrap/outputs-files.tf
similarity index 97%
rename from fast/stages/00-bootstrap/outputs-files.tf
rename to fast/stages/0-bootstrap/outputs-files.tf
index ded88cd5..90e195b5 100644
--- a/fast/stages/00-bootstrap/outputs-files.tf
+++ b/fast/stages/0-bootstrap/outputs-files.tf
@@ -26,7 +26,7 @@ resource "local_file" "providers" {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/00-bootstrap.auto.tfvars.json"
+ filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/0-bootstrap.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/00-bootstrap/outputs-gcs.tf b/fast/stages/0-bootstrap/outputs-gcs.tf
similarity index 96%
rename from fast/stages/00-bootstrap/outputs-gcs.tf
rename to fast/stages/0-bootstrap/outputs-gcs.tf
index 2c281d4c..0aded986 100644
--- a/fast/stages/00-bootstrap/outputs-gcs.tf
+++ b/fast/stages/0-bootstrap/outputs-gcs.tf
@@ -26,7 +26,7 @@ resource "google_storage_bucket_object" "providers" {
resource "google_storage_bucket_object" "tfvars" {
bucket = module.automation-tf-output-gcs.name
- name = "tfvars/00-bootstrap.auto.tfvars.json"
+ name = "tfvars/0-bootstrap.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/0-bootstrap/outputs.tf
similarity index 77%
rename from fast/stages/00-bootstrap/outputs.tf
rename to fast/stages/0-bootstrap/outputs.tf
index 73dd64f4..364abd68 100644
--- a/fast/stages/00-bootstrap/outputs.tf
+++ b/fast/stages/0-bootstrap/outputs.tf
@@ -21,7 +21,7 @@ locals {
for k, v in local.cicd_repositories : k => templatefile(
"${path.module}/templates/workflow-${v.type}.yaml", {
identity_provider = try(
- local.wif_providers[v["identity_provider"]].name, ""
+ local.cicd_providers[v["identity_provider"]].name, ""
)
outputs_bucket = module.automation-tf-output-gcs.name
service_account = try(
@@ -38,19 +38,26 @@ locals {
k => try(module.organization.custom_role_id[v], null)
}
providers = {
- "00-bootstrap" = templatefile(local._tpl_providers, {
- bucket = module.automation-tf-bootstrap-gcs.name
- name = "bootstrap"
- sa = module.automation-tf-bootstrap-sa.email
+ "0-bootstrap" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.automation-tf-bootstrap-gcs.name
+ name = "bootstrap"
+ sa = module.automation-tf-bootstrap-sa.email
})
- "00-cicd" = templatefile(local._tpl_providers, {
- bucket = module.automation-tf-cicd-gcs.name
- name = "cicd"
- sa = module.automation-tf-cicd-provisioning-sa.email
+ "1-resman" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.automation-tf-resman-gcs.name
+ name = "resman"
+ sa = module.automation-tf-resman-sa.email
})
- "01-resman" = templatefile(local._tpl_providers, {
+ "0-bootstrap-tenant" = templatefile(local._tpl_providers, {
+ backend_extra = join("\n", [
+ "# remove the newline between quotes and set the tenant name as prefix",
+ "prefix = \"",
+ "\""
+ ])
bucket = module.automation-tf-resman-gcs.name
- name = "resman"
+ name = "bootstrap-tenant"
sa = module.automation-tf-resman-sa.email
})
}
@@ -59,7 +66,7 @@ locals {
federated_identity_pool = try(
google_iam_workload_identity_pool.default.0.name, null
)
- federated_identity_providers = local.wif_providers
+ federated_identity_providers = local.cicd_providers
outputs_bucket = module.automation-tf-output-gcs.name
project_id = module.automation-project.project_id
project_number = module.automation-project.number
@@ -74,16 +81,6 @@ locals {
organization = var.organization
prefix = var.prefix
}
- wif_providers = {
- for k, v in google_iam_workload_identity_pool_provider.default :
- k => {
- issuer = local.identity_providers[k].issuer
- issuer_uri = local.identity_providers[k].issuer_uri
- name = v.name
- principal_tpl = local.identity_providers[k].principal_tpl
- principalset_tpl = local.identity_providers[k].principalset_tpl
- }
- }
}
output "automation" {
@@ -102,7 +99,7 @@ output "cicd_repositories" {
for k, v in local.cicd_repositories : k => {
branch = v.branch
name = v.name
- provider = try(local.wif_providers[v.identity_provider].name, null)
+ provider = try(local.cicd_providers[v.identity_provider].name, null)
service_account = try(module.automation-tf-cicd-sa[k].email, null)
}
}
@@ -119,7 +116,7 @@ output "federated_identity" {
pool = try(
google_iam_workload_identity_pool.default.0.name, null
)
- providers = local.wif_providers
+ providers = local.cicd_providers
}
}
@@ -149,7 +146,6 @@ output "service_accounts" {
description = "Automation service accounts created by this stage."
value = {
bootstrap = module.automation-tf-bootstrap-sa.email
- cicd = module.automation-tf-cicd-provisioning-sa.email
resman = module.automation-tf-resman-sa.email
}
}
diff --git a/tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/variables.tf b/fast/stages/0-bootstrap/templates/providers.tf.tpl
similarity index 62%
rename from tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/variables.tf
rename to fast/stages/0-bootstrap/templates/providers.tf.tpl
index 6a534739..d1c224c5 100644
--- a/tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/variables.tf
+++ b/fast/stages/0-bootstrap/templates/providers.tf.tpl
@@ -14,13 +14,20 @@
* limitations under the License.
*/
-variable "billing_account" {
- type = string
- default = "123456-123456-123456"
+terraform {
+ backend "gcs" {
+ bucket = "${bucket}"
+ impersonate_service_account = "${sa}"
+ %{~ if backend_extra != null ~}
+ ${indent(4, backend_extra)}
+ %{~ endif ~}
+ }
+}
+provider "google" {
+ impersonate_service_account = "${sa}"
+}
+provider "google-beta" {
+ impersonate_service_account = "${sa}"
}
-variable "root_node" {
- description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id."
- type = string
- default = "folders/12345678"
-}
+# end provider.tf for ${name}
diff --git a/fast/stages/0-bootstrap/templates/workflow-github.yaml b/fast/stages/0-bootstrap/templates/workflow-github.yaml
new file mode 100644
index 00000000..87b5ae1a
--- /dev/null
+++ b/fast/stages/0-bootstrap/templates/workflow-github.yaml
@@ -0,0 +1,202 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "FAST ${stage_name} stage"
+
+on:
+ pull_request:
+ branches:
+ - main
+ types:
+ - closed
+ - opened
+ - synchronize
+
+env:
+ FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+ FAST_SERVICE_ACCOUNT: ${service_account}
+ FAST_WIF_PROVIDER: ${identity_provider}
+ SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+ TF_PROVIDERS_FILE: ${tf_providers_file}
+ %{~ if tf_var_files != [] ~}
+ TF_VAR_FILES: ${join("\n ", tf_var_files)}
+ %{~ endif ~}
+ TF_VERSION: 1.3.2
+
+jobs:
+ fast-pr:
+ permissions:
+ contents: read
+ id-token: write
+ issues: write
+ pull-requests: write
+ runs-on: ubuntu-latest
+ steps:
+ - id: checkout
+ name: Checkout repository
+ uses: actions/checkout@v3
+
+ # set up SSH key authentication to the modules repository
+ - id: ssh-config
+ name: Configure SSH authentication
+ run: |
+ ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
+ ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
+
+ # set up authentication via Workload identity Federation
+ - id: gcp-auth
+ name: Authenticate to Google Cloud
+ uses: google-github-actions/auth@v0
+ with:
+ workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+ service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+ access_token_lifetime: 3600s
+
+ - id: gcp-sdk
+ name: Set up Cloud SDK
+ uses: google-github-actions/setup-gcloud@v0
+ with:
+ install_components: alpha
+
+ # copy provider and tfvars files
+ - id: tf-config
+ name: Copy Terraform output files
+ run: |
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+ %{~ if tf_var_files != [] ~}
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+ for f in $${{env.TF_VAR_FILES}}; do
+ ln -s "tfvars/$f" ./
+ done
+ %{~ endif ~}
+
+ - id: tf-setup
+ name: Set up Terraform
+ uses: hashicorp/setup-terraform@v2.0.3
+ with:
+ terraform_version: $${{ env.TF_VERSION }}
+
+ # run Terraform init/validate/plan
+ - id: tf-init
+ name: Terraform init
+ continue-on-error: true
+ run: |
+ terraform init -no-color
+
+ - id: tf-validate
+ continue-on-error: true
+ name: Terraform validate
+ run: terraform validate -no-color
+
+ - id: tf-plan
+ name: Terraform plan
+ continue-on-error: true
+ run: |
+ terraform plan -input=false -out ../plan.out -no-color
+
+ - id: tf-apply
+ if: github.event.pull_request.merged == true && success()
+ name: Terraform apply
+ continue-on-error: true
+ run: |
+ terraform apply -input=false -auto-approve -no-color ../plan.out
+
+ - id: pr-comment
+ name: Post comment to Pull Request
+ continue-on-error: true
+ uses: actions/github-script@v6
+ if: github.event_name == 'pull_request'
+ env:
+ PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+ with:
+ script: |
+ const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+
+ ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+
+ organization | google_billing_account_iam_member |
-| [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | folder · gcs · iam-service-account | |
+| [billing.tf](./billing.tf) | Billing resources for external billing use cases. | | google_billing_account_iam_member |
+| [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | folder · gcs · iam-service-account | google_organization_iam_member |
| [branch-gke.tf](./branch-gke.tf) | GKE multitenant stage resources. | folder · gcs · iam-service-account | |
| [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | folder · gcs · iam-service-account | |
-| [branch-project-factory.tf](./branch-project-factory.tf) | Project factory stage resources. | gcs · iam-service-account | |
+| [branch-project-factory.tf](./branch-project-factory.tf) | Project factory stage resources. | gcs · iam-service-account | google_organization_iam_member |
| [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | folder · gcs · iam-service-account | |
| [branch-security.tf](./branch-security.tf) | Security stage resources. | folder · gcs · iam-service-account | |
| [branch-teams.tf](./branch-teams.tf) | Team stage resources. | folder · gcs · iam-service-account | |
@@ -170,7 +174,7 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
| [cicd-project-factory.tf](./cicd-project-factory.tf) | CI/CD resources for the teams branch. | iam-service-account · source-repository | |
| [cicd-security.tf](./cicd-security.tf) | CI/CD resources for the security branch. | iam-service-account · source-repository | |
| [main.tf](./main.tf) | Module-level locals and resources. | | |
-| [organization.tf](./organization.tf) | Organization policies. | organization | google_organization_iam_member |
+| [organization.tf](./organization.tf) | Organization policies. | organization | |
| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file |
| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object |
| [outputs.tf](./outputs.tf) | Module outputs. | | |
@@ -180,34 +184,34 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L38) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [organization](variables.tf#L197) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L221) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [cicd_repositories](variables.tf#L47) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | |
-| [custom_roles](variables.tf#L129) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap |
-| [data_dir](variables.tf#L138) | Relative path for the folder storing configuration data. | string | | "data" | |
-| [fast_features](variables.tf#L144) | Selective control for top-level FAST features. | object({…}) | | {…} | 00-bootstrap |
-| [groups](variables.tf#L164) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap |
-| [locations](variables.tf#L179) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | 00-bootstrap |
-| [organization_policy_configs](variables.tf#L207) | Organization policies customization. | object({…}) | | null | |
-| [outputs_location](variables.tf#L215) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | |
-| [tag_names](variables.tf#L232) | Customized names for resource management tags. | object({…}) | | {…} | |
-| [team_folders](variables.tf#L249) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | |
+| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L38) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [organization](variables.tf#L193) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L217) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [cicd_repositories](variables.tf#L51) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | |
+| [custom_roles](variables.tf#L133) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [data_dir](variables.tf#L142) | Relative path for the folder storing configuration data. | string | | "data" | |
+| [fast_features](variables.tf#L148) | Selective control for top-level FAST features. | object({…}) | | {} | 0-0-bootstrap |
+| [groups](variables.tf#L162) | Group names to grant organization-level permissions. | object({…}) | | {} | 0-bootstrap |
+| [locations](variables.tf#L175) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | 0-bootstrap |
+| [organization_policy_configs](variables.tf#L203) | Organization policies customization. | object({…}) | | null | |
+| [outputs_location](variables.tf#L211) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | |
+| [tag_names](variables.tf#L228) | Customized names for resource management tags. | object({…}) | | {…} | |
+| [team_folders](variables.tf#L247) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | |
## Outputs
| name | description | sensitive | consumers |
|---|---|:---:|---|
-| [cicd_repositories](outputs.tf#L197) | WIF configuration for CI/CD repositories. | | |
-| [dataplatform](outputs.tf#L211) | Data for the Data Platform stage. | | |
-| [gke_multitenant](outputs.tf#L227) | Data for the GKE multitenant stage. | | 03-gke-multitenant |
-| [networking](outputs.tf#L248) | Data for the networking stage. | | |
-| [project_factories](outputs.tf#L257) | Data for the project factories stage. | | |
-| [providers](outputs.tf#L272) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams |
-| [sandbox](outputs.tf#L279) | Data for the sandbox stage. | | xx-sandbox |
-| [security](outputs.tf#L293) | Data for the networking stage. | | 02-security |
-| [teams](outputs.tf#L303) | Data for the teams stage. | | |
-| [tfvars](outputs.tf#L315) | Terraform variable files for the following stages. | ✓ | |
+| [cicd_repositories](outputs.tf#L210) | WIF configuration for CI/CD repositories. | | |
+| [dataplatform](outputs.tf#L224) | Data for the Data Platform stage. | | |
+| [gke_multitenant](outputs.tf#L240) | Data for the GKE multitenant stage. | | 03-gke-multitenant |
+| [networking](outputs.tf#L261) | Data for the networking stage. | | |
+| [project_factories](outputs.tf#L270) | Data for the project factories stage. | | |
+| [providers](outputs.tf#L285) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams |
+| [sandbox](outputs.tf#L292) | Data for the sandbox stage. | | xx-sandbox |
+| [security](outputs.tf#L306) | Data for the networking stage. | | 02-security |
+| [teams](outputs.tf#L316) | Data for the teams stage. | | |
+| [tfvars](outputs.tf#L328) | Terraform variable files for the following stages. | ✓ | |
diff --git a/fast/stages/01-resman/billing.tf b/fast/stages/1-resman/billing.tf
similarity index 77%
rename from fast/stages/01-resman/billing.tf
rename to fast/stages/1-resman/billing.tf
index fe497c7c..ba20ab05 100644
--- a/fast/stages/01-resman/billing.tf
+++ b/fast/stages/1-resman/billing.tf
@@ -34,23 +34,11 @@ locals {
# billing account in same org (resources is in the organization.tf file)
-# billing account in a different org
-
-module "billing-organization-ext" {
- source = "../../../modules/organization"
- count = local.billing_org_ext ? 1 : 0
- organization_id = "organizations/${var.billing_account.organization_id}"
- iam_additive = {
- "roles/billing.user" = local.billing_ext_users
- "roles/billing.costsManager" = local.billing_ext_users
- }
-}
-
# standalone billing account
resource "google_billing_account_iam_member" "billing_ext_admin" {
for_each = toset(
- local.billing_ext ? local.billing_ext_users : []
+ !var.billing_account.is_org_level ? local.billing_ext_users : []
)
billing_account_id = var.billing_account.id
role = "roles/billing.user"
@@ -59,7 +47,7 @@ resource "google_billing_account_iam_member" "billing_ext_admin" {
resource "google_billing_account_iam_member" "billing_ext_costsmanager" {
for_each = toset(
- local.billing_ext ? local.billing_ext_users : []
+ !var.billing_account.is_org_level ? local.billing_ext_users : []
)
billing_account_id = var.billing_account.id
role = "roles/billing.costsManager"
diff --git a/fast/stages/01-resman/branch-data-platform.tf b/fast/stages/1-resman/branch-data-platform.tf
similarity index 86%
rename from fast/stages/01-resman/branch-data-platform.tf
rename to fast/stages/1-resman/branch-data-platform.tf
index 66cc9fbb..7a93e7a5 100644
--- a/fast/stages/01-resman/branch-data-platform.tf
+++ b/fast/stages/1-resman/branch-data-platform.tf
@@ -137,3 +137,22 @@ module "branch-dp-prod-gcs" {
"roles/storage.objectAdmin" = [module.branch-dp-prod-sa.0.iam_email]
}
}
+
+resource "google_organization_iam_member" "org_policy_admin_dp" {
+ for_each = !var.fast_features.data_platform ? {} : {
+ data-dev = ["data", "development", module.branch-dp-dev-sa.0.iam_email]
+ data-prod = ["data", "production", module.branch-dp-prod-sa.0.iam_email]
+ }
+ org_id = var.organization.id
+ role = "roles/orgpolicy.policyAdmin"
+ member = each.value.2
+ condition {
+ title = "org_policy_tag_dp_scoped"
+ description = "Org policy tag scoped grant for ${each.value.0}/${each.value.1}."
+ expression = <<-END
+ resource.matchTag('${var.organization.id}/${var.tag_names.context}', '${each.value.0}')
+ &&
+ resource.matchTag('${var.organization.id}/${var.tag_names.environment}', '${each.value.1}')
+ END
+ }
+}
diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/1-resman/branch-gke.tf
similarity index 94%
rename from fast/stages/01-resman/branch-gke.tf
rename to fast/stages/1-resman/branch-gke.tf
index 84ca41ed..76777d8f 100644
--- a/fast/stages/01-resman/branch-gke.tf
+++ b/fast/stages/1-resman/branch-gke.tf
@@ -77,7 +77,11 @@ module "branch-gke-dev-sa" {
prefix = var.prefix
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
- ["group:${local.groups.gcp-devops}"],
+ (
+ local.groups.gcp-devops == null
+ ? []
+ : ["group:${local.groups.gcp-devops}"]
+ ),
compact([
try(module.branch-gke-dev-sa-cicd.0.iam_email, null)
])
@@ -97,7 +101,11 @@ module "branch-gke-prod-sa" {
prefix = var.prefix
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
- ["group:${local.groups.gcp-devops}"],
+ (
+ local.groups.gcp-devops == null
+ ? []
+ : ["group:${local.groups.gcp-devops}"]
+ ),
compact([
try(module.branch-gke-prod-sa-cicd.0.iam_email, null)
])
diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/1-resman/branch-networking.tf
similarity index 98%
rename from fast/stages/01-resman/branch-networking.tf
rename to fast/stages/1-resman/branch-networking.tf
index 530cf6b0..1fd7a6b3 100644
--- a/fast/stages/01-resman/branch-networking.tf
+++ b/fast/stages/1-resman/branch-networking.tf
@@ -20,7 +20,7 @@ module "branch-network-folder" {
source = "../../../modules/folder"
parent = "organizations/${var.organization.id}"
name = "Networking"
- group_iam = {
+ group_iam = local.groups.gcp-network-admins == null ? {} : {
(local.groups.gcp-network-admins) = [
# add any needed roles for resources/services not managed via Terraform,
# or replace editor with ~viewer if no broad resource management needed
diff --git a/fast/stages/01-resman/branch-project-factory.tf b/fast/stages/1-resman/branch-project-factory.tf
similarity index 78%
rename from fast/stages/01-resman/branch-project-factory.tf
rename to fast/stages/1-resman/branch-project-factory.tf
index 41651a28..d74a8acb 100644
--- a/fast/stages/01-resman/branch-project-factory.tf
+++ b/fast/stages/1-resman/branch-project-factory.tf
@@ -79,3 +79,22 @@ module "branch-pf-prod-gcs" {
"roles/storage.objectAdmin" = [module.branch-pf-prod-sa.0.iam_email]
}
}
+
+resource "google_organization_iam_member" "org_policy_admin_pf" {
+ for_each = !var.fast_features.project_factory ? {} : {
+ pf-dev = ["teams", "development", module.branch-pf-dev-sa.0.iam_email]
+ pf-prod = ["teams", "production", module.branch-pf-prod-sa.0.iam_email]
+ }
+ org_id = var.organization.id
+ role = "roles/orgpolicy.policyAdmin"
+ member = each.value.2
+ condition {
+ title = "org_policy_tag_pf_scoped"
+ description = "Org policy tag scoped grant for ${each.value.0}/${each.value.1}."
+ expression = <<-END
+ resource.matchTag('${var.organization.id}/${var.tag_names.context}', '${each.value.0}')
+ &&
+ resource.matchTag('${var.organization.id}/${var.tag_names.environment}', '${each.value.1}')
+ END
+ }
+}
diff --git a/fast/stages/01-resman/branch-sandbox.tf b/fast/stages/1-resman/branch-sandbox.tf
similarity index 100%
rename from fast/stages/01-resman/branch-sandbox.tf
rename to fast/stages/1-resman/branch-sandbox.tf
diff --git a/fast/stages/01-resman/branch-security.tf b/fast/stages/1-resman/branch-security.tf
similarity index 96%
rename from fast/stages/01-resman/branch-security.tf
rename to fast/stages/1-resman/branch-security.tf
index c7b4fc97..4b0c0fb1 100644
--- a/fast/stages/01-resman/branch-security.tf
+++ b/fast/stages/1-resman/branch-security.tf
@@ -20,7 +20,7 @@ module "branch-security-folder" {
source = "../../../modules/folder"
parent = "organizations/${var.organization.id}"
name = "Security"
- group_iam = {
+ group_iam = local.groups.gcp-security-admins == null ? {} : {
(local.groups.gcp-security-admins) = [
# add any needed roles for resources/services not managed via Terraform,
# e.g.
@@ -51,7 +51,7 @@ module "branch-security-folder" {
module "branch-security-sa" {
source = "../../../modules/iam-service-account"
project_id = var.automation.project_id
- name = "prod-resman-sec-0"
+ name = "security-0"
display_name = "Terraform resman security service account."
prefix = var.prefix
iam = {
diff --git a/fast/stages/01-resman/branch-teams.tf b/fast/stages/1-resman/branch-teams.tf
similarity index 100%
rename from fast/stages/01-resman/branch-teams.tf
rename to fast/stages/1-resman/branch-teams.tf
diff --git a/fast/stages/01-resman/cicd-data-platform.tf b/fast/stages/1-resman/cicd-data-platform.tf
similarity index 99%
rename from fast/stages/01-resman/cicd-data-platform.tf
rename to fast/stages/1-resman/cicd-data-platform.tf
index 5b07883c..e69fd5bc 100644
--- a/fast/stages/01-resman/cicd-data-platform.tf
+++ b/fast/stages/1-resman/cicd-data-platform.tf
@@ -103,7 +103,7 @@ module "branch-dp-dev-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
@@ -146,7 +146,7 @@ module "branch-dp-prod-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
diff --git a/fast/stages/01-resman/cicd-gke.tf b/fast/stages/1-resman/cicd-gke.tf
similarity index 99%
rename from fast/stages/01-resman/cicd-gke.tf
rename to fast/stages/1-resman/cicd-gke.tf
index fa4f8767..4388a3ac 100644
--- a/fast/stages/01-resman/cicd-gke.tf
+++ b/fast/stages/1-resman/cicd-gke.tf
@@ -103,7 +103,7 @@ module "branch-gke-dev-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
@@ -146,7 +146,7 @@ module "branch-gke-prod-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
diff --git a/fast/stages/01-resman/cicd-networking.tf b/fast/stages/1-resman/cicd-networking.tf
similarity index 99%
rename from fast/stages/01-resman/cicd-networking.tf
rename to fast/stages/1-resman/cicd-networking.tf
index 894348ff..245d5ed0 100644
--- a/fast/stages/01-resman/cicd-networking.tf
+++ b/fast/stages/1-resman/cicd-networking.tf
@@ -65,7 +65,7 @@ module "branch-network-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
diff --git a/fast/stages/01-resman/cicd-project-factory.tf b/fast/stages/1-resman/cicd-project-factory.tf
similarity index 99%
rename from fast/stages/01-resman/cicd-project-factory.tf
rename to fast/stages/1-resman/cicd-project-factory.tf
index 8f357ce6..1e2b4565 100644
--- a/fast/stages/01-resman/cicd-project-factory.tf
+++ b/fast/stages/1-resman/cicd-project-factory.tf
@@ -114,7 +114,7 @@ module "branch-pf-dev-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
@@ -162,7 +162,7 @@ module "branch-pf-prod-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
diff --git a/fast/stages/01-resman/cicd-security.tf b/fast/stages/1-resman/cicd-security.tf
similarity index 99%
rename from fast/stages/01-resman/cicd-security.tf
rename to fast/stages/1-resman/cicd-security.tf
index dd27a473..c35bfbfb 100644
--- a/fast/stages/01-resman/cicd-security.tf
+++ b/fast/stages/1-resman/cicd-security.tf
@@ -65,7 +65,7 @@ module "branch-security-sa-cicd" {
each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos
? {
- "roles/iam.serviceAccountUser" = local.automation_resman_sa
+ "roles/iam.serviceAccountUser" = local.automation_resman_sa_iam
}
# impersonated via workload identity federation for external repos
: {
diff --git a/fast/stages/1-resman/data/org-policies/compute.yaml b/fast/stages/1-resman/data/org-policies/compute.yaml
new file mode 100644
index 00000000..0d27ac42
--- /dev/null
+++ b/fast/stages/1-resman/data/org-policies/compute.yaml
@@ -0,0 +1,73 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+compute.disableGuestAttributesAccess:
+ enforce: true
+
+compute.requireOsLogin:
+ enforce: true
+
+compute.restrictLoadBalancerCreationForTypes:
+ allow:
+ values:
+ - in:INTERNAL
+
+compute.skipDefaultNetworkCreation:
+ enforce: true
+
+compute.vmExternalIpAccess:
+ deny:
+ all: true
+
+
+# compute.disableInternetNetworkEndpointGroup:
+# enforce: true
+
+# compute.disableNestedVirtualization:
+# enforce: true
+
+# compute.disableSerialPortAccess:
+# enforce: true
+
+# compute.restrictCloudNATUsage:
+# deny:
+# all: true
+
+# compute.restrictDedicatedInterconnectUsage:
+# deny:
+# all: true
+
+# compute.restrictPartnerInterconnectUsage:
+# deny:
+# all: true
+
+# compute.restrictProtocolForwardingCreationForTypes:
+# deny:
+# all: true
+
+# compute.restrictSharedVpcHostProjects:
+# deny:
+# all: true
+
+# compute.restrictSharedVpcSubnetworks:
+# deny:
+# all: true
+
+# compute.restrictVpcPeering:
+# deny:
+# all: true
+
+# compute.restrictVpnPeerIPs:
+# deny:
+# all: true
+
+# compute.restrictXpnProjectLienRemoval:
+# enforce: true
+
+# compute.setNewProjectDefaultToZonalDNSOnly:
+# enforce: true
+
+# compute.vmCanIpForward:
+# deny:
+# all: true
diff --git a/fast/stages/1-resman/data/org-policies/iam.yaml b/fast/stages/1-resman/data/org-policies/iam.yaml
new file mode 100644
index 00000000..4d83f827
--- /dev/null
+++ b/fast/stages/1-resman/data/org-policies/iam.yaml
@@ -0,0 +1,12 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+iam.automaticIamGrantsForDefaultServiceAccounts:
+ enforce: true
+
+iam.disableServiceAccountKeyCreation:
+ enforce: true
+
+iam.disableServiceAccountKeyUpload:
+ enforce: true
diff --git a/fast/stages/1-resman/data/org-policies/serverless.yaml b/fast/stages/1-resman/data/org-policies/serverless.yaml
new file mode 100644
index 00000000..de62e6c7
--- /dev/null
+++ b/fast/stages/1-resman/data/org-policies/serverless.yaml
@@ -0,0 +1,26 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+run.allowedIngress:
+ allow:
+ values:
+ - is:internal
+
+# run.allowedVPCEgress:
+# allow:
+# values:
+# - is:private-ranges-only
+
+# cloudfunctions.allowedIngressSettings:
+# allow:
+# values:
+# - is:ALLOW_INTERNAL_ONLY
+
+# cloudfunctions.allowedVpcConnectorEgressSettings:
+# allow:
+# values:
+# - is:PRIVATE_RANGES_ONLY
+
+# cloudfunctions.requireVPCConnector:
+# enforce: true
diff --git a/fast/stages/1-resman/data/org-policies/sql.yaml b/fast/stages/1-resman/data/org-policies/sql.yaml
new file mode 100644
index 00000000..88b84d9d
--- /dev/null
+++ b/fast/stages/1-resman/data/org-policies/sql.yaml
@@ -0,0 +1,9 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+sql.restrictAuthorizedNetworks:
+ enforce: true
+
+sql.restrictPublicIp:
+ enforce: true
diff --git a/fast/stages/1-resman/data/org-policies/storage.yaml b/fast/stages/1-resman/data/org-policies/storage.yaml
new file mode 100644
index 00000000..6c0a673f
--- /dev/null
+++ b/fast/stages/1-resman/data/org-policies/storage.yaml
@@ -0,0 +1,6 @@
+# skip boilerplate check
+#
+# sample subset of useful organization policies, edit to suit requirements
+
+storage.uniformBucketLevelAccess:
+ enforce: true
diff --git a/fast/stages/1-resman/diagram.png b/fast/stages/1-resman/diagram.png
new file mode 100644
index 00000000..d1026318
Binary files /dev/null and b/fast/stages/1-resman/diagram.png differ
diff --git a/fast/stages/1-resman/diagram.svg b/fast/stages/1-resman/diagram.svg
new file mode 100644
index 00000000..541db3f4
--- /dev/null
+++ b/fast/stages/1-resman/diagram.svg
@@ -0,0 +1,1340 @@
+
+
diff --git a/fast/stages/01-resman/main.tf b/fast/stages/1-resman/main.tf
similarity index 78%
rename from fast/stages/01-resman/main.tf
rename to fast/stages/1-resman/main.tf
index 0651ee3f..ff08b8c5 100644
--- a/fast/stages/01-resman/main.tf
+++ b/fast/stages/1-resman/main.tf
@@ -17,15 +17,13 @@
locals {
# convenience flags that express where billing account resides
automation_resman_sa = try(
- [format(
- "serviceAccount:%s",
- data.google_client_openid_userinfo.provider_identity.0.email
- )],
- []
+ data.google_client_openid_userinfo.provider_identity.0.email, null
+ )
+ automation_resman_sa_iam = (
+ local.automation_resman_sa == null
+ ? []
+ : ["serviceAccount:${local.automation_resman_sa}"]
)
- billing_ext = var.billing_account.organization_id == null
- billing_org = var.billing_account.organization_id == var.organization.id
- billing_org_ext = !local.billing_ext && !local.billing_org
branch_optional_sa_lists = {
dp-dev = compact([try(module.branch-dp-dev-sa.0.iam_email, "")])
dp-prod = compact([try(module.branch-dp-prod-sa.0.iam_email, "")])
@@ -51,16 +49,16 @@ locals {
}
cicd_workflow_var_files = {
stage_2 = [
- "00-bootstrap.auto.tfvars.json",
- "01-resman.auto.tfvars.json",
+ "0-bootstrap.auto.tfvars.json",
+ "1-resman.auto.tfvars.json",
"globals.auto.tfvars.json"
]
stage_3 = [
- "00-bootstrap.auto.tfvars.json",
- "01-resman.auto.tfvars.json",
+ "0-bootstrap.auto.tfvars.json",
+ "1-resman.auto.tfvars.json",
"globals.auto.tfvars.json",
- "02-networking.auto.tfvars.json",
- "02-security.auto.tfvars.json"
+ "2-networking.auto.tfvars.json",
+ "2-security.auto.tfvars.json"
]
}
custom_roles = coalesce(var.custom_roles, {})
@@ -74,8 +72,7 @@ locals {
k => "${v}@${var.organization.domain}"
}
groups_iam = {
- for k, v in local.groups :
- k => "group:${v}"
+ for k, v in local.groups : k => v != null ? "group:${v}" : null
}
identity_providers = coalesce(
try(var.automation.federated_identity_providers, null), {}
diff --git a/fast/stages/01-resman/organization.tf b/fast/stages/1-resman/organization.tf
similarity index 66%
rename from fast/stages/01-resman/organization.tf
rename to fast/stages/1-resman/organization.tf
index 7ecf7952..3d7db46d 100644
--- a/fast/stages/01-resman/organization.tf
+++ b/fast/stages/1-resman/organization.tf
@@ -16,7 +16,6 @@
# tfdoc:file:description Organization policies.
-
locals {
all_drs_domains = concat(
[var.organization.customer_id],
@@ -47,7 +46,7 @@ module "organization" {
module.branch-network-sa.iam_email
]
},
- local.billing_org ? {
+ var.billing_account.is_org_level ? {
"roles/billing.costsManager" = concat(
local.branch_optional_sa_lists.pf-dev,
local.branch_optional_sa_lists.pf-prod
@@ -85,6 +84,9 @@ module "organization" {
}
org_policies_data_path = "${var.data_dir}/org-policies"
+ # do not assign tagViewer or tagUser roles here on tag keys and values as
+ # they are managed authoritatively and will break multitenant stages
+
tags = {
(var.tag_names.context) = {
description = "Resource management context."
@@ -106,45 +108,10 @@ module "organization" {
production = null
}
}
+ (var.tag_names.tenant) = {
+ description = "Organization tenant."
+ }
}
}
-# organization policy admin role assigned with a condition on tags
-
-resource "google_organization_iam_member" "org_policy_admin_dp" {
- for_each = !var.fast_features.data_platform ? {} : {
- data-dev = ["data", "development", module.branch-dp-dev-sa.0.iam_email]
- data-prod = ["data", "production", module.branch-dp-prod-sa.0.iam_email]
- }
- org_id = var.organization.id
- role = "roles/orgpolicy.policyAdmin"
- member = each.value.2
- condition {
- title = "org_policy_tag_dp_scoped"
- description = "Org policy tag scoped grant for ${each.value.0}/${each.value.1}."
- expression = <<-END
- resource.matchTag('${var.organization.id}/${var.tag_names.context}', '${each.value.0}')
- &&
- resource.matchTag('${var.organization.id}/${var.tag_names.environment}', '${each.value.1}')
- END
- }
-}
-
-resource "google_organization_iam_member" "org_policy_admin_pf" {
- for_each = !var.fast_features.project_factory ? {} : {
- pf-dev = ["teams", "development", module.branch-pf-dev-sa.0.iam_email]
- pf-prod = ["teams", "production", module.branch-pf-prod-sa.0.iam_email]
- }
- org_id = var.organization.id
- role = "roles/orgpolicy.policyAdmin"
- member = each.value.2
- condition {
- title = "org_policy_tag_pf_scoped"
- description = "Org policy tag scoped grant for ${each.value.0}/${each.value.1}."
- expression = <<-END
- resource.matchTag('${var.organization.id}/${var.tag_names.context}', '${each.value.0}')
- &&
- resource.matchTag('${var.organization.id}/${var.tag_names.environment}', '${each.value.1}')
- END
- }
-}
+# organization policy conditional roles are in the relevant branch files
diff --git a/fast/stages/01-resman/outputs-files.tf b/fast/stages/1-resman/outputs-files.tf
similarity index 94%
rename from fast/stages/01-resman/outputs-files.tf
rename to fast/stages/1-resman/outputs-files.tf
index bd281d45..f7f080dd 100644
--- a/fast/stages/01-resman/outputs-files.tf
+++ b/fast/stages/1-resman/outputs-files.tf
@@ -30,7 +30,7 @@ resource "local_file" "providers" {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${local.outputs_location}/tfvars/01-resman.auto.tfvars.json"
+ filename = "${local.outputs_location}/tfvars/1-resman.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/01-resman/outputs-gcs.tf b/fast/stages/1-resman/outputs-gcs.tf
similarity index 96%
rename from fast/stages/01-resman/outputs-gcs.tf
rename to fast/stages/1-resman/outputs-gcs.tf
index f1db11ef..5b9f5d85 100644
--- a/fast/stages/01-resman/outputs-gcs.tf
+++ b/fast/stages/1-resman/outputs-gcs.tf
@@ -25,7 +25,7 @@ resource "google_storage_bucket_object" "providers" {
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/01-resman.auto.tfvars.json"
+ name = "tfvars/1-resman.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/1-resman/outputs.tf
similarity index 72%
rename from fast/stages/01-resman/outputs.tf
rename to fast/stages/1-resman/outputs.tf
index e07df577..80ec2b9d 100644
--- a/fast/stages/01-resman/outputs.tf
+++ b/fast/stages/1-resman/outputs.tf
@@ -19,42 +19,42 @@ locals {
cicd_workflow_attrs = {
data_platform_dev = {
service_account = try(module.branch-dp-dev-sa-cicd.0.email, null)
- tf_providers_file = "03-data-platform-dev-providers.tf"
+ tf_providers_file = "3-data-platform-dev-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_3
}
data_platform_prod = {
service_account = try(module.branch-dp-prod-sa-cicd.0.email, null)
- tf_providers_file = "03-data-platform-prod-providers.tf"
+ tf_providers_file = "3-data-platform-prod-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_3
}
gke_dev = {
service_account = try(module.branch-gke-dev-sa-cicd.0.email, null)
- tf_providers_file = "03-gke-dev-providers.tf"
+ tf_providers_file = "3-gke-dev-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_3
}
gke_prod = {
service_account = try(module.branch-gke-prod-sa-cicd.0.email, null)
- tf_providers_file = "03-gke-prod-providers.tf"
+ tf_providers_file = "3-gke-prod-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_3
}
networking = {
service_account = try(module.branch-network-sa-cicd.0.email, null)
- tf_providers_file = "02-networking-providers.tf"
+ tf_providers_file = "2-networking-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_2
}
project_factory_dev = {
service_account = try(module.branch-pf-dev-sa-cicd.0.email, null)
- tf_providers_file = "03-project-factory-dev-providers.tf"
+ tf_providers_file = "3-project-factory-dev-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_3
}
project_factory_prod = {
service_account = try(module.branch-pf-prod-sa-cicd.0.email, null)
- tf_providers_file = "03-project-factory-prod-providers.tf"
+ tf_providers_file = "3-project-factory-prod-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_3
}
security = {
service_account = try(module.branch-security-sa-cicd.0.email, null)
- tf_providers_file = "02-security-providers.tf"
+ tf_providers_file = "2-security-providers.tf"
tf_var_files = local.cicd_workflow_var_files.stage_2
}
}
@@ -98,74 +98,85 @@ locals {
)
providers = merge(
{
- "02-networking" = templatefile(local._tpl_providers, {
- bucket = module.branch-network-gcs.name
- name = "networking"
- sa = module.branch-network-sa.email
+ "2-networking" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-network-gcs.name
+ name = "networking"
+ sa = module.branch-network-sa.email
})
- "02-security" = templatefile(local._tpl_providers, {
- bucket = module.branch-security-gcs.name
- name = "security"
- sa = module.branch-security-sa.email
+ "2-security" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-security-gcs.name
+ name = "security"
+ sa = module.branch-security-sa.email
})
},
!var.fast_features.data_platform ? {} : {
- "03-data-platform-dev" = templatefile(local._tpl_providers, {
- bucket = module.branch-dp-dev-gcs.0.name
- name = "dp-dev"
- sa = module.branch-dp-dev-sa.0.email
+ "3-data-platform-dev" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-dp-dev-gcs.0.name
+ name = "dp-dev"
+ sa = module.branch-dp-dev-sa.0.email
})
- "03-data-platform-prod" = templatefile(local._tpl_providers, {
- bucket = module.branch-dp-prod-gcs.0.name
- name = "dp-prod"
- sa = module.branch-dp-prod-sa.0.email
+ "3-data-platform-prod" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-dp-prod-gcs.0.name
+ name = "dp-prod"
+ sa = module.branch-dp-prod-sa.0.email
})
},
!var.fast_features.gke ? {} : {
- "03-gke-dev" = templatefile(local._tpl_providers, {
- bucket = module.branch-gke-dev-gcs.0.name
- name = "gke-dev"
- sa = module.branch-gke-dev-sa.0.email
+ "3-gke-dev" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-gke-dev-gcs.0.name
+ name = "gke-dev"
+ sa = module.branch-gke-dev-sa.0.email
})
- "03-gke-prod" = templatefile(local._tpl_providers, {
- bucket = module.branch-gke-prod-gcs.0.name
- name = "gke-prod"
- sa = module.branch-gke-prod-sa.0.email
+ "3-gke-prod" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-gke-prod-gcs.0.name
+ name = "gke-prod"
+ sa = module.branch-gke-prod-sa.0.email
})
},
!var.fast_features.project_factory ? {} : {
- "03-project-factory-dev" = templatefile(local._tpl_providers, {
- bucket = module.branch-pf-dev-gcs.0.name
- name = "team-dev"
- sa = module.branch-pf-dev-sa.0.email
+ "3-project-factory-dev" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-pf-dev-gcs.0.name
+ name = "team-dev"
+ sa = module.branch-pf-dev-sa.0.email
})
- "03-project-factory-prod" = templatefile(local._tpl_providers, {
- bucket = module.branch-pf-prod-gcs.0.name
- name = "team-prod"
- sa = module.branch-pf-prod-sa.0.email
+ "3-project-factory-prod" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-pf-prod-gcs.0.name
+ name = "team-prod"
+ sa = module.branch-pf-prod-sa.0.email
})
},
!var.fast_features.sandbox ? {} : {
- "99-sandbox" = templatefile(local._tpl_providers, {
- bucket = module.branch-sandbox-gcs.0.name
- name = "sandbox"
- sa = module.branch-sandbox-sa.0.email
+ "9-sandbox" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-sandbox-gcs.0.name
+ name = "sandbox"
+ sa = module.branch-sandbox-sa.0.email
})
},
!var.fast_features.teams ? {} : merge(
{
- "03-teams" = templatefile(local._tpl_providers, {
- bucket = module.branch-teams-gcs.0.name
- name = "teams"
- sa = module.branch-teams-sa.0.email
+ "3-teams" = templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-teams-gcs.0.name
+ name = "teams"
+ sa = module.branch-teams-sa.0.email
})
},
{
for k, v in module.branch-teams-team-sa :
- "03-teams-${k}" => templatefile(local._tpl_providers, {
- bucket = module.branch-teams-team-gcs[k].name
- name = "teams"
- sa = v.email
+ "3-teams-${k}" => templatefile(local._tpl_providers, {
+ backend_extra = null
+ bucket = module.branch-teams-team-gcs[k].name
+ name = "teams"
+ sa = v.email
})
}
)
@@ -190,7 +201,9 @@ locals {
tfvars = {
folder_ids = local.folder_ids
service_accounts = local.service_accounts
+ tag_keys = { for k, v in module.organization.tag_keys : k => v.id }
tag_names = var.tag_names
+ tag_values = { for k, v in module.organization.tag_values : k => v.id }
}
}
diff --git a/tests/blueprints/factories/bigquery_factory/fixture/variables.tf b/fast/stages/1-resman/templates/providers.tf.tpl
similarity index 61%
rename from tests/blueprints/factories/bigquery_factory/fixture/variables.tf
rename to fast/stages/1-resman/templates/providers.tf.tpl
index 8269dbbe..d1c224c5 100644
--- a/tests/blueprints/factories/bigquery_factory/fixture/variables.tf
+++ b/fast/stages/1-resman/templates/providers.tf.tpl
@@ -14,21 +14,20 @@
* limitations under the License.
*/
-variable "views_dir" {
- description = "Relative path for the folder storing view data."
- type = string
- default = "/views"
+terraform {
+ backend "gcs" {
+ bucket = "${bucket}"
+ impersonate_service_account = "${sa}"
+ %{~ if backend_extra != null ~}
+ ${indent(4, backend_extra)}
+ %{~ endif ~}
+ }
+}
+provider "google" {
+ impersonate_service_account = "${sa}"
+}
+provider "google-beta" {
+ impersonate_service_account = "${sa}"
}
-variable "tables_dir" {
- description = "Relative path for the folder storing table data."
- type = string
- default = "tables"
-}
-
-variable "project_id" {
- description = "Project ID"
- type = string
- default = "test-project"
-
-}
+# end provider.tf for ${name}
diff --git a/fast/stages/1-resman/templates/workflow-github.yaml b/fast/stages/1-resman/templates/workflow-github.yaml
new file mode 100644
index 00000000..41c51411
--- /dev/null
+++ b/fast/stages/1-resman/templates/workflow-github.yaml
@@ -0,0 +1,198 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "FAST ${stage_name} stage"
+
+on:
+ pull_request:
+ branches:
+ - main
+ types:
+ - closed
+ - opened
+ - synchronize
+
+env:
+ FAST_OUTPUTS_BUCKET: ${outputs_bucket}
+ FAST_SERVICE_ACCOUNT: ${service_account}
+ FAST_WIF_PROVIDER: ${identity_provider}
+ SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+ TF_PROVIDERS_FILE: ${tf_providers_file}
+ TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)}
+ TF_VERSION: 1.3.2
+
+jobs:
+ fast-pr:
+ permissions:
+ contents: read
+ id-token: write
+ issues: write
+ pull-requests: write
+ runs-on: ubuntu-latest
+ steps:
+ - id: checkout
+ name: Checkout repository
+ uses: actions/checkout@v3
+
+ # set up SSH key authentication to the modules repository
+ - id: ssh-config
+ name: Configure SSH authentication
+ run: |
+ ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
+ ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}"
+
+ # set up authentication via Workload identity Federation
+ - id: gcp-auth
+ name: Authenticate to Google Cloud
+ uses: google-github-actions/auth@v0
+ with:
+ workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }}
+ service_account: $${{ env.FAST_SERVICE_ACCOUNT }}
+ access_token_lifetime: 3600s
+
+ - id: gcp-sdk
+ name: Set up Cloud SDK
+ uses: google-github-actions/setup-gcloud@v0
+ with:
+ install_components: alpha
+
+ # copy provider and tfvars files
+ - id: tf-config
+ name: Copy Terraform output files
+ run: |
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./
+ gcloud alpha storage cp -r \
+ "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./
+ for f in $${{env.TF_VAR_FILES}}; do
+ ln -s "tfvars/$f" ./
+ done
+
+ - id: tf-setup
+ name: Set up Terraform
+ uses: hashicorp/setup-terraform@v2.0.3
+ with:
+ terraform_version: $${{ env.TF_VERSION }}
+
+ # run Terraform init/validate/plan
+ - id: tf-init
+ name: Terraform init
+ continue-on-error: true
+ run: |
+ terraform init -no-color
+
+ - id: tf-validate
+ continue-on-error: true
+ name: Terraform validate
+ run: terraform validate -no-color
+
+ - id: tf-plan
+ name: Terraform plan
+ continue-on-error: true
+ run: |
+ terraform plan -input=false -out ../plan.out -no-color
+
+ - id: tf-apply
+ if: github.event.pull_request.merged == true && success()
+ name: Terraform apply
+ continue-on-error: true
+ run: |
+ terraform apply -input=false -auto-approve -no-color ../plan.out
+
+ - id: pr-comment
+ name: Post comment to Pull Request
+ continue-on-error: true
+ uses: actions/github-script@v6
+ if: github.event_name == 'pull_request'
+ env:
+ PLAN: $${{ steps.tf-plan.outputs.stdout }}\n$${{ steps.tf-plan.outputs.stderr }}
+ with:
+ script: |
+ const output = `### Terraform Initialization \`$${{ steps.tf-init.outcome }}\`
+
+ ### Terraform Validation \`$${{ steps.tf-validate.outcome }}\`
+
+ google_monitoring_dashboard |
| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object · local_file |
| [peerings.tf](./peerings.tf) | None | net-vpc-peering | |
+| [regions.tf](./regions.tf) | Compute short names for regions. | | |
| [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-cloudnat · net-vpc · net-vpc-firewall · project | google_project_iam_binding |
| [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-cloudnat · net-vpc · net-vpc-firewall · project | google_project_iam_binding |
| [test-resources.tf](./test-resources.tf) | temporary instances for testing | compute-vm | |
@@ -295,23 +296,23 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman |
-| [organization](variables.tf#L102) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
-| [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap |
-| [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | string | | "data" | |
-| [dns](variables.tf#L66) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
-| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
-| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L92) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman |
+| [organization](variables.tf#L126) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L142) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
+| [custom_roles](variables.tf#L55) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [dns](variables.tf#L64) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
+| [factories_config](variables.tf#L72) | Configuration for network resource factories. | object({…}) | | {…} | |
+| [l7ilb_subnets](variables.tf#L102) | Subnets used for L7 ILBs. | object({…}) | | {…} | |
+| [outputs_location](variables.tf#L136) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
| [peering_configs](variables-peerings.tf#L19) | Peering configurations. | map(object({…})) | | {…} | |
-| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
-| [region_trigram](variables.tf#L166) | Short names for GCP regions. | map(string) | | {…} | |
-| [router_onprem_configs](variables.tf#L175) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | |
-| [service_accounts](variables.tf#L193) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman |
-| [vpn_onprem_configs](variables.tf#L207) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
+| [psa_ranges](variables.tf#L153) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
+| [regions](variables.tf#L190) | Region definitions. | object({…}) | | {…} | |
+| [router_onprem_configs](variables.tf#L202) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | |
+| [service_accounts](variables.tf#L220) | Automation service accounts in name => email format. | object({…}) | | null | 1-resman |
+| [vpn_onprem_configs](variables.tf#L234) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
## Outputs
diff --git a/fast/stages/02-networking-nva/data/cidrs.yaml b/fast/stages/2-networking-a-peering/data/cidrs.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/cidrs.yaml
rename to fast/stages/2-networking-a-peering/data/cidrs.yaml
diff --git a/fast/stages/02-networking-peering/data/dashboards/firewall_insights.json b/fast/stages/2-networking-a-peering/data/dashboards/firewall_insights.json
similarity index 100%
rename from fast/stages/02-networking-peering/data/dashboards/firewall_insights.json
rename to fast/stages/2-networking-a-peering/data/dashboards/firewall_insights.json
diff --git a/fast/stages/02-networking-peering/data/dashboards/vpn.json b/fast/stages/2-networking-a-peering/data/dashboards/vpn.json
similarity index 100%
rename from fast/stages/02-networking-peering/data/dashboards/vpn.json
rename to fast/stages/2-networking-a-peering/data/dashboards/vpn.json
diff --git a/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml b/fast/stages/2-networking-a-peering/data/firewall-rules/dev/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml
rename to fast/stages/2-networking-a-peering/data/firewall-rules/dev/rules.yaml
diff --git a/fast/stages/02-networking-peering/data/firewall-rules/landing/rules.yaml b/fast/stages/2-networking-a-peering/data/firewall-rules/landing/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/firewall-rules/landing/rules.yaml
rename to fast/stages/2-networking-a-peering/data/firewall-rules/landing/rules.yaml
diff --git a/fast/stages/02-networking-nva/data/hierarchical-policy-rules.yaml b/fast/stages/2-networking-a-peering/data/hierarchical-policy-rules.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/hierarchical-policy-rules.yaml
rename to fast/stages/2-networking-a-peering/data/hierarchical-policy-rules.yaml
diff --git a/fast/stages/02-networking-peering/data/subnets/dev/dev-dataplatform-ew1.yaml b/fast/stages/2-networking-a-peering/data/subnets/dev/dev-dataplatform-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/subnets/dev/dev-dataplatform-ew1.yaml
rename to fast/stages/2-networking-a-peering/data/subnets/dev/dev-dataplatform-ew1.yaml
diff --git a/fast/stages/02-networking-peering/data/subnets/dev/dev-default-ew1.yaml b/fast/stages/2-networking-a-peering/data/subnets/dev/dev-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/subnets/dev/dev-default-ew1.yaml
rename to fast/stages/2-networking-a-peering/data/subnets/dev/dev-default-ew1.yaml
diff --git a/fast/stages/02-networking-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml b/fast/stages/2-networking-a-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml
rename to fast/stages/2-networking-a-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml
diff --git a/fast/stages/02-networking-peering/data/subnets/landing/landing-default-ew1.yaml b/fast/stages/2-networking-a-peering/data/subnets/landing/landing-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/subnets/landing/landing-default-ew1.yaml
rename to fast/stages/2-networking-a-peering/data/subnets/landing/landing-default-ew1.yaml
diff --git a/fast/stages/02-networking-peering/data/subnets/prod/prod-default-ew1.yaml b/fast/stages/2-networking-a-peering/data/subnets/prod/prod-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/subnets/prod/prod-default-ew1.yaml
rename to fast/stages/2-networking-a-peering/data/subnets/prod/prod-default-ew1.yaml
diff --git a/fast/stages/02-networking-peering/diagram.png b/fast/stages/2-networking-a-peering/diagram.png
similarity index 100%
rename from fast/stages/02-networking-peering/diagram.png
rename to fast/stages/2-networking-a-peering/diagram.png
diff --git a/fast/stages/02-networking-peering/diagram.svg b/fast/stages/2-networking-a-peering/diagram.svg
similarity index 100%
rename from fast/stages/02-networking-peering/diagram.svg
rename to fast/stages/2-networking-a-peering/diagram.svg
diff --git a/fast/stages/02-networking-peering/dns-dev.tf b/fast/stages/2-networking-a-peering/dns-dev.tf
similarity index 100%
rename from fast/stages/02-networking-peering/dns-dev.tf
rename to fast/stages/2-networking-a-peering/dns-dev.tf
diff --git a/fast/stages/02-networking-peering/dns-landing.tf b/fast/stages/2-networking-a-peering/dns-landing.tf
similarity index 100%
rename from fast/stages/02-networking-peering/dns-landing.tf
rename to fast/stages/2-networking-a-peering/dns-landing.tf
diff --git a/fast/stages/02-networking-peering/dns-prod.tf b/fast/stages/2-networking-a-peering/dns-prod.tf
similarity index 100%
rename from fast/stages/02-networking-peering/dns-prod.tf
rename to fast/stages/2-networking-a-peering/dns-prod.tf
diff --git a/fast/stages/02-networking-peering/landing.tf b/fast/stages/2-networking-a-peering/landing.tf
similarity index 82%
rename from fast/stages/02-networking-peering/landing.tf
rename to fast/stages/2-networking-a-peering/landing.tf
index 83a0d509..37e3adfd 100644
--- a/fast/stages/02-networking-peering/landing.tf
+++ b/fast/stages/2-networking-a-peering/landing.tf
@@ -63,7 +63,7 @@ module "landing-vpc" {
next_hop = "default-internet-gateway"
}
}
- data_folder = "${var.data_dir}/subnets/landing"
+ data_folder = "${var.factories_config.data_dir}/subnets/landing"
}
module "landing-firewall" {
@@ -74,18 +74,23 @@ module "landing-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/landing"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/landing"
}
}
-module "landing-nat-ew1" {
+moved {
+ from = module.landing-nat-ew1
+ to = module.landing-nat-primary
+}
+
+module "landing-nat-primary" {
source = "../../../modules/net-cloudnat"
project_id = module.landing-project.project_id
- region = "europe-west1"
- name = "ew1"
+ region = var.regions.primary
+ name = local.region_shortnames[var.regions.primary]
router_create = true
- router_name = "prod-nat-ew1"
+ router_name = "prod-nat-${local.region_shortnames[var.regions.primary]}"
router_network = module.landing-vpc.name
router_asn = 4200001024
}
diff --git a/fast/stages/02-networking-vpn/main.tf b/fast/stages/2-networking-a-peering/main.tf
similarity index 76%
rename from fast/stages/02-networking-vpn/main.tf
rename to fast/stages/2-networking-a-peering/main.tf
index f68d39eb..5888752d 100644
--- a/fast/stages/02-networking-vpn/main.tf
+++ b/fast/stages/2-networking-a-peering/main.tf
@@ -17,14 +17,14 @@
# tfdoc:file:description Networking folder and hierarchical policy.
locals {
+ # combine all regions from variables and subnets
+ regions = distinct(concat(
+ values(var.regions),
+ values(module.dev-spoke-vpc.subnet_regions),
+ values(module.landing-vpc.subnet_regions),
+ values(module.prod-spoke-vpc.subnet_regions),
+ ))
custom_roles = coalesce(var.custom_roles, {})
- l7ilb_subnets = {
- for env, v in var.l7ilb_subnets : env => [
- for s in v : merge(s, {
- active = true
- name = "${env}-l7ilb-${s.region}"
- })]
- }
stage3_sas_delegated_grants = [
"roles/composer.sharedVpcAgent",
"roles/compute.networkUser",
@@ -46,9 +46,9 @@ module "folder" {
folder_create = var.folder_ids.networking == null
id = var.folder_ids.networking
firewall_policy_factory = {
- cidr_file = "${var.data_dir}/cidrs.yaml"
- policy_name = null
- rules_file = "${var.data_dir}/hierarchical-policy-rules.yaml"
+ cidr_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ policy_name = var.factories_config.firewall_policy_name
+ rules_file = "${var.factories_config.data_dir}/hierarchical-policy-rules.yaml"
}
firewall_policy_association = {
factory-policy = "factory"
diff --git a/fast/stages/02-networking-vpn/monitoring.tf b/fast/stages/2-networking-a-peering/monitoring.tf
similarity index 93%
rename from fast/stages/02-networking-vpn/monitoring.tf
rename to fast/stages/2-networking-a-peering/monitoring.tf
index 7b8b70c5..be3a47fa 100644
--- a/fast/stages/02-networking-vpn/monitoring.tf
+++ b/fast/stages/2-networking-a-peering/monitoring.tf
@@ -17,7 +17,7 @@
# tfdoc:file:description Network monitoring dashboards.
locals {
- dashboard_path = "${var.data_dir}/dashboards"
+ dashboard_path = "${var.factories_config.data_dir}/dashboards"
dashboard_files = fileset(local.dashboard_path, "*.json")
dashboards = {
for filename in local.dashboard_files :
diff --git a/fast/stages/02-networking-peering/outputs.tf b/fast/stages/2-networking-a-peering/outputs.tf
similarity index 93%
rename from fast/stages/02-networking-peering/outputs.tf
rename to fast/stages/2-networking-a-peering/outputs.tf
index 3b97b7f2..628c706b 100644
--- a/fast/stages/02-networking-peering/outputs.tf
+++ b/fast/stages/2-networking-a-peering/outputs.tf
@@ -48,13 +48,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/02-networking.auto.tfvars.json"
+ filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/02-networking.auto.tfvars.json"
+ name = "tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
@@ -89,8 +89,8 @@ output "tfvars" {
output "vpn_gateway_endpoints" {
description = "External IP Addresses for the GCP VPN gateways."
value = local.enable_onprem_vpn == false ? null : {
- onprem-ew1 = {
- for v in module.landing-to-onprem-ew1-vpn[0].gateway.vpn_interfaces :
+ onprem-primary = {
+ for v in module.landing-to-onprem-primary-vpn[0].gateway.vpn_interfaces :
v.id => v.ip_address
}
}
diff --git a/fast/stages/02-networking-peering/peerings.tf b/fast/stages/2-networking-a-peering/peerings.tf
similarity index 100%
rename from fast/stages/02-networking-peering/peerings.tf
rename to fast/stages/2-networking-a-peering/peerings.tf
diff --git a/fast/stages/2-networking-a-peering/regions.tf b/fast/stages/2-networking-a-peering/regions.tf
new file mode 100644
index 00000000..53514afa
--- /dev/null
+++ b/fast/stages/2-networking-a-peering/regions.tf
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Compute short names for regions.
+
+locals {
+ # only map when the first character would not work
+ _region_cardinal = {
+ southeast = "se"
+ }
+ # only map when the first character would not work
+ _region_geo = {
+ australia = "o"
+ }
+ # split in [geo, cardinal, number] tokens
+ _region_tokens = {
+ for v in local.regions : v => regexall("(?:[a-z]+)|(?:[0-9]+)", v)
+ }
+ region_shortnames = {
+ for k, v in local._region_tokens : k => join("", [
+ # first token via geo alias map or first character
+ lookup(local._region_geo, v.0, substr(v.0, 0, 1)),
+ # first token via cardinal alias map or first character
+ lookup(local._region_cardinal, v.1, substr(v.1, 0, 1)),
+ # region number as is
+ v.2
+ ])
+ }
+}
diff --git a/fast/stages/02-networking-peering/spoke-dev.tf b/fast/stages/2-networking-a-peering/spoke-dev.tf
similarity index 84%
rename from fast/stages/02-networking-peering/spoke-dev.tf
rename to fast/stages/2-networking-a-peering/spoke-dev.tf
index e67cfb70..ed7d13cb 100644
--- a/fast/stages/02-networking-peering/spoke-dev.tf
+++ b/fast/stages/2-networking-a-peering/spoke-dev.tf
@@ -16,6 +16,19 @@
# tfdoc:file:description Dev spoke VPC and related resources.
+locals {
+ _l7ilb_subnets_dev = [
+ for v in var.l7ilb_subnets.dev : merge(v, {
+ active = true
+ region = lookup(var.regions, v.region, v.region)
+ })]
+ l7ilb_subnets_dev = [
+ for v in local._l7ilb_subnets_dev : merge(v, {
+ name = "dev-l7ilb-${local.region_shortnames[v.region]}"
+ })
+ ]
+}
+
module "dev-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account.id
@@ -48,9 +61,9 @@ module "dev-spoke-vpc" {
project_id = module.dev-spoke-project.project_id
name = "dev-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/dev"
+ data_folder = "${var.factories_config.data_dir}/subnets/dev"
psa_config = try(var.psa_ranges.dev, null)
- subnets_proxy_only = local.l7ilb_subnets.dev
+ subnets_proxy_only = local.l7ilb_subnets_dev
# set explicit routes for googleapis in case the default route is deleted
routes = {
private-googleapis = {
@@ -74,8 +87,8 @@ module "dev-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/dev"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/dev"
}
}
@@ -84,7 +97,7 @@ module "dev-spoke-cloudnat" {
source = "../../../modules/net-cloudnat"
project_id = module.dev-spoke-project.project_id
region = each.value
- name = "dev-nat-${var.region_trigram[each.value]}"
+ name = "dev-nat-${local.region_shortnames[each.value]}"
router_create = true
router_network = module.dev-spoke-vpc.name
router_asn = 4200001024
diff --git a/fast/stages/02-networking-vpn/spoke-prod.tf b/fast/stages/2-networking-a-peering/spoke-prod.tf
similarity index 84%
rename from fast/stages/02-networking-vpn/spoke-prod.tf
rename to fast/stages/2-networking-a-peering/spoke-prod.tf
index cf49152f..f584b32d 100644
--- a/fast/stages/02-networking-vpn/spoke-prod.tf
+++ b/fast/stages/2-networking-a-peering/spoke-prod.tf
@@ -16,6 +16,19 @@
# tfdoc:file:description Production spoke VPC and related resources.
+locals {
+ _l7ilb_subnets_prod = [
+ for v in var.l7ilb_subnets.prod : merge(v, {
+ active = true
+ region = lookup(var.regions, v.region, v.region)
+ })]
+ l7ilb_subnets_prod = [
+ for v in local._l7ilb_subnets_prod : merge(v, {
+ name = "prod-l7ilb-${local.region_shortnames[v.region]}"
+ })
+ ]
+}
+
module "prod-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account.id
@@ -48,9 +61,9 @@ module "prod-spoke-vpc" {
project_id = module.prod-spoke-project.project_id
name = "prod-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/prod"
+ data_folder = "${var.factories_config.data_dir}/subnets/prod"
psa_config = try(var.psa_ranges.prod, null)
- subnets_proxy_only = local.l7ilb_subnets.prod
+ subnets_proxy_only = local.l7ilb_subnets_prod
# set explicit routes for googleapis in case the default route is deleted
routes = {
private-googleapis = {
@@ -74,8 +87,8 @@ module "prod-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/prod"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/prod"
}
}
@@ -84,7 +97,7 @@ module "prod-spoke-cloudnat" {
source = "../../../modules/net-cloudnat"
project_id = module.prod-spoke-project.project_id
region = each.value
- name = "prod-nat-${var.region_trigram[each.value]}"
+ name = "prod-nat-${local.region_shortnames[each.value]}"
router_create = true
router_network = module.prod-spoke-vpc.name
router_asn = 4200001024
diff --git a/fast/stages/02-networking-peering/test-resources.tf b/fast/stages/2-networking-a-peering/test-resources.tf
similarity index 76%
rename from fast/stages/02-networking-peering/test-resources.tf
rename to fast/stages/2-networking-a-peering/test-resources.tf
index 204971fe..67073845 100644
--- a/fast/stages/02-networking-peering/test-resources.tf
+++ b/fast/stages/2-networking-a-peering/test-resources.tf
@@ -19,11 +19,11 @@
# module "test-vm-landing-0" {
# source = "../../../modules/compute-vm"
# project_id = module.landing-project.project_id
-# zone = "europe-west1-b"
+# zone = "${var.regions.primary}-b"
# name = "test-vm-0"
# network_interfaces = [{
# network = module.landing-vpc.self_link
-# subnetwork = module.landing-vpc.subnet_self_links["europe-west1/landing-default-ew1"]
+# subnetwork = module.landing-vpc.subnet_self_links["${var.regions.primary}/landing-default-${local.region_shortnames[var.regions.primary]}"]
# }]
# tags = ["ssh"]
# service_account_create = true
@@ -31,8 +31,8 @@
# image = "projects/debian-cloud/global/images/family/debian-10"
# }
# options = {
-# spot = true
-# termination_action = "STOP"
+# spot = true
+# termination_action = "STOP"
# }
# metadata = {
# startup-script = <folder | |
| [monitoring.tf](./monitoring.tf) | Network monitoring dashboards. | | google_monitoring_dashboard |
| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object · local_file |
+| [regions.tf](./regions.tf) | Compute short names for regions. | | |
| [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-cloudnat · net-vpc · net-vpc-firewall · project | google_project_iam_binding |
| [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-cloudnat · net-vpc · net-vpc-firewall · project | google_project_iam_binding |
| [test-resources.tf](./test-resources.tf) | temporary instances for testing | compute-vm | |
@@ -313,31 +314,31 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
| [variables.tf](./variables.tf) | Module variables. | | |
| [vpn-onprem.tf](./vpn-onprem.tf) | VPN between landing and onprem. | net-vpn-ha | |
| [vpn-spoke-dev.tf](./vpn-spoke-dev.tf) | VPN between landing and development spoke. | net-vpn-ha | |
-| [vpn-spoke-prod-ew1.tf](./vpn-spoke-prod-ew1.tf) | VPN between landing and production spoke in ew1. | net-vpn-ha | |
-| [vpn-spoke-prod-ew4.tf](./vpn-spoke-prod-ew4.tf) | VPN between landing and production spoke in ew4. | net-vpn-ha | |
+| [vpn-spoke-prod-primary.tf](./vpn-spoke-prod-primary.tf) | VPN between landing and production spoke in ew1. | net-vpn-ha | |
+| [vpn-spoke-prod-secondary.tf](./vpn-spoke-prod-secondary.tf) | VPN between landing and production spoke in ew4. | net-vpn-ha | |
## Variables
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman |
-| [organization](variables.tf#L102) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
-| [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap |
-| [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | string | | "data" | |
-| [dns](variables.tf#L66) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
-| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
-| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
-| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
-| [region_trigram](variables.tf#L166) | Short names for GCP regions. | map(string) | | {…} | |
-| [router_onprem_configs](variables.tf#L175) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | |
-| [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | |
-| [service_accounts](variables.tf#L193) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman |
-| [vpn_onprem_configs](variables.tf#L207) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
-| [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L92) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman |
+| [organization](variables.tf#L126) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L142) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
+| [custom_roles](variables.tf#L55) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [dns](variables.tf#L64) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
+| [factories_config](variables.tf#L72) | Configuration for network resource factories. | object({…}) | | {…} | |
+| [l7ilb_subnets](variables.tf#L102) | Subnets used for L7 ILBs. | object({…}) | | {…} | |
+| [outputs_location](variables.tf#L136) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
+| [psa_ranges](variables.tf#L153) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
+| [regions](variables.tf#L190) | Region definitions. | object({…}) | | {…} | |
+| [router_onprem_configs](variables.tf#L202) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | |
+| [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | |
+| [service_accounts](variables.tf#L220) | Automation service accounts in name => email format. | object({…}) | | null | 1-resman |
+| [vpn_onprem_configs](variables.tf#L234) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
+| [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | |
## Outputs
diff --git a/fast/stages/02-networking-peering/data/cidrs.yaml b/fast/stages/2-networking-b-vpn/data/cidrs.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/cidrs.yaml
rename to fast/stages/2-networking-b-vpn/data/cidrs.yaml
diff --git a/fast/stages/02-networking-separate-envs/data/dashboards/firewall_insights.json b/fast/stages/2-networking-b-vpn/data/dashboards/firewall_insights.json
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/dashboards/firewall_insights.json
rename to fast/stages/2-networking-b-vpn/data/dashboards/firewall_insights.json
diff --git a/fast/stages/02-networking-separate-envs/data/dashboards/vpn.json b/fast/stages/2-networking-b-vpn/data/dashboards/vpn.json
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/dashboards/vpn.json
rename to fast/stages/2-networking-b-vpn/data/dashboards/vpn.json
diff --git a/fast/stages/02-networking-peering/data/firewall-rules/dev/rules.yaml b/fast/stages/2-networking-b-vpn/data/firewall-rules/dev/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/firewall-rules/dev/rules.yaml
rename to fast/stages/2-networking-b-vpn/data/firewall-rules/dev/rules.yaml
diff --git a/fast/stages/02-networking-vpn/data/firewall-rules/landing/rules.yaml b/fast/stages/2-networking-b-vpn/data/firewall-rules/landing/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/firewall-rules/landing/rules.yaml
rename to fast/stages/2-networking-b-vpn/data/firewall-rules/landing/rules.yaml
diff --git a/fast/stages/02-networking-peering/data/hierarchical-policy-rules.yaml b/fast/stages/2-networking-b-vpn/data/hierarchical-policy-rules.yaml
similarity index 100%
rename from fast/stages/02-networking-peering/data/hierarchical-policy-rules.yaml
rename to fast/stages/2-networking-b-vpn/data/hierarchical-policy-rules.yaml
diff --git a/fast/stages/02-networking-separate-envs/data/subnets/dev/dev-dataplatform-ew1.yaml b/fast/stages/2-networking-b-vpn/data/subnets/dev/dev-dataplatform-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/subnets/dev/dev-dataplatform-ew1.yaml
rename to fast/stages/2-networking-b-vpn/data/subnets/dev/dev-dataplatform-ew1.yaml
diff --git a/fast/stages/02-networking-separate-envs/data/subnets/dev/dev-default-ew1.yaml b/fast/stages/2-networking-b-vpn/data/subnets/dev/dev-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/subnets/dev/dev-default-ew1.yaml
rename to fast/stages/2-networking-b-vpn/data/subnets/dev/dev-default-ew1.yaml
diff --git a/fast/stages/02-networking-vpn/data/subnets/dev/dev-gke-nodes-ew1.yaml b/fast/stages/2-networking-b-vpn/data/subnets/dev/dev-gke-nodes-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/subnets/dev/dev-gke-nodes-ew1.yaml
rename to fast/stages/2-networking-b-vpn/data/subnets/dev/dev-gke-nodes-ew1.yaml
diff --git a/fast/stages/02-networking-vpn/data/subnets/landing/landing-default-ew1.yaml b/fast/stages/2-networking-b-vpn/data/subnets/landing/landing-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/subnets/landing/landing-default-ew1.yaml
rename to fast/stages/2-networking-b-vpn/data/subnets/landing/landing-default-ew1.yaml
diff --git a/fast/stages/02-networking-separate-envs/data/subnets/prod/prod-default-ew1.yaml b/fast/stages/2-networking-b-vpn/data/subnets/prod/prod-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/subnets/prod/prod-default-ew1.yaml
rename to fast/stages/2-networking-b-vpn/data/subnets/prod/prod-default-ew1.yaml
diff --git a/fast/stages/02-networking-vpn/diagram.png b/fast/stages/2-networking-b-vpn/diagram.png
similarity index 100%
rename from fast/stages/02-networking-vpn/diagram.png
rename to fast/stages/2-networking-b-vpn/diagram.png
diff --git a/fast/stages/02-networking-vpn/diagram.svg b/fast/stages/2-networking-b-vpn/diagram.svg
similarity index 100%
rename from fast/stages/02-networking-vpn/diagram.svg
rename to fast/stages/2-networking-b-vpn/diagram.svg
diff --git a/fast/stages/02-networking-vpn/dns-dev.tf b/fast/stages/2-networking-b-vpn/dns-dev.tf
similarity index 100%
rename from fast/stages/02-networking-vpn/dns-dev.tf
rename to fast/stages/2-networking-b-vpn/dns-dev.tf
diff --git a/fast/stages/02-networking-vpn/dns-landing.tf b/fast/stages/2-networking-b-vpn/dns-landing.tf
similarity index 100%
rename from fast/stages/02-networking-vpn/dns-landing.tf
rename to fast/stages/2-networking-b-vpn/dns-landing.tf
diff --git a/fast/stages/02-networking-vpn/dns-prod.tf b/fast/stages/2-networking-b-vpn/dns-prod.tf
similarity index 100%
rename from fast/stages/02-networking-vpn/dns-prod.tf
rename to fast/stages/2-networking-b-vpn/dns-prod.tf
diff --git a/fast/stages/02-networking-vpn/landing.tf b/fast/stages/2-networking-b-vpn/landing.tf
similarity index 82%
rename from fast/stages/02-networking-vpn/landing.tf
rename to fast/stages/2-networking-b-vpn/landing.tf
index 83a0d509..37e3adfd 100644
--- a/fast/stages/02-networking-vpn/landing.tf
+++ b/fast/stages/2-networking-b-vpn/landing.tf
@@ -63,7 +63,7 @@ module "landing-vpc" {
next_hop = "default-internet-gateway"
}
}
- data_folder = "${var.data_dir}/subnets/landing"
+ data_folder = "${var.factories_config.data_dir}/subnets/landing"
}
module "landing-firewall" {
@@ -74,18 +74,23 @@ module "landing-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/landing"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/landing"
}
}
-module "landing-nat-ew1" {
+moved {
+ from = module.landing-nat-ew1
+ to = module.landing-nat-primary
+}
+
+module "landing-nat-primary" {
source = "../../../modules/net-cloudnat"
project_id = module.landing-project.project_id
- region = "europe-west1"
- name = "ew1"
+ region = var.regions.primary
+ name = local.region_shortnames[var.regions.primary]
router_create = true
- router_name = "prod-nat-ew1"
+ router_name = "prod-nat-${local.region_shortnames[var.regions.primary]}"
router_network = module.landing-vpc.name
router_asn = 4200001024
}
diff --git a/fast/stages/02-networking-peering/main.tf b/fast/stages/2-networking-b-vpn/main.tf
similarity index 76%
rename from fast/stages/02-networking-peering/main.tf
rename to fast/stages/2-networking-b-vpn/main.tf
index f68d39eb..5888752d 100644
--- a/fast/stages/02-networking-peering/main.tf
+++ b/fast/stages/2-networking-b-vpn/main.tf
@@ -17,14 +17,14 @@
# tfdoc:file:description Networking folder and hierarchical policy.
locals {
+ # combine all regions from variables and subnets
+ regions = distinct(concat(
+ values(var.regions),
+ values(module.dev-spoke-vpc.subnet_regions),
+ values(module.landing-vpc.subnet_regions),
+ values(module.prod-spoke-vpc.subnet_regions),
+ ))
custom_roles = coalesce(var.custom_roles, {})
- l7ilb_subnets = {
- for env, v in var.l7ilb_subnets : env => [
- for s in v : merge(s, {
- active = true
- name = "${env}-l7ilb-${s.region}"
- })]
- }
stage3_sas_delegated_grants = [
"roles/composer.sharedVpcAgent",
"roles/compute.networkUser",
@@ -46,9 +46,9 @@ module "folder" {
folder_create = var.folder_ids.networking == null
id = var.folder_ids.networking
firewall_policy_factory = {
- cidr_file = "${var.data_dir}/cidrs.yaml"
- policy_name = null
- rules_file = "${var.data_dir}/hierarchical-policy-rules.yaml"
+ cidr_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ policy_name = var.factories_config.firewall_policy_name
+ rules_file = "${var.factories_config.data_dir}/hierarchical-policy-rules.yaml"
}
firewall_policy_association = {
factory-policy = "factory"
diff --git a/fast/stages/02-networking-nva/monitoring.tf b/fast/stages/2-networking-b-vpn/monitoring.tf
similarity index 93%
rename from fast/stages/02-networking-nva/monitoring.tf
rename to fast/stages/2-networking-b-vpn/monitoring.tf
index 7b8b70c5..be3a47fa 100644
--- a/fast/stages/02-networking-nva/monitoring.tf
+++ b/fast/stages/2-networking-b-vpn/monitoring.tf
@@ -17,7 +17,7 @@
# tfdoc:file:description Network monitoring dashboards.
locals {
- dashboard_path = "${var.data_dir}/dashboards"
+ dashboard_path = "${var.factories_config.data_dir}/dashboards"
dashboard_files = fileset(local.dashboard_path, "*.json")
dashboards = {
for filename in local.dashboard_files :
diff --git a/fast/stages/02-networking-vpn/outputs.tf b/fast/stages/2-networking-b-vpn/outputs.tf
similarity index 93%
rename from fast/stages/02-networking-vpn/outputs.tf
rename to fast/stages/2-networking-b-vpn/outputs.tf
index 3b97b7f2..628c706b 100644
--- a/fast/stages/02-networking-vpn/outputs.tf
+++ b/fast/stages/2-networking-b-vpn/outputs.tf
@@ -48,13 +48,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/02-networking.auto.tfvars.json"
+ filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/02-networking.auto.tfvars.json"
+ name = "tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
@@ -89,8 +89,8 @@ output "tfvars" {
output "vpn_gateway_endpoints" {
description = "External IP Addresses for the GCP VPN gateways."
value = local.enable_onprem_vpn == false ? null : {
- onprem-ew1 = {
- for v in module.landing-to-onprem-ew1-vpn[0].gateway.vpn_interfaces :
+ onprem-primary = {
+ for v in module.landing-to-onprem-primary-vpn[0].gateway.vpn_interfaces :
v.id => v.ip_address
}
}
diff --git a/fast/stages/2-networking-b-vpn/regions.tf b/fast/stages/2-networking-b-vpn/regions.tf
new file mode 100644
index 00000000..53514afa
--- /dev/null
+++ b/fast/stages/2-networking-b-vpn/regions.tf
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Compute short names for regions.
+
+locals {
+ # only map when the first character would not work
+ _region_cardinal = {
+ southeast = "se"
+ }
+ # only map when the first character would not work
+ _region_geo = {
+ australia = "o"
+ }
+ # split in [geo, cardinal, number] tokens
+ _region_tokens = {
+ for v in local.regions : v => regexall("(?:[a-z]+)|(?:[0-9]+)", v)
+ }
+ region_shortnames = {
+ for k, v in local._region_tokens : k => join("", [
+ # first token via geo alias map or first character
+ lookup(local._region_geo, v.0, substr(v.0, 0, 1)),
+ # first token via cardinal alias map or first character
+ lookup(local._region_cardinal, v.1, substr(v.1, 0, 1)),
+ # region number as is
+ v.2
+ ])
+ }
+}
diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/2-networking-b-vpn/spoke-dev.tf
similarity index 84%
rename from fast/stages/02-networking-vpn/spoke-dev.tf
rename to fast/stages/2-networking-b-vpn/spoke-dev.tf
index e67cfb70..ed7d13cb 100644
--- a/fast/stages/02-networking-vpn/spoke-dev.tf
+++ b/fast/stages/2-networking-b-vpn/spoke-dev.tf
@@ -16,6 +16,19 @@
# tfdoc:file:description Dev spoke VPC and related resources.
+locals {
+ _l7ilb_subnets_dev = [
+ for v in var.l7ilb_subnets.dev : merge(v, {
+ active = true
+ region = lookup(var.regions, v.region, v.region)
+ })]
+ l7ilb_subnets_dev = [
+ for v in local._l7ilb_subnets_dev : merge(v, {
+ name = "dev-l7ilb-${local.region_shortnames[v.region]}"
+ })
+ ]
+}
+
module "dev-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account.id
@@ -48,9 +61,9 @@ module "dev-spoke-vpc" {
project_id = module.dev-spoke-project.project_id
name = "dev-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/dev"
+ data_folder = "${var.factories_config.data_dir}/subnets/dev"
psa_config = try(var.psa_ranges.dev, null)
- subnets_proxy_only = local.l7ilb_subnets.dev
+ subnets_proxy_only = local.l7ilb_subnets_dev
# set explicit routes for googleapis in case the default route is deleted
routes = {
private-googleapis = {
@@ -74,8 +87,8 @@ module "dev-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/dev"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/dev"
}
}
@@ -84,7 +97,7 @@ module "dev-spoke-cloudnat" {
source = "../../../modules/net-cloudnat"
project_id = module.dev-spoke-project.project_id
region = each.value
- name = "dev-nat-${var.region_trigram[each.value]}"
+ name = "dev-nat-${local.region_shortnames[each.value]}"
router_create = true
router_network = module.dev-spoke-vpc.name
router_asn = 4200001024
diff --git a/fast/stages/02-networking-peering/spoke-prod.tf b/fast/stages/2-networking-b-vpn/spoke-prod.tf
similarity index 84%
rename from fast/stages/02-networking-peering/spoke-prod.tf
rename to fast/stages/2-networking-b-vpn/spoke-prod.tf
index cf49152f..f584b32d 100644
--- a/fast/stages/02-networking-peering/spoke-prod.tf
+++ b/fast/stages/2-networking-b-vpn/spoke-prod.tf
@@ -16,6 +16,19 @@
# tfdoc:file:description Production spoke VPC and related resources.
+locals {
+ _l7ilb_subnets_prod = [
+ for v in var.l7ilb_subnets.prod : merge(v, {
+ active = true
+ region = lookup(var.regions, v.region, v.region)
+ })]
+ l7ilb_subnets_prod = [
+ for v in local._l7ilb_subnets_prod : merge(v, {
+ name = "prod-l7ilb-${local.region_shortnames[v.region]}"
+ })
+ ]
+}
+
module "prod-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account.id
@@ -48,9 +61,9 @@ module "prod-spoke-vpc" {
project_id = module.prod-spoke-project.project_id
name = "prod-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/prod"
+ data_folder = "${var.factories_config.data_dir}/subnets/prod"
psa_config = try(var.psa_ranges.prod, null)
- subnets_proxy_only = local.l7ilb_subnets.prod
+ subnets_proxy_only = local.l7ilb_subnets_prod
# set explicit routes for googleapis in case the default route is deleted
routes = {
private-googleapis = {
@@ -74,8 +87,8 @@ module "prod-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/prod"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/prod"
}
}
@@ -84,7 +97,7 @@ module "prod-spoke-cloudnat" {
source = "../../../modules/net-cloudnat"
project_id = module.prod-spoke-project.project_id
region = each.value
- name = "prod-nat-${var.region_trigram[each.value]}"
+ name = "prod-nat-${local.region_shortnames[each.value]}"
router_create = true
router_network = module.prod-spoke-vpc.name
router_asn = 4200001024
diff --git a/fast/stages/02-networking-vpn/test-resources.tf b/fast/stages/2-networking-b-vpn/test-resources.tf
similarity index 76%
rename from fast/stages/02-networking-vpn/test-resources.tf
rename to fast/stages/2-networking-b-vpn/test-resources.tf
index 204971fe..67073845 100644
--- a/fast/stages/02-networking-vpn/test-resources.tf
+++ b/fast/stages/2-networking-b-vpn/test-resources.tf
@@ -19,11 +19,11 @@
# module "test-vm-landing-0" {
# source = "../../../modules/compute-vm"
# project_id = module.landing-project.project_id
-# zone = "europe-west1-b"
+# zone = "${var.regions.primary}-b"
# name = "test-vm-0"
# network_interfaces = [{
# network = module.landing-vpc.self_link
-# subnetwork = module.landing-vpc.subnet_self_links["europe-west1/landing-default-ew1"]
+# subnetwork = module.landing-vpc.subnet_self_links["${var.regions.primary}/landing-default-${local.region_shortnames[var.regions.primary]}"]
# }]
# tags = ["ssh"]
# service_account_create = true
@@ -31,8 +31,8 @@
# image = "projects/debian-cloud/global/images/family/debian-10"
# }
# options = {
-# spot = true
-# termination_action = "STOP"
+# spot = true
+# termination_action = "STOP"
# }
# metadata = {
# startup-script = <google_monitoring_dashboard |
| [nva.tf](./nva.tf) | None | compute-mig · compute-vm · simple-nva | |
| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object · local_file |
+| [regions.tf](./regions.tf) | Compute short names for regions. | | |
| [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-vpc · net-vpc-firewall · net-vpc-peering · project | google_project_iam_binding |
| [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-vpc · net-vpc-firewall · net-vpc-peering · project | google_project_iam_binding |
| [test-resources.tf](./test-resources.tf) | temporary instances for testing | compute-vm | |
@@ -371,23 +369,23 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [folder_ids](variables.tf#L79) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman |
-| [organization](variables.tf#L115) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L131) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
-| [custom_roles](variables.tf#L56) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap |
-| [data_dir](variables.tf#L65) | Relative path for the folder storing configuration data for network resources. | string | | "data" | |
-| [dns](variables.tf#L71) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
-| [l7ilb_subnets](variables.tf#L89) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
-| [onprem_cidr](variables.tf#L107) | Onprem addresses in name => range format. | map(string) | | {…} | |
-| [outputs_location](variables.tf#L125) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
-| [psa_ranges](variables.tf#L142) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
-| [region_trigram](variables.tf#L183) | Short names for GCP regions. | map(string) | | {…} | |
-| [router_configs](variables.tf#L192) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | |
-| [service_accounts](variables.tf#L215) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman |
-| [vpn_onprem_configs](variables.tf#L229) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L97) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman |
+| [organization](variables.tf#L133) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L149) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
+| [custom_roles](variables.tf#L60) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [dns](variables.tf#L69) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
+| [factories_config](variables.tf#L77) | Configuration for network resource factories. | object({…}) | | {…} | |
+| [l7ilb_subnets](variables.tf#L107) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
+| [onprem_cidr](variables.tf#L125) | Onprem addresses in name => range format. | map(string) | | {…} | |
+| [outputs_location](variables.tf#L143) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
+| [psa_ranges](variables.tf#L160) | IP ranges used for Private Service Access (e.g. CloudSQL). Ranges is in name => range format. | object({…}) | | null | |
+| [regions](variables.tf#L181) | Region definitions. | object({…}) | | {…} | |
+| [router_configs](variables.tf#L193) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | |
+| [service_accounts](variables.tf#L216) | Automation service accounts in name => email format. | object({…}) | | null | 1-resman |
+| [vpn_onprem_configs](variables.tf#L230) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
## Outputs
diff --git a/fast/stages/02-networking-separate-envs/data/cidrs.yaml b/fast/stages/2-networking-c-nva/data/cidrs.yaml
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/cidrs.yaml
rename to fast/stages/2-networking-c-nva/data/cidrs.yaml
diff --git a/fast/stages/02-networking-nva/data/dashboards/firewall_insights.json b/fast/stages/2-networking-c-nva/data/dashboards/firewall_insights.json
similarity index 100%
rename from fast/stages/02-networking-nva/data/dashboards/firewall_insights.json
rename to fast/stages/2-networking-c-nva/data/dashboards/firewall_insights.json
diff --git a/fast/stages/02-networking-nva/data/dashboards/vpn.json b/fast/stages/2-networking-c-nva/data/dashboards/vpn.json
similarity index 100%
rename from fast/stages/02-networking-nva/data/dashboards/vpn.json
rename to fast/stages/2-networking-c-nva/data/dashboards/vpn.json
diff --git a/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml b/fast/stages/2-networking-c-nva/data/firewall-rules/dev/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml
rename to fast/stages/2-networking-c-nva/data/firewall-rules/dev/rules.yaml
diff --git a/fast/stages/02-networking-nva/data/firewall-rules/landing-trusted/rules.yaml b/fast/stages/2-networking-c-nva/data/firewall-rules/landing-trusted/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/firewall-rules/landing-trusted/rules.yaml
rename to fast/stages/2-networking-c-nva/data/firewall-rules/landing-trusted/rules.yaml
diff --git a/fast/stages/02-networking-nva/data/firewall-rules/landing-untrusted/rules.yaml b/fast/stages/2-networking-c-nva/data/firewall-rules/landing-untrusted/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/firewall-rules/landing-untrusted/rules.yaml
rename to fast/stages/2-networking-c-nva/data/firewall-rules/landing-untrusted/rules.yaml
diff --git a/fast/stages/02-networking-separate-envs/data/hierarchical-policy-rules.yaml b/fast/stages/2-networking-c-nva/data/hierarchical-policy-rules.yaml
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/hierarchical-policy-rules.yaml
rename to fast/stages/2-networking-c-nva/data/hierarchical-policy-rules.yaml
diff --git a/fast/stages/02-networking-nva/data/nva-startup-script.tftpl b/fast/stages/2-networking-c-nva/data/nva-startup-script.tftpl
similarity index 100%
rename from fast/stages/02-networking-nva/data/nva-startup-script.tftpl
rename to fast/stages/2-networking-c-nva/data/nva-startup-script.tftpl
diff --git a/fast/stages/02-networking-nva/data/subnets/dev/dev-dataplatform-ew1.yaml b/fast/stages/2-networking-c-nva/data/subnets/dev/dev-dataplatform-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/dev/dev-dataplatform-ew1.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/dev/dev-dataplatform-ew1.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/dev/dev-default-ew1.yaml b/fast/stages/2-networking-c-nva/data/subnets/dev/dev-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/dev/dev-default-ew1.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/dev/dev-default-ew1.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/dev/dev-default-ew4.yaml b/fast/stages/2-networking-c-nva/data/subnets/dev/dev-default-ew4.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/dev/dev-default-ew4.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/dev/dev-default-ew4.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/landing-trusted/landing-trusted-default-ew1.yaml b/fast/stages/2-networking-c-nva/data/subnets/landing-trusted/landing-trusted-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/landing-trusted/landing-trusted-default-ew1.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/landing-trusted/landing-trusted-default-ew1.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/landing-trusted/landing-trusted-default-ew4.yaml b/fast/stages/2-networking-c-nva/data/subnets/landing-trusted/landing-trusted-default-ew4.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/landing-trusted/landing-trusted-default-ew4.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/landing-trusted/landing-trusted-default-ew4.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew1.yaml b/fast/stages/2-networking-c-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew1.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew1.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew4.yaml b/fast/stages/2-networking-c-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew4.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew4.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/landing-untrusted/landing-untrusted-default-ew4.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/prod/prod-default-ew1.yaml b/fast/stages/2-networking-c-nva/data/subnets/prod/prod-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/prod/prod-default-ew1.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/prod/prod-default-ew1.yaml
diff --git a/fast/stages/02-networking-nva/data/subnets/prod/prod-default-ew4.yaml b/fast/stages/2-networking-c-nva/data/subnets/prod/prod-default-ew4.yaml
similarity index 100%
rename from fast/stages/02-networking-nva/data/subnets/prod/prod-default-ew4.yaml
rename to fast/stages/2-networking-c-nva/data/subnets/prod/prod-default-ew4.yaml
diff --git a/fast/stages/02-networking-nva/diagram.png b/fast/stages/2-networking-c-nva/diagram.png
similarity index 100%
rename from fast/stages/02-networking-nva/diagram.png
rename to fast/stages/2-networking-c-nva/diagram.png
diff --git a/fast/stages/02-networking-nva/diagram.svg b/fast/stages/2-networking-c-nva/diagram.svg
similarity index 100%
rename from fast/stages/02-networking-nva/diagram.svg
rename to fast/stages/2-networking-c-nva/diagram.svg
diff --git a/fast/stages/02-networking-nva/dns-dev.tf b/fast/stages/2-networking-c-nva/dns-dev.tf
similarity index 100%
rename from fast/stages/02-networking-nva/dns-dev.tf
rename to fast/stages/2-networking-c-nva/dns-dev.tf
diff --git a/fast/stages/02-networking-nva/dns-landing.tf b/fast/stages/2-networking-c-nva/dns-landing.tf
similarity index 100%
rename from fast/stages/02-networking-nva/dns-landing.tf
rename to fast/stages/2-networking-c-nva/dns-landing.tf
diff --git a/fast/stages/02-networking-nva/dns-prod.tf b/fast/stages/2-networking-c-nva/dns-prod.tf
similarity index 100%
rename from fast/stages/02-networking-nva/dns-prod.tf
rename to fast/stages/2-networking-c-nva/dns-prod.tf
diff --git a/fast/stages/02-networking-nva/landing.tf b/fast/stages/2-networking-c-nva/landing.tf
similarity index 76%
rename from fast/stages/02-networking-nva/landing.tf
rename to fast/stages/2-networking-c-nva/landing.tf
index 5a990030..e66b03db 100644
--- a/fast/stages/02-networking-nva/landing.tf
+++ b/fast/stages/2-networking-c-nva/landing.tf
@@ -53,7 +53,7 @@ module "landing-untrusted-vpc" {
inbound = false
logging = false
}
- data_folder = "${var.data_dir}/subnets/landing-untrusted"
+ data_folder = "${var.factories_config.data_dir}/subnets/landing-untrusted"
}
module "landing-untrusted-firewall" {
@@ -64,31 +64,41 @@ module "landing-untrusted-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/landing-untrusted"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/landing-untrusted"
}
}
# NAT
-module "landing-nat-ew1" {
+moved {
+ from = module.landing-nat-ew1
+ to = module.landing-nat-primary
+}
+
+module "landing-nat-primary" {
source = "../../../modules/net-cloudnat"
project_id = module.landing-project.project_id
- region = "europe-west1"
- name = "ew1"
+ region = var.regions.primary
+ name = local.region_shortnames[var.regions.primary]
router_create = true
- router_name = "prod-nat-ew1"
+ router_name = "prod-nat-${local.region_shortnames[var.regions.primary]}"
router_network = module.landing-untrusted-vpc.name
router_asn = 4200001024
}
-module "landing-nat-ew4" {
+moved {
+ from = module.landing-nat-ew4
+ to = module.landing-nat-secondary
+}
+
+module "landing-nat-secondary" {
source = "../../../modules/net-cloudnat"
project_id = module.landing-project.project_id
- region = "europe-west4"
- name = "ew4"
+ region = var.regions.secondary
+ name = local.region_shortnames[var.regions.secondary]
router_create = true
- router_name = "prod-nat-ew4"
+ router_name = "prod-nat-${local.region_shortnames[var.regions.secondary]}"
router_network = module.landing-untrusted-vpc.name
router_asn = 4200001024
}
@@ -101,7 +111,10 @@ module "landing-trusted-vpc" {
name = "prod-trusted-landing-0"
delete_default_routes_on_create = true
mtu = 1500
-
+ data_folder = "${var.factories_config.data_dir}/subnets/landing-trusted"
+ dns_policy = {
+ inbound = true
+ }
# Set explicit routes for googleapis in case the default route is deleted
routes = {
private-googleapis = {
@@ -115,12 +128,6 @@ module "landing-trusted-vpc" {
next_hop = "default-internet-gateway"
}
}
-
- dns_policy = {
- inbound = true
- }
-
- data_folder = "${var.data_dir}/subnets/landing-trusted"
}
module "landing-trusted-firewall" {
@@ -131,7 +138,7 @@ module "landing-trusted-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/landing-trusted"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/landing-trusted"
}
}
diff --git a/fast/stages/02-networking-nva/main.tf b/fast/stages/2-networking-c-nva/main.tf
similarity index 73%
rename from fast/stages/02-networking-nva/main.tf
rename to fast/stages/2-networking-c-nva/main.tf
index 4db5061b..e4066ba3 100644
--- a/fast/stages/02-networking-nva/main.tf
+++ b/fast/stages/2-networking-c-nva/main.tf
@@ -24,6 +24,14 @@ locals {
name = "${env}-l7ilb-${s.region}"
})]
}
+ # combine all regions from variables and subnets
+ regions = distinct(concat(
+ values(var.regions),
+ values(module.dev-spoke-vpc.subnet_regions),
+ values(module.landing-trusted-vpc.subnet_regions),
+ values(module.landing-untrusted-vpc.subnet_regions),
+ values(module.prod-spoke-vpc.subnet_regions),
+ ))
service_accounts = {
for k, v in coalesce(var.service_accounts, {}) :
k => "serviceAccount:${v}" if v != null
@@ -45,11 +53,11 @@ module "folder" {
folder_create = var.folder_ids.networking == null
id = var.folder_ids.networking
firewall_policy_factory = {
- cidr_file = "${var.data_dir}/cidrs.yaml"
- policy_name = null
- rules_file = "${var.data_dir}/hierarchical-policy-rules.yaml"
+ cidr_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ policy_name = var.factories_config.firewall_policy_name
+ rules_file = "${var.factories_config.data_dir}/hierarchical-policy-rules.yaml"
}
firewall_policy_association = {
- factory-policy = "factory"
+ factory-policy = var.factories_config.firewall_policy_name
}
}
diff --git a/fast/stages/02-networking-peering/monitoring.tf b/fast/stages/2-networking-c-nva/monitoring.tf
similarity index 93%
rename from fast/stages/02-networking-peering/monitoring.tf
rename to fast/stages/2-networking-c-nva/monitoring.tf
index 7b8b70c5..be3a47fa 100644
--- a/fast/stages/02-networking-peering/monitoring.tf
+++ b/fast/stages/2-networking-c-nva/monitoring.tf
@@ -17,7 +17,7 @@
# tfdoc:file:description Network monitoring dashboards.
locals {
- dashboard_path = "${var.data_dir}/dashboards"
+ dashboard_path = "${var.factories_config.data_dir}/dashboards"
dashboard_files = fileset(local.dashboard_path, "*.json")
dashboards = {
for filename in local.dashboard_files :
diff --git a/fast/stages/02-networking-nva/nva.tf b/fast/stages/2-networking-c-nva/nva.tf
similarity index 66%
rename from fast/stages/02-networking-nva/nva.tf
rename to fast/stages/2-networking-c-nva/nva.tf
index f4f7b9e5..7ae9c30c 100644
--- a/fast/stages/02-networking-nva/nva.tf
+++ b/fast/stages/2-networking-c-nva/nva.tf
@@ -21,29 +21,32 @@ locals {
{
name = "untrusted"
routes = [
- var.custom_adv.gcp_landing_untrusted_ew1,
- var.custom_adv.gcp_landing_untrusted_ew4,
+ var.custom_adv.gcp_landing_untrusted_primary,
+ var.custom_adv.gcp_landing_untrusted_secondary,
]
},
{
name = "trusted"
routes = [
- var.custom_adv.gcp_dev_ew1,
- var.custom_adv.gcp_dev_ew4,
- var.custom_adv.gcp_landing_trusted_ew1,
- var.custom_adv.gcp_landing_trusted_ew4,
- var.custom_adv.gcp_prod_ew1,
- var.custom_adv.gcp_prod_ew4,
+ var.custom_adv.gcp_dev_primary,
+ var.custom_adv.gcp_dev_secondary,
+ var.custom_adv.gcp_landing_trusted_primary,
+ var.custom_adv.gcp_landing_trusted_secondary,
+ var.custom_adv.gcp_prod_primary,
+ var.custom_adv.gcp_prod_secondary,
]
},
]
nva_locality = {
- europe-west1-b = { region = "europe-west1", trigram = "ew1", zone = "b" },
- europe-west1-c = { region = "europe-west1", trigram = "ew1", zone = "c" },
- europe-west4-b = { region = "europe-west4", trigram = "ew4", zone = "b" },
- europe-west4-c = { region = "europe-west4", trigram = "ew4", zone = "c" },
+ for v in setproduct(keys(var.regions), local.nva_zones) :
+ join("-", v) => {
+ name = v.0
+ region = var.regions[v.0]
+ shortname = local.region_shortnames[var.regions[v.0]]
+ zone = v.1
+ }
}
-
+ nva_zones = ["b", "c"]
}
# NVA config
@@ -57,7 +60,7 @@ module "nva-template" {
for_each = local.nva_locality
source = "../../../modules/compute-vm"
project_id = module.landing-project.project_id
- name = "nva-template-${each.value.trigram}-${each.value.zone}"
+ name = "nva-template-${each.key}"
zone = "${each.value.region}-${each.value.zone}"
instance_type = "e2-standard-2"
tags = ["nva"]
@@ -66,13 +69,13 @@ module "nva-template" {
network_interfaces = [
{
network = module.landing-untrusted-vpc.self_link
- subnetwork = module.landing-untrusted-vpc.subnet_self_links["${each.value.region}/landing-untrusted-default-${each.value.trigram}"]
+ subnetwork = module.landing-untrusted-vpc.subnet_self_links["${each.value.region}/landing-untrusted-default-${each.value.shortname}"]
nat = false
addresses = null
},
{
network = module.landing-trusted-vpc.self_link
- subnetwork = module.landing-trusted-vpc.subnet_self_links["${each.value.region}/landing-trusted-default-${each.value.trigram}"]
+ subnetwork = module.landing-trusted-vpc.subnet_self_links["${each.value.region}/landing-trusted-default-${each.value.shortname}"]
nat = false
addresses = null
}
@@ -98,7 +101,7 @@ module "nva-mig" {
source = "../../../modules/compute-mig"
project_id = module.landing-project.project_id
location = each.value.region
- name = "nva-cos-${each.value.trigram}-${each.value.zone}"
+ name = "nva-cos-${each.key}"
instance_template = module.nva-template[each.key].template.self_link
target_size = 1
auto_healing_policies = {
@@ -113,21 +116,27 @@ module "nva-mig" {
}
module "ilb-nva-untrusted" {
- for_each = { for l in local.nva_locality : l.region => l.trigram... }
+ for_each = {
+ for k, v in var.regions : k => {
+ region = v
+ shortname = local.region_shortnames[v]
+ subnet = "${v}/landing-untrusted-default-${local.region_shortnames[v]}"
+ }
+ }
source = "../../../modules/net-ilb"
project_id = module.landing-project.project_id
- region = each.key
- name = "nva-untrusted-${each.value.0}"
+ region = each.value.region
+ name = "nva-untrusted-${each.key}"
service_label = var.prefix
global_access = true
vpc_config = {
network = module.landing-untrusted-vpc.self_link
- subnetwork = module.landing-untrusted-vpc.subnet_self_links["${each.key}/landing-untrusted-default-${each.value.0}"]
+ subnetwork = module.landing-untrusted-vpc.subnet_self_links[each.value.subnet]
}
backends = [
- for key, _ in local.nva_locality : {
- group = module.nva-mig[key].group_manager.instance_group
- } if local.nva_locality[key].region == each.key
+ for k, v in module.nva-mig :
+ { group = v.group_manager.instance_group }
+ if startswith(k, each.key)
]
health_check_config = {
enable_logging = true
@@ -137,23 +146,28 @@ module "ilb-nva-untrusted" {
}
}
-
module "ilb-nva-trusted" {
- for_each = { for l in local.nva_locality : l.region => l.trigram... }
+ for_each = {
+ for k, v in var.regions : k => {
+ region = v
+ shortname = local.region_shortnames[v]
+ subnet = "${v}/landing-trusted-default-${local.region_shortnames[v]}"
+ }
+ }
source = "../../../modules/net-ilb"
project_id = module.landing-project.project_id
- region = each.key
- name = "nva-trusted-${each.value.0}"
+ region = each.value.region
+ name = "nva-trusted-${each.key}"
service_label = var.prefix
global_access = true
vpc_config = {
network = module.landing-trusted-vpc.self_link
- subnetwork = module.landing-trusted-vpc.subnet_self_links["${each.key}/landing-trusted-default-${each.value.0}"]
+ subnetwork = module.landing-trusted-vpc.subnet_self_links[each.value.subnet]
}
backends = [
- for key, _ in local.nva_locality : {
- group = module.nva-mig[key].group_manager.instance_group
- } if local.nva_locality[key].region == each.key
+ for k, v in module.nva-mig :
+ { group = v.group_manager.instance_group }
+ if startswith(k, each.key)
]
health_check_config = {
enable_logging = true
@@ -162,4 +176,3 @@ module "ilb-nva-trusted" {
}
}
}
-
diff --git a/fast/stages/02-networking-nva/outputs.tf b/fast/stages/2-networking-c-nva/outputs.tf
similarity index 89%
rename from fast/stages/02-networking-nva/outputs.tf
rename to fast/stages/2-networking-c-nva/outputs.tf
index df324570..74620242 100644
--- a/fast/stages/02-networking-nva/outputs.tf
+++ b/fast/stages/2-networking-c-nva/outputs.tf
@@ -43,13 +43,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/02-networking.auto.tfvars.json"
+ filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/02-networking.auto.tfvars.json"
+ name = "tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
@@ -79,12 +79,12 @@ output "tfvars" {
output "vpn_gateway_endpoints" {
description = "External IP Addresses for the GCP VPN gateways."
value = local.enable_onprem_vpn == false ? null : {
- onprem-ew1 = {
- for v in module.landing-to-onprem-ew1-vpn[0].gateway.vpn_interfaces :
+ onprem-primary = {
+ for v in module.landing-to-onprem-primary-vpn[0].gateway.vpn_interfaces :
v.id => v.ip_address
}
- onprem-ew4 = {
- for v in module.landing-to-onprem-ew4-vpn[0].gateway.vpn_interfaces :
+ onprem-secondary = {
+ for v in module.landing-to-onprem-secondary-vpn[0].gateway.vpn_interfaces :
v.id => v.ip_address
}
}
diff --git a/fast/stages/2-networking-c-nva/regions.tf b/fast/stages/2-networking-c-nva/regions.tf
new file mode 100644
index 00000000..53514afa
--- /dev/null
+++ b/fast/stages/2-networking-c-nva/regions.tf
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Compute short names for regions.
+
+locals {
+ # only map when the first character would not work
+ _region_cardinal = {
+ southeast = "se"
+ }
+ # only map when the first character would not work
+ _region_geo = {
+ australia = "o"
+ }
+ # split in [geo, cardinal, number] tokens
+ _region_tokens = {
+ for v in local.regions : v => regexall("(?:[a-z]+)|(?:[0-9]+)", v)
+ }
+ region_shortnames = {
+ for k, v in local._region_tokens : k => join("", [
+ # first token via geo alias map or first character
+ lookup(local._region_geo, v.0, substr(v.0, 0, 1)),
+ # first token via cardinal alias map or first character
+ lookup(local._region_cardinal, v.1, substr(v.1, 0, 1)),
+ # region number as is
+ v.2
+ ])
+ }
+}
diff --git a/fast/stages/02-networking-nva/spoke-dev.tf b/fast/stages/2-networking-c-nva/spoke-dev.tf
similarity index 76%
rename from fast/stages/02-networking-nva/spoke-dev.tf
rename to fast/stages/2-networking-c-nva/spoke-dev.tf
index fb88384c..b2293813 100644
--- a/fast/stages/02-networking-nva/spoke-dev.tf
+++ b/fast/stages/2-networking-c-nva/spoke-dev.tf
@@ -16,6 +16,19 @@
# tfdoc:file:description Dev spoke VPC and related resources.
+locals {
+ _l7ilb_subnets_dev = [
+ for v in var.l7ilb_subnets.dev : merge(v, {
+ active = true
+ region = lookup(var.regions, v.region, v.region)
+ })]
+ l7ilb_subnets_dev = [
+ for v in local._l7ilb_subnets_dev : merge(v, {
+ name = "dev-l7ilb-${local.region_shortnames[v.region]}"
+ })
+ ]
+}
+
module "dev-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account.id
@@ -47,10 +60,10 @@ module "dev-spoke-vpc" {
project_id = module.dev-spoke-project.project_id
name = "dev-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/dev"
+ data_folder = "${var.factories_config.data_dir}/subnets/dev"
delete_default_routes_on_create = true
psa_config = try(var.psa_ranges.dev, null)
- subnets_proxy_only = local.l7ilb_subnets.dev
+ subnets_proxy_only = local.l7ilb_subnets_dev
# Set explicit routes for googleapis; send everything else to NVAs
routes = {
private-googleapis = {
@@ -65,33 +78,33 @@ module "dev-spoke-vpc" {
next_hop_type = "gateway"
next_hop = "default-internet-gateway"
}
- nva-ew1-to-ew1 = {
+ nva-primary-to-primary = {
dest_range = "0.0.0.0/0"
priority = 1000
- tags = ["ew1"]
+ tags = ["primary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west1"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["primary"].forwarding_rule_address
}
- nva-ew4-to-ew4 = {
+ nva-secondary-to-secondary = {
dest_range = "0.0.0.0/0"
priority = 1000
- tags = ["ew4"]
+ tags = ["secondary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west4"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["secondary"].forwarding_rule_address
}
- nva-ew1-to-ew4 = {
+ nva-primary-to-secondary = {
dest_range = "0.0.0.0/0"
priority = 1001
- tags = ["ew1"]
+ tags = ["primary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west4"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["primary"].forwarding_rule_address
}
- nva-ew4-to-ew1 = {
+ nva-secondary-to-primary = {
dest_range = "0.0.0.0/0"
priority = 1001
- tags = ["ew4"]
+ tags = ["secondary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west1"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["secondary"].forwarding_rule_address
}
}
}
@@ -104,8 +117,8 @@ module "dev-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/dev"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/dev"
}
}
diff --git a/fast/stages/02-networking-nva/spoke-prod.tf b/fast/stages/2-networking-c-nva/spoke-prod.tf
similarity index 76%
rename from fast/stages/02-networking-nva/spoke-prod.tf
rename to fast/stages/2-networking-c-nva/spoke-prod.tf
index 484550ac..51ad2974 100644
--- a/fast/stages/02-networking-nva/spoke-prod.tf
+++ b/fast/stages/2-networking-c-nva/spoke-prod.tf
@@ -16,6 +16,19 @@
# tfdoc:file:description Production spoke VPC and related resources.
+locals {
+ _l7ilb_subnets_prod = [
+ for v in var.l7ilb_subnets.prod : merge(v, {
+ active = true
+ region = lookup(var.regions, v.region, v.region)
+ })]
+ l7ilb_subnets_prod = [
+ for v in local._l7ilb_subnets_prod : merge(v, {
+ name = "prod-l7ilb-${local.region_shortnames[v.region]}"
+ })
+ ]
+}
+
module "prod-spoke-project" {
source = "../../../modules/project"
billing_account = var.billing_account.id
@@ -47,10 +60,10 @@ module "prod-spoke-vpc" {
project_id = module.prod-spoke-project.project_id
name = "prod-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/prod"
+ data_folder = "${var.factories_config.data_dir}/subnets/prod"
delete_default_routes_on_create = true
psa_config = try(var.psa_ranges.prod, null)
- subnets_proxy_only = local.l7ilb_subnets.prod
+ subnets_proxy_only = local.l7ilb_subnets_prod
# Set explicit routes for googleapis; send everything else to NVAs
routes = {
private-googleapis = {
@@ -65,33 +78,33 @@ module "prod-spoke-vpc" {
next_hop_type = "gateway"
next_hop = "default-internet-gateway"
}
- nva-ew1-to-ew1 = {
+ nva-primary-to-primary = {
dest_range = "0.0.0.0/0"
priority = 1000
- tags = ["ew1"]
+ tags = ["primary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west1"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["primary"].forwarding_rule_address
}
- nva-ew4-to-ew4 = {
+ nva-secondary-to-secondary = {
dest_range = "0.0.0.0/0"
priority = 1000
- tags = ["ew4"]
+ tags = ["secondary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west4"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["secondary"].forwarding_rule_address
}
- nva-ew1-to-ew4 = {
+ nva-primary-to-secondary = {
dest_range = "0.0.0.0/0"
priority = 1001
- tags = ["ew1"]
+ tags = ["primary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west4"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["secondary"].forwarding_rule_address
}
- nva-ew4-to-ew1 = {
+ nva-secondary-to-primary = {
dest_range = "0.0.0.0/0"
priority = 1001
- tags = ["ew4"]
+ tags = ["secondary"]
next_hop_type = "ilb"
- next_hop = module.ilb-nva-trusted["europe-west1"].forwarding_rule_address
+ next_hop = module.ilb-nva-trusted["primary"].forwarding_rule_address
}
}
}
@@ -104,8 +117,8 @@ module "prod-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/prod"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/prod"
}
}
diff --git a/fast/stages/02-networking-nva/test-resources.tf b/fast/stages/2-networking-c-nva/test-resources.tf
similarity index 63%
rename from fast/stages/02-networking-nva/test-resources.tf
rename to fast/stages/2-networking-c-nva/test-resources.tf
index d3b6109e..14676e81 100644
--- a/fast/stages/02-networking-nva/test-resources.tf
+++ b/fast/stages/2-networking-c-nva/test-resources.tf
@@ -18,23 +18,23 @@
# # Untrusted (Landing)
-# module "test-vm-landing-untrusted-ew1-0" {
+# module "test-vm-landing-untrusted-primary-0" {
# source = "../../../modules/compute-vm"
# project_id = module.landing-project.project_id
-# zone = "europe-west1-b"
-# name = "test-vm-lnd-unt-ew1-0"
+# zone = "${var.regions.primary}-b"
+# name = "test-vm-lnd-unt-primary-0"
# network_interfaces = [{
# network = module.landing-untrusted-vpc.self_link
-# subnetwork = module.landing-untrusted-vpc.subnet_self_links["europe-west1/landing-untrusted-default-ew1"]
+# subnetwork = module.landing-untrusted-vpc.subnet_self_links["${var.regions.primary}/landing-untrusted-default-${local.region_shortnames[var.regions.primary]}"]
# }]
-# tags = ["ew1", "ssh"]
+# tags = ["primary", "ssh"]
# service_account_create = true
# boot_disk = {
# image = "projects/debian-cloud/global/images/family/debian-10"
# }
# options = {
-# spot = true
-# termination_action = "STOP"
+# spot = true
+# termination_action = "STOP"
# }
# metadata = {
# startup-script = <folder | |
| [monitoring.tf](./monitoring.tf) | Network monitoring dashboards. | | google_monitoring_dashboard |
| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object · local_file |
+| [regions.tf](./regions.tf) | Compute short names for regions. | | |
| [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-cloudnat · net-vpc · net-vpc-firewall · project | google_project_iam_binding |
| [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-cloudnat · net-vpc · net-vpc-firewall · project | google_project_iam_binding |
| [test-resources.tf](./test-resources.tf) | Temporary instances for testing | compute-vm | |
@@ -238,21 +239,22 @@ You're now ready to run `terraform init` and `apply`.
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman |
-| [organization](variables.tf#L102) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
-| [custom_roles](variables.tf#L50) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap |
-| [data_dir](variables.tf#L59) | Relative path for the folder storing configuration data for network resources. | string | | "data" | |
-| [dns](variables.tf#L65) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
-| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
-| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
-| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
-| [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | |
-| [service_accounts](variables.tf#L189) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman |
-| [vpn_onprem_configs](variables.tf#L201) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L92) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman |
+| [organization](variables.tf#L118) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L134) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | map(string) | | {…} | |
+| [custom_roles](variables.tf#L54) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap |
+| [dns](variables.tf#L63) | Onprem DNS resolvers. | map(list(string)) | | {…} | |
+| [factories_config](variables.tf#L72) | Configuration for network resource factories. | object({…}) | | {…} | |
+| [l7ilb_subnets](variables.tf#L102) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | |
+| [outputs_location](variables.tf#L128) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | |
+| [psa_ranges](variables.tf#L145) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | |
+| [regions](variables.tf#L182) | Region definitions. | object({…}) | | {…} | |
+| [router_onprem_configs](variables.tf#L192) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | |
+| [service_accounts](variables.tf#L215) | Automation service accounts in name => email format. | object({…}) | | null | 1-resman |
+| [vpn_onprem_configs](variables.tf#L227) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | |
## Outputs
diff --git a/fast/stages/02-networking-vpn/data/cidrs.yaml b/fast/stages/2-networking-d-separate-envs/data/cidrs.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/cidrs.yaml
rename to fast/stages/2-networking-d-separate-envs/data/cidrs.yaml
diff --git a/fast/stages/02-networking-vpn/data/dashboards/firewall_insights.json b/fast/stages/2-networking-d-separate-envs/data/dashboards/firewall_insights.json
similarity index 100%
rename from fast/stages/02-networking-vpn/data/dashboards/firewall_insights.json
rename to fast/stages/2-networking-d-separate-envs/data/dashboards/firewall_insights.json
diff --git a/fast/stages/02-networking-vpn/data/dashboards/vpn.json b/fast/stages/2-networking-d-separate-envs/data/dashboards/vpn.json
similarity index 100%
rename from fast/stages/02-networking-vpn/data/dashboards/vpn.json
rename to fast/stages/2-networking-d-separate-envs/data/dashboards/vpn.json
diff --git a/fast/stages/02-networking-separate-envs/data/firewall-rules/dev/rules.yaml b/fast/stages/2-networking-d-separate-envs/data/firewall-rules/dev/rules.yaml
similarity index 100%
rename from fast/stages/02-networking-separate-envs/data/firewall-rules/dev/rules.yaml
rename to fast/stages/2-networking-d-separate-envs/data/firewall-rules/dev/rules.yaml
diff --git a/fast/stages/02-networking-vpn/data/hierarchical-policy-rules.yaml b/fast/stages/2-networking-d-separate-envs/data/hierarchical-policy-rules.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/hierarchical-policy-rules.yaml
rename to fast/stages/2-networking-d-separate-envs/data/hierarchical-policy-rules.yaml
diff --git a/fast/stages/02-networking-vpn/data/subnets/dev/dev-dataplatform-ew1.yaml b/fast/stages/2-networking-d-separate-envs/data/subnets/dev/dev-dataplatform-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/subnets/dev/dev-dataplatform-ew1.yaml
rename to fast/stages/2-networking-d-separate-envs/data/subnets/dev/dev-dataplatform-ew1.yaml
diff --git a/fast/stages/02-networking-vpn/data/subnets/dev/dev-default-ew1.yaml b/fast/stages/2-networking-d-separate-envs/data/subnets/dev/dev-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/subnets/dev/dev-default-ew1.yaml
rename to fast/stages/2-networking-d-separate-envs/data/subnets/dev/dev-default-ew1.yaml
diff --git a/fast/stages/02-networking-vpn/data/subnets/prod/prod-default-ew1.yaml b/fast/stages/2-networking-d-separate-envs/data/subnets/prod/prod-default-ew1.yaml
similarity index 100%
rename from fast/stages/02-networking-vpn/data/subnets/prod/prod-default-ew1.yaml
rename to fast/stages/2-networking-d-separate-envs/data/subnets/prod/prod-default-ew1.yaml
diff --git a/fast/stages/02-networking-separate-envs/diagram.png b/fast/stages/2-networking-d-separate-envs/diagram.png
similarity index 100%
rename from fast/stages/02-networking-separate-envs/diagram.png
rename to fast/stages/2-networking-d-separate-envs/diagram.png
diff --git a/fast/stages/02-networking-separate-envs/diagram.svg b/fast/stages/2-networking-d-separate-envs/diagram.svg
similarity index 100%
rename from fast/stages/02-networking-separate-envs/diagram.svg
rename to fast/stages/2-networking-d-separate-envs/diagram.svg
diff --git a/fast/stages/02-networking-separate-envs/dns-dev.tf b/fast/stages/2-networking-d-separate-envs/dns-dev.tf
similarity index 100%
rename from fast/stages/02-networking-separate-envs/dns-dev.tf
rename to fast/stages/2-networking-d-separate-envs/dns-dev.tf
diff --git a/fast/stages/02-networking-separate-envs/dns-prod.tf b/fast/stages/2-networking-d-separate-envs/dns-prod.tf
similarity index 100%
rename from fast/stages/02-networking-separate-envs/dns-prod.tf
rename to fast/stages/2-networking-d-separate-envs/dns-prod.tf
diff --git a/fast/stages/02-networking-separate-envs/main.tf b/fast/stages/2-networking-d-separate-envs/main.tf
similarity index 65%
rename from fast/stages/02-networking-separate-envs/main.tf
rename to fast/stages/2-networking-d-separate-envs/main.tf
index 0b901e33..dda1c252 100644
--- a/fast/stages/02-networking-separate-envs/main.tf
+++ b/fast/stages/2-networking-d-separate-envs/main.tf
@@ -18,17 +18,25 @@
locals {
custom_roles = coalesce(var.custom_roles, {})
- l7ilb_subnets = {
- for env, v in var.l7ilb_subnets : env => [
+ _l7ilb_subnets = {
+ for k, v in var.l7ilb_subnets : k => [
for s in v : merge(s, {
active = true
- name = "${env}-l7ilb-${s.region}"
+ region = lookup(var.regions, s.region, s.region)
})]
}
- region_trigram = {
- europe-west1 = "ew1"
- europe-west3 = "ew3"
+ l7ilb_subnets = {
+ for k, v in local._l7ilb_subnets : k => [
+ for s in v : merge(s, {
+ name = "${k}-l7ilb-${local.region_shortnames[s.region]}"
+ })]
}
+ # combine all regions from variables and subnets
+ regions = distinct(concat(
+ values(var.regions),
+ values(module.dev-spoke-vpc.subnet_regions),
+ values(module.prod-spoke-vpc.subnet_regions),
+ ))
stage3_sas_delegated_grants = [
"roles/composer.sharedVpcAgent",
"roles/compute.networkUser",
@@ -47,12 +55,12 @@ module "folder" {
folder_create = var.folder_ids.networking == null
id = var.folder_ids.networking
firewall_policy_factory = {
- cidr_file = "${var.data_dir}/cidrs.yaml"
- policy_name = null
- rules_file = "${var.data_dir}/hierarchical-policy-rules.yaml"
+ cidr_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ policy_name = var.factories_config.firewall_policy_name
+ rules_file = "${var.factories_config.data_dir}/hierarchical-policy-rules.yaml"
}
firewall_policy_association = {
- factory-policy = "factory"
+ factory-policy = var.factories_config.firewall_policy_name
}
}
diff --git a/fast/stages/02-networking-separate-envs/monitoring.tf b/fast/stages/2-networking-d-separate-envs/monitoring.tf
similarity index 94%
rename from fast/stages/02-networking-separate-envs/monitoring.tf
rename to fast/stages/2-networking-d-separate-envs/monitoring.tf
index 463c6a08..01ed0c47 100644
--- a/fast/stages/02-networking-separate-envs/monitoring.tf
+++ b/fast/stages/2-networking-d-separate-envs/monitoring.tf
@@ -17,7 +17,7 @@
# tfdoc:file:description Network monitoring dashboards.
locals {
- dashboard_path = "${var.data_dir}/dashboards"
+ dashboard_path = "${var.factories_config.data_dir}/dashboards"
dashboard_files = fileset(local.dashboard_path, "*.json")
dashboards = {
for filename in local.dashboard_files :
diff --git a/fast/stages/02-networking-separate-envs/outputs.tf b/fast/stages/2-networking-d-separate-envs/outputs.tf
similarity index 90%
rename from fast/stages/02-networking-separate-envs/outputs.tf
rename to fast/stages/2-networking-d-separate-envs/outputs.tf
index d06d499d..59d70db6 100644
--- a/fast/stages/02-networking-separate-envs/outputs.tf
+++ b/fast/stages/2-networking-d-separate-envs/outputs.tf
@@ -44,13 +44,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/02-networking.auto.tfvars.json"
+ filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/02-networking.auto.tfvars.json"
+ name = "tfvars/2-networking.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
@@ -90,12 +90,12 @@ output "tfvars" {
output "vpn_gateway_endpoints" {
description = "External IP Addresses for the GCP VPN gateways."
value = local.enable_onprem_vpn == false ? null : {
- dev-onprem-ew1 = {
- for v in module.dev-to-onprem-ew1-vpn[0].gateway.vpn_interfaces :
+ dev-onprem-primary = {
+ for v in module.dev-to-onprem-primary-vpn[0].gateway.vpn_interfaces :
v.id => v.ip_address
}
- prod-onprem-ew1 = {
- for v in module.prod-to-onprem-ew1-vpn[0].gateway.vpn_interfaces :
+ prod-onprem-primary = {
+ for v in module.prod-to-onprem-primary-vpn[0].gateway.vpn_interfaces :
v.id => v.ip_address
}
}
diff --git a/fast/stages/2-networking-d-separate-envs/regions.tf b/fast/stages/2-networking-d-separate-envs/regions.tf
new file mode 100644
index 00000000..53514afa
--- /dev/null
+++ b/fast/stages/2-networking-d-separate-envs/regions.tf
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# tfdoc:file:description Compute short names for regions.
+
+locals {
+ # only map when the first character would not work
+ _region_cardinal = {
+ southeast = "se"
+ }
+ # only map when the first character would not work
+ _region_geo = {
+ australia = "o"
+ }
+ # split in [geo, cardinal, number] tokens
+ _region_tokens = {
+ for v in local.regions : v => regexall("(?:[a-z]+)|(?:[0-9]+)", v)
+ }
+ region_shortnames = {
+ for k, v in local._region_tokens : k => join("", [
+ # first token via geo alias map or first character
+ lookup(local._region_geo, v.0, substr(v.0, 0, 1)),
+ # first token via cardinal alias map or first character
+ lookup(local._region_cardinal, v.1, substr(v.1, 0, 1)),
+ # region number as is
+ v.2
+ ])
+ }
+}
diff --git a/fast/stages/02-networking-separate-envs/spoke-dev.tf b/fast/stages/2-networking-d-separate-envs/spoke-dev.tf
similarity index 92%
rename from fast/stages/02-networking-separate-envs/spoke-dev.tf
rename to fast/stages/2-networking-d-separate-envs/spoke-dev.tf
index ca7d8d46..ecd0b073 100644
--- a/fast/stages/02-networking-separate-envs/spoke-dev.tf
+++ b/fast/stages/2-networking-d-separate-envs/spoke-dev.tf
@@ -47,7 +47,7 @@ module "dev-spoke-vpc" {
project_id = module.dev-spoke-project.project_id
name = "dev-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/dev"
+ data_folder = "${var.factories_config.data_dir}/subnets/dev"
psa_config = try(var.psa_ranges.dev, null)
subnets_proxy_only = local.l7ilb_subnets.dev
# set explicit routes for googleapis in case the default route is deleted
@@ -73,8 +73,8 @@ module "dev-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/dev"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/dev"
}
}
@@ -83,7 +83,7 @@ module "dev-spoke-cloudnat" {
source = "../../../modules/net-cloudnat"
project_id = module.dev-spoke-project.project_id
region = each.value
- name = "dev-nat-${local.region_trigram[each.value]}"
+ name = "dev-nat-${local.region_shortnames[each.value]}"
router_create = true
router_network = module.dev-spoke-vpc.name
router_asn = 4200001024
diff --git a/fast/stages/02-networking-separate-envs/spoke-prod.tf b/fast/stages/2-networking-d-separate-envs/spoke-prod.tf
similarity index 92%
rename from fast/stages/02-networking-separate-envs/spoke-prod.tf
rename to fast/stages/2-networking-d-separate-envs/spoke-prod.tf
index eba530a6..2e95a09e 100644
--- a/fast/stages/02-networking-separate-envs/spoke-prod.tf
+++ b/fast/stages/2-networking-d-separate-envs/spoke-prod.tf
@@ -47,7 +47,7 @@ module "prod-spoke-vpc" {
project_id = module.prod-spoke-project.project_id
name = "prod-spoke-0"
mtu = 1500
- data_folder = "${var.data_dir}/subnets/prod"
+ data_folder = "${var.factories_config.data_dir}/subnets/prod"
psa_config = try(var.psa_ranges.prod, null)
subnets_proxy_only = local.l7ilb_subnets.prod
# set explicit routes for googleapis in case the default route is deleted
@@ -73,8 +73,8 @@ module "prod-spoke-firewall" {
disabled = true
}
factories_config = {
- cidr_tpl_file = "${var.data_dir}/cidrs.yaml"
- rules_folder = "${var.data_dir}/firewall-rules/prod"
+ cidr_tpl_file = "${var.factories_config.data_dir}/cidrs.yaml"
+ rules_folder = "${var.factories_config.data_dir}/firewall-rules/prod"
}
}
@@ -83,7 +83,7 @@ module "prod-spoke-cloudnat" {
source = "../../../modules/net-cloudnat"
project_id = module.prod-spoke-project.project_id
region = each.value
- name = "prod-nat-${local.region_trigram[each.value]}"
+ name = "prod-nat-${local.region_shortnames[each.value]}"
router_create = true
router_network = module.prod-spoke-vpc.name
router_asn = 4200001024
diff --git a/fast/stages/02-networking-separate-envs/test-resources.tf b/fast/stages/2-networking-d-separate-envs/test-resources.tf
similarity index 86%
rename from fast/stages/02-networking-separate-envs/test-resources.tf
rename to fast/stages/2-networking-d-separate-envs/test-resources.tf
index ae3ba90a..55c42c25 100644
--- a/fast/stages/02-networking-separate-envs/test-resources.tf
+++ b/fast/stages/2-networking-d-separate-envs/test-resources.tf
@@ -19,12 +19,12 @@
# module "test-vm-dev-0" {
# source = "../../../modules/compute-vm"
# project_id = module.dev-spoke-project.project_id
-# zone = "europe-west1-b"
+# zone = "${var.regions.primary}-b"
# name = "test-vm-0"
# network_interfaces = [{
# network = module.dev-spoke-vpc.self_link
# # change the subnet name to match the values you are actually using
-# subnetwork = module.dev-spoke-vpc.subnet_self_links["europe-west1/dev-default-ew1"]
+# subnetwork = module.dev-spoke-vpc.subnet_self_links["${var.regions.primary}/dev-default-${local.region_shortnames[var.regions.primary]}"]
# alias_ips = {}
# nat = false
# addresses = null
@@ -52,12 +52,12 @@
# module "test-vm-prod-0" {
# source = "../../../modules/compute-vm"
# project_id = module.prod-spoke-project.project_id
-# zone = "europe-west1-b"
+# zone = "${var.regions.primary}-b"
# name = "test-vm-0"
# network_interfaces = [{
# network = module.prod-spoke-vpc.self_link
# # change the subnet name to match the values you are actually using
-# subnetwork = module.prod-spoke-vpc.subnet_self_links["europe-west1/prod-default-ew1"]
+# subnetwork = module.prod-spoke-vpc.subnet_self_links["${var.regions.primary}/prod-default-${local.region_shortnames[var.regions.primary]}"]
# alias_ips = {}
# nat = false
# addresses = null
diff --git a/fast/stages/02-networking-separate-envs/variables.tf b/fast/stages/2-networking-d-separate-envs/variables.tf
similarity index 82%
rename from fast/stages/02-networking-separate-envs/variables.tf
rename to fast/stages/2-networking-d-separate-envs/variables.tf
index 019d0b2e..03d56513 100644
--- a/fast/stages/02-networking-separate-envs/variables.tf
+++ b/fast/stages/2-networking-d-separate-envs/variables.tf
@@ -15,7 +15,7 @@
*/
variable "automation" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Automation resources created by the bootstrap stage."
type = object({
outputs_bucket = string
@@ -23,12 +23,16 @@ variable "automation" {
}
variable "billing_account" {
- # tfdoc:variable:source 00-bootstrap
- description = "Billing account id and organization id ('nnnnnnnn' or null)."
+ # tfdoc:variable:source 0-bootstrap
+ description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false."
type = object({
- id = string
- organization_id = number
+ id = string
+ is_org_level = optional(bool, true)
})
+ validation {
+ condition = var.billing_account.is_org_level != null
+ error_message = "Invalid `null` value for `billing_account.is_org_level`."
+ }
}
variable "custom_adv" {
@@ -48,7 +52,7 @@ variable "custom_adv" {
}
variable "custom_roles" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Custom roles defined at the org level, in key => id format."
type = object({
service_project_network_admin = string
@@ -56,12 +60,6 @@ variable "custom_roles" {
default = null
}
-variable "data_dir" {
- description = "Relative path for the folder storing configuration data for network resources."
- type = string
- default = "data"
-}
-
variable "dns" {
description = "Onprem DNS resolvers."
type = map(list(string))
@@ -71,8 +69,28 @@ variable "dns" {
}
}
+variable "factories_config" {
+ description = "Configuration for network resource factories."
+ type = object({
+ data_dir = optional(string, "data")
+ firewall_policy_name = optional(string, "factory")
+ })
+ default = {
+ data_dir = "data"
+ }
+ nullable = false
+ validation {
+ condition = var.factories_config.data_dir != null
+ error_message = "Data folder needs to be non-null."
+ }
+ validation {
+ condition = var.factories_config.firewall_policy_name != null
+ error_message = "Firewall policy name needs to be non-null."
+ }
+}
+
variable "folder_ids" {
- # tfdoc:variable:source 01-resman
+ # tfdoc:variable:source 1-resman
description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
type = object({
networking = string
@@ -90,17 +108,15 @@ variable "l7ilb_subnets" {
default = {
prod = [
{ ip_cidr_range = "10.128.92.0/24", region = "europe-west1" },
- { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" }
]
dev = [
{ ip_cidr_range = "10.128.60.0/24", region = "europe-west1" },
- { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" }
]
}
}
variable "organization" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Organization details."
type = object({
domain = string
@@ -116,7 +132,7 @@ variable "outputs_location" {
}
variable "prefix" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Prefix used for resources that need unique names. Use 9 characters or less."
type = string
@@ -163,6 +179,16 @@ variable "psa_ranges" {
# }
}
+variable "regions" {
+ description = "Region definitions."
+ type = object({
+ primary = string
+ })
+ default = {
+ primary = "europe-west1"
+ }
+}
+
variable "router_onprem_configs" {
description = "Configurations for routers used for onprem connectivity."
type = map(object({
@@ -173,12 +199,12 @@ variable "router_onprem_configs" {
asn = number
}))
default = {
- prod-ew1 = {
+ prod-primary = {
asn = "65533"
adv = null
# adv = { default = false, custom = [] }
}
- dev-ew1 = {
+ dev-primary = {
asn = "65534"
adv = null
# adv = { default = false, custom = [] }
@@ -187,7 +213,7 @@ variable "router_onprem_configs" {
}
variable "service_accounts" {
- # tfdoc:variable:source 01-resman
+ # tfdoc:variable:source 1-resman
description = "Automation service accounts in name => email format."
type = object({
data-platform-dev = string
@@ -218,7 +244,7 @@ variable "vpn_onprem_configs" {
}))
}))
default = {
- dev-ew1 = {
+ dev-primary = {
adv = {
default = false
custom = [
@@ -247,7 +273,7 @@ variable "vpn_onprem_configs" {
}
]
}
- prod-ew1 = {
+ prod-primary = {
adv = {
default = false
custom = [
diff --git a/fast/stages/02-networking-separate-envs/vpn-onprem-dev.tf b/fast/stages/2-networking-d-separate-envs/vpn-onprem-dev.tf
similarity index 78%
rename from fast/stages/02-networking-separate-envs/vpn-onprem-dev.tf
rename to fast/stages/2-networking-d-separate-envs/vpn-onprem-dev.tf
index 1b0ff117..e95cd14f 100644
--- a/fast/stages/02-networking-separate-envs/vpn-onprem-dev.tf
+++ b/fast/stages/2-networking-d-separate-envs/vpn-onprem-dev.tf
@@ -32,28 +32,33 @@ locals {
}
}
-module "dev-to-onprem-ew1-vpn" {
+moved {
+ from = module.dev-to-onprem-ew1-vpn
+ to = module.dev-to-onprem-primary-vpn
+}
+
+module "dev-to-onprem-primary-vpn" {
count = local.enable_onprem_vpn ? 1 : 0
source = "../../../modules/net-vpn-ha"
project_id = module.dev-spoke-project.project_id
network = module.dev-spoke-vpc.self_link
- region = "europe-west1"
- name = "vpn-to-onprem-ew1"
+ region = var.regions.primary
+ name = "vpn-to-onprem-${local.region_shortnames[var.regions.primary]}"
router_config = {
- name = "dev-onprem-vpn-ew1"
- asn = var.router_onprem_configs.dev-ew1.asn
+ name = "dev-onprem-vpn-${local.region_shortnames[var.regions.primary]}"
+ asn = var.router_onprem_configs.dev-primary.asn
}
peer_gateway = {
- external = var.vpn_onprem_configs.dev-ew1.peer_external_gateway
+ external = var.vpn_onprem_configs.dev-primary.peer_external_gateway
}
tunnels = {
- for t in var.vpn_onprem_configs.dev-ew1.tunnels :
+ for t in var.vpn_onprem_configs.dev-primary.tunnels :
"remote-${t.vpn_gateway_interface}-${t.peer_external_gateway_interface}" => {
bgp_peer = {
address = cidrhost(t.session_range, 1)
asn = t.peer_asn
}
- bgp_peer_options = local.bgp_peer_options_onprem.dev-ew1
+ bgp_peer_options = local.bgp_peer_options_onprem.dev-primary
bgp_session_range = "${cidrhost(t.session_range, 2)}/30"
peer_external_gateway_interface = t.peer_external_gateway_interface
shared_secret = t.secret
diff --git a/fast/stages/02-networking-separate-envs/vpn-onprem-prod.tf b/fast/stages/2-networking-d-separate-envs/vpn-onprem-prod.tf
similarity index 73%
rename from fast/stages/02-networking-separate-envs/vpn-onprem-prod.tf
rename to fast/stages/2-networking-d-separate-envs/vpn-onprem-prod.tf
index d4b2af24..0793e274 100644
--- a/fast/stages/02-networking-separate-envs/vpn-onprem-prod.tf
+++ b/fast/stages/2-networking-d-separate-envs/vpn-onprem-prod.tf
@@ -16,28 +16,33 @@
# tfdoc:file:description VPN between prod and onprem.
-module "prod-to-onprem-ew1-vpn" {
+moved {
+ from = module.prod-to-onprem-ew1-vpn
+ to = module.prod-to-onprem-primary-vpn
+}
+
+module "prod-to-onprem-primary-vpn" {
count = local.enable_onprem_vpn ? 1 : 0
source = "../../../modules/net-vpn-ha"
project_id = module.prod-spoke-project.project_id
network = module.prod-spoke-vpc.self_link
- region = "europe-west1"
- name = "vpn-to-onprem-ew1"
+ region = var.regions.primary
+ name = "vpn-to-onprem-${local.region_shortnames[var.regions.primary]}"
router_config = {
- name = "prod-onprem-vpn-ew1"
- asn = var.router_onprem_configs.prod-ew1.asn
+ name = "prod-onprem-vpn-${local.region_shortnames[var.regions.primary]}"
+ asn = var.router_onprem_configs.prod-primary.asn
}
peer_gateway = {
- external = var.vpn_onprem_configs.prod-ew1.peer_external_gateway
+ external = var.vpn_onprem_configs.prod-primary.peer_external_gateway
}
tunnels = {
- for t in var.vpn_onprem_configs.prod-ew1.tunnels :
+ for t in var.vpn_onprem_configs.prod-primary.tunnels :
"remote-${t.vpn_gateway_interface}-${t.peer_external_gateway_interface}" => {
bgp_peer = {
address = cidrhost(t.session_range, 1)
asn = t.peer_asn
}
- bgp_peer_options = local.bgp_peer_options_onprem.prod-ew1
+ bgp_peer_options = local.bgp_peer_options_onprem.prod-primary
bgp_session_range = "${cidrhost(t.session_range, 2)}/30"
peer_external_gateway_interface = t.peer_external_gateway_interface
shared_secret = t.secret
diff --git a/fast/stages/02-security/IAM.md b/fast/stages/2-security/IAM.md
similarity index 100%
rename from fast/stages/02-security/IAM.md
rename to fast/stages/2-security/IAM.md
diff --git a/fast/stages/02-security/README.md b/fast/stages/2-security/README.md
similarity index 92%
rename from fast/stages/02-security/README.md
rename to fast/stages/2-security/README.md
index 024ababb..a609cd81 100644
--- a/fast/stages/02-security/README.md
+++ b/fast/stages/2-security/README.md
@@ -42,7 +42,7 @@ Some care needs to be taken with project membership in perimeters, which can onl
## How to run this stage
-This stage is meant to be executed after the [resource management](../01-resman) stage has run, as it leverages the folder and automation resources created there. The relevant user groups must also exist, but that's one of the requirements for the previous stages too, so if you ran those successfully, you're good to go.
+This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the folder and automation resources created there. The relevant user groups must also exist, but that's one of the requirements for the previous stages too, so if you ran those successfully, you're good to go.
It's possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the bootstrap stage for the required roles.
@@ -64,7 +64,7 @@ ln -s ~/fast-config/providers/02-security-providers.tf .
If you have not configured `outputs_location` in resource management, you can derive the providers file from that stage's outputs:
```bash
-cd ../01-resman
+cd ../1-resman
terraform output -json providers | jq -r '.["02-security"]' \
> ../02-security/providers.tf
```
@@ -85,7 +85,7 @@ If you configured a valid path for `outputs_location` in the previous stages, si
ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
# also copy the tfvars file used for the bootstrap stage
-cp ../00-bootstrap/terraform.tfvars .
+cp ../0-bootstrap/terraform.tfvars .
```
A second set of optional variables is specific to this stage. If you need to customize them add them to the file copied from bootstrap.
@@ -262,20 +262,20 @@ Some references that might be useful in setting up this stage:
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [folder_ids](variables.tf#L34) | Folder name => id mappings, the 'security' folder name must exist. | object({…}) | ✓ | | 01-resman |
-| [organization](variables.tf#L80) | Organization details. | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L96) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [service_accounts](variables.tf#L107) | Automation service accounts that can assign the encrypt/decrypt roles on keys. | object({…}) | ✓ | | 01-resman |
-| [groups](variables.tf#L42) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap |
-| [kms_defaults](variables.tf#L57) | Defaults used for KMS keys. | object({…}) | | {…} | |
-| [kms_keys](variables.tf#L69) | KMS keys to create, keyed by name. Null attributes will be interpolated with defaults. | map(object({…})) | | {} | |
-| [outputs_location](variables.tf#L90) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
-| [vpc_sc_access_levels](variables.tf#L118) | VPC SC access level definitions. | map(object({…})) | | {} | |
-| [vpc_sc_egress_policies](variables.tf#L147) | VPC SC egress policy defnitions. | map(object({…})) | | {} | |
-| [vpc_sc_ingress_policies](variables.tf#L167) | VPC SC ingress policy defnitions. | map(object({…})) | | {} | |
-| [vpc_sc_perimeters](variables.tf#L188) | VPC SC regular perimeter definitions. | object({…}) | | {} | |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L38) | Folder name => id mappings, the 'security' folder name must exist. | object({…}) | ✓ | | 1-resman |
+| [organization](variables.tf#L84) | Organization details. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L100) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [service_accounts](variables.tf#L111) | Automation service accounts that can assign the encrypt/decrypt roles on keys. | object({…}) | ✓ | | 1-resman |
+| [groups](variables.tf#L46) | Group names to grant organization-level permissions. | map(string) | | {…} | 0-bootstrap |
+| [kms_defaults](variables.tf#L61) | Defaults used for KMS keys. | object({…}) | | {…} | |
+| [kms_keys](variables.tf#L73) | KMS keys to create, keyed by name. Null attributes will be interpolated with defaults. | map(object({…})) | | {} | |
+| [outputs_location](variables.tf#L94) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
+| [vpc_sc_access_levels](variables.tf#L122) | VPC SC access level definitions. | map(object({…})) | | {} | |
+| [vpc_sc_egress_policies](variables.tf#L151) | VPC SC egress policy defnitions. | map(object({…})) | | {} | |
+| [vpc_sc_ingress_policies](variables.tf#L171) | VPC SC ingress policy defnitions. | map(object({…})) | | {} | |
+| [vpc_sc_perimeters](variables.tf#L192) | VPC SC regular perimeter definitions. | object({…}) | | {} | |
## Outputs
diff --git a/fast/stages/02-security/core-dev.tf b/fast/stages/2-security/core-dev.tf
similarity index 100%
rename from fast/stages/02-security/core-dev.tf
rename to fast/stages/2-security/core-dev.tf
diff --git a/fast/stages/02-security/core-prod.tf b/fast/stages/2-security/core-prod.tf
similarity index 100%
rename from fast/stages/02-security/core-prod.tf
rename to fast/stages/2-security/core-prod.tf
diff --git a/fast/stages/02-security/diagram.png b/fast/stages/2-security/diagram.png
similarity index 100%
rename from fast/stages/02-security/diagram.png
rename to fast/stages/2-security/diagram.png
diff --git a/fast/stages/02-security/diagram.svg b/fast/stages/2-security/diagram.svg
similarity index 100%
rename from fast/stages/02-security/diagram.svg
rename to fast/stages/2-security/diagram.svg
diff --git a/fast/stages/02-security/main.tf b/fast/stages/2-security/main.tf
similarity index 100%
rename from fast/stages/02-security/main.tf
rename to fast/stages/2-security/main.tf
diff --git a/fast/stages/02-security/outputs.tf b/fast/stages/2-security/outputs.tf
similarity index 96%
rename from fast/stages/02-security/outputs.tf
rename to fast/stages/2-security/outputs.tf
index b7e42e49..ff0c13ed 100644
--- a/fast/stages/02-security/outputs.tf
+++ b/fast/stages/2-security/outputs.tf
@@ -44,13 +44,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${pathexpand(var.outputs_location)}/tfvars/02-security.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/tfvars/2-security.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/02-security.auto.tfvars.json"
+ name = "tfvars/2-security.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/02-security/variables.tf b/fast/stages/2-security/variables.tf
similarity index 91%
rename from fast/stages/02-security/variables.tf
rename to fast/stages/2-security/variables.tf
index 349589c9..e14d6376 100644
--- a/fast/stages/02-security/variables.tf
+++ b/fast/stages/2-security/variables.tf
@@ -15,7 +15,7 @@
*/
variable "automation" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Automation resources created by the bootstrap stage."
type = object({
outputs_bucket = string
@@ -23,16 +23,20 @@ variable "automation" {
}
variable "billing_account" {
- # tfdoc:variable:source 00-bootstrap
- description = "Billing account id and organization id ('nnnnnnnn' or null)."
+ # tfdoc:variable:source 0-bootstrap
+ description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false."
type = object({
- id = string
- organization_id = number
+ id = string
+ is_org_level = optional(bool, true)
})
+ validation {
+ condition = var.billing_account.is_org_level != null
+ error_message = "Invalid `null` value for `billing_account.is_org_level`."
+ }
}
variable "folder_ids" {
- # tfdoc:variable:source 01-resman
+ # tfdoc:variable:source 1-resman
description = "Folder name => id mappings, the 'security' folder name must exist."
type = object({
security = string
@@ -40,7 +44,7 @@ variable "folder_ids" {
}
variable "groups" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Group names to grant organization-level permissions."
type = map(string)
# https://cloud.google.com/docs/enterprise/setup-checklist
@@ -78,7 +82,7 @@ variable "kms_keys" {
}
variable "organization" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Organization details."
type = object({
domain = string
@@ -94,7 +98,7 @@ variable "outputs_location" {
}
variable "prefix" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Prefix used for resources that need unique names. Use 9 characters or less."
type = string
@@ -105,7 +109,7 @@ variable "prefix" {
}
variable "service_accounts" {
- # tfdoc:variable:source 01-resman
+ # tfdoc:variable:source 1-resman
description = "Automation service accounts that can assign the encrypt/decrypt roles on keys."
type = object({
data-platform-dev = string
diff --git a/fast/stages/02-security/vpc-sc-restricted-services.yaml b/fast/stages/2-security/vpc-sc-restricted-services.yaml
similarity index 100%
rename from fast/stages/02-security/vpc-sc-restricted-services.yaml
rename to fast/stages/2-security/vpc-sc-restricted-services.yaml
diff --git a/fast/stages/02-security/vpc-sc.tf b/fast/stages/2-security/vpc-sc.tf
similarity index 100%
rename from fast/stages/02-security/vpc-sc.tf
rename to fast/stages/2-security/vpc-sc.tf
diff --git a/fast/stages/03-data-platform/README.md b/fast/stages/3-data-platform/README.md
similarity index 100%
rename from fast/stages/03-data-platform/README.md
rename to fast/stages/3-data-platform/README.md
diff --git a/fast/stages/03-data-platform/dev/IAM.md b/fast/stages/3-data-platform/dev/IAM.md
similarity index 100%
rename from fast/stages/03-data-platform/dev/IAM.md
rename to fast/stages/3-data-platform/dev/IAM.md
diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md
similarity index 82%
rename from fast/stages/03-data-platform/dev/README.md
rename to fast/stages/3-data-platform/dev/README.md
index 35f9bf1c..615dbde8 100644
--- a/fast/stages/03-data-platform/dev/README.md
+++ b/fast/stages/3-data-platform/dev/README.md
@@ -42,11 +42,11 @@ As per our GCP best practices the Data Platform relies on user groups to assign
### Network
-A Shared VPC is used here, either from one of the FAST networking stages (e.g. [hub and spoke via VPN](../../02-networking-vpn)) or from an external source.
+A Shared VPC is used here, either from one of the FAST networking stages (e.g. [hub and spoke via VPN](../../2-networking-b-vpn)) or from an external source.
### Encryption
-Cloud KMS crypto keys can be configured wither from the [FAST security stage](../../02-security) or from an external source. This step is optional and depends on customer policies and security best practices.
+Cloud KMS crypto keys can be configured wither from the [FAST security stage](../../2-security) or from an external source. This step is optional and depends on customer policies and security best practices.
To configure the use of Cloud KMS on resources, you have to specify the key id on the `service_encryption_keys` variable. Key locations should match resource locations.
@@ -55,19 +55,20 @@ To configure the use of Cloud KMS on resources, you have to specify the key id o
[Data Catalog](https://cloud.google.com/data-catalog) helps you to document your data entry at scale. Data Catalog relies on [tags](https://cloud.google.com/data-catalog/docs/tags-and-tag-templates#tags) and [tag template](https://cloud.google.com/data-catalog/docs/tags-and-tag-templates#tag-templates) to manage metadata for all data entries in a unified and centralized service. To implement [column-level security](https://cloud.google.com/bigquery/docs/column-level-security-intro) on BigQuery, we suggest to use `Tags` and `Tag templates`.
The default configuration will implement 3 tags:
- - `3_Confidential`: policy tag for columns that include very sensitive information, such as credit card numbers.
- - `2_Private`: policy tag for columns that include sensitive personal identifiable information (PII) information, such as a person's first name.
- - `1_Sensitive`: policy tag for columns that include data that cannot be made public, such as the credit limit.
+
+- `3_Confidential`: policy tag for columns that include very sensitive information, such as credit card numbers.
+- `2_Private`: policy tag for columns that include sensitive personal identifiable information (PII) information, such as a person's first name.
+- `1_Sensitive`: policy tag for columns that include data that cannot be made public, such as the credit limit.
Anything that is not tagged is available to all users who have access to the data warehouse.
-You can configure your tags and roles associated by configuring the `data_catalog_tags` variable. We suggest useing the "[Best practices for using policy tags in BigQuery](https://cloud.google.com/bigquery/docs/best-practices-policy-tags)" article as a guide to designing your tags structure and access pattern. By default, no groups has access to tagged data.
+You can configure your tags and roles associated by configuring the `data_catalog_tags` variable. We suggest useing the "[Best practices for using policy tags in BigQuery](https://cloud.google.com/bigquery/docs/best-practices-policy-tags)" article as a guide to designing your tags structure and access pattern. By default, no groups has access to tagged data.
### VPC-SC
-As is often the case in real-world configurations, [VPC-SC](https://cloud.google.com/vpc-service-controls) is needed to mitigate data exfiltration. VPC-SC can be configured from the [FAST security stage](../../02-security). This step is optional, but highly recomended, and depends on customer policies and security best practices.
+As is often the case in real-world configurations, [VPC-SC](https://cloud.google.com/vpc-service-controls) is needed to mitigate data exfiltration. VPC-SC can be configured from the [FAST security stage](../../2-security). This step is optional, but highly recomended, and depends on customer policies and security best practices.
-To configure the use of VPC-SC on the data platform, you have to specify the data platform project numbers on the `vpc_sc_perimeter_projects.dev` variable on [FAST security stage](../../02-security#perimeter-resources).
+To configure the use of VPC-SC on the data platform, you have to specify the data platform project numbers on the `vpc_sc_perimeter_projects.dev` variable on [FAST security stage](../../2-security#perimeter-resources).
In the case your Data Warehouse need to handle confidential data and you have the requirement to separate them deeply from other data and IAM is not enough, the suggested configuration is to keep the confidential project in a separate VPC-SC perimeter with the adequate ingress/egress rules needed for the load and tranformation service account. Below you can find an high level diagram describing the configuration.
@@ -77,7 +78,7 @@ In the case your Data Warehouse need to handle confidential data and you have th
## How to run this stage
-This stage can be run in isolation by prviding the necessary variables, but it's really meant to be used as part of the FAST flow after the "foundational stages" ([`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), [`02-networking`](../../02-networking-vpn) and [`02-security`](../../02-security)).
+This stage can be run in isolation by prviding the necessary variables, but it's really meant to be used as part of the FAST flow after the "foundational stages" ([`00-bootstrap`](../../0-bootstrap), [`01-resman`](../../1-resman), [`02-networking`](../../2-networking-b-vpn) and [`02-security`](../../2-security)).
When running in isolation, the following roles are needed on the principal used to apply Terraform:
@@ -111,9 +112,9 @@ ln -s ~/fast-config/providers/03-data-platform-dev-providers.tf .
If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
```bash
-cd ../../01-resman
+cd ../../1-resman
terraform output -json providers | jq -r '.["03-data-platform-dev"]' \
- > ../03-data-platform/dev/providers.tf
+ > ../3-data-platform/dev/providers.tf
```
### Variable configuration
@@ -133,7 +134,7 @@ ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json .
# also copy the tfvars file used for the bootstrap stage
-cp ../../00-bootstrap/terraform.tfvars .
+cp ../../0-bootstrap/terraform.tfvars .
```
If you're not using FAST or its output files, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning.
@@ -147,7 +148,7 @@ terraform apply
## Demo pipeline
-The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `landing` area to the `DataWarehouse Confidential` dataset suing different features.
+The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `landing` area to the `DataWarehouse Confidential` dataset suing different features.
You can find examples in the `[demo](../../../../blueprints/data-solutions/data-platform-foundations/demo)` folder.
@@ -166,24 +167,24 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data-
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-globals |
-| [folder_ids](variables.tf#L98) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 01-resman |
-| [host_project_ids](variables.tf#L116) | Shared VPC project ids. | object({…}) | ✓ | | 02-networking |
-| [organization](variables.tf#L146) | Organization details. | object({…}) | ✓ | | 00-globals |
-| [prefix](variables.tf#L162) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals |
-| [composer_config](variables.tf#L34) | Cloud Composer configuration options. | object({…}) | | {…} | |
-| [data_catalog_tags](variables.tf#L81) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | |
-| [data_force_destroy](variables.tf#L92) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | |
-| [groups](variables.tf#L106) | Groups. | map(string) | | {…} | |
-| [location](variables.tf#L124) | Location used for multi-regional resources. | string | | "eu" | |
-| [network_config_composer](variables.tf#L130) | Network configurations to use for Composer. | object({…}) | | {…} | |
-| [outputs_location](variables.tf#L156) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
-| [project_services](variables.tf#L168) | List of core services enabled on all projects. | list(string) | | […] | |
-| [region](variables.tf#L179) | Region used for regional resources. | string | | "europe-west1" | |
-| [service_encryption_keys](variables.tf#L185) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | |
-| [subnet_self_links](variables.tf#L197) | Shared VPC subnet self links. | object({…}) | | null | 02-networking |
-| [vpc_self_links](variables.tf#L206) | Shared VPC self links. | object({…}) | | null | 02-networking |
+| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L102) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman |
+| [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking |
+| [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals |
+| [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals |
+| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | |
+| [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | |
+| [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | |
+| [groups](variables.tf#L110) | Groups. | map(string) | | {…} | |
+| [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | |
+| [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | |
+| [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
+| [project_services](variables.tf#L172) | List of core services enabled on all projects. | list(string) | | […] | |
+| [region](variables.tf#L183) | Region used for regional resources. | string | | "europe-west1" | |
+| [service_encryption_keys](variables.tf#L189) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | |
+| [subnet_self_links](variables.tf#L201) | Shared VPC subnet self links. | object({…}) | | null | 2-networking |
+| [vpc_self_links](variables.tf#L210) | Shared VPC self links. | object({…}) | | null | 2-networking |
## Outputs
diff --git a/fast/stages/03-data-platform/dev/demo b/fast/stages/3-data-platform/dev/demo
similarity index 100%
rename from fast/stages/03-data-platform/dev/demo
rename to fast/stages/3-data-platform/dev/demo
diff --git a/fast/stages/03-data-platform/dev/diagram.png b/fast/stages/3-data-platform/dev/diagram.png
similarity index 100%
rename from fast/stages/03-data-platform/dev/diagram.png
rename to fast/stages/3-data-platform/dev/diagram.png
diff --git a/fast/stages/03-data-platform/dev/diagram_vpcsc.png b/fast/stages/3-data-platform/dev/diagram_vpcsc.png
similarity index 100%
rename from fast/stages/03-data-platform/dev/diagram_vpcsc.png
rename to fast/stages/3-data-platform/dev/diagram_vpcsc.png
diff --git a/fast/stages/03-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf
similarity index 100%
rename from fast/stages/03-data-platform/dev/main.tf
rename to fast/stages/3-data-platform/dev/main.tf
diff --git a/fast/stages/03-data-platform/dev/outputs.tf b/fast/stages/3-data-platform/dev/outputs.tf
similarity index 95%
rename from fast/stages/03-data-platform/dev/outputs.tf
rename to fast/stages/3-data-platform/dev/outputs.tf
index d0f79358..2eb813b4 100644
--- a/fast/stages/03-data-platform/dev/outputs.tf
+++ b/fast/stages/3-data-platform/dev/outputs.tf
@@ -27,13 +27,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${pathexpand(var.outputs_location)}/tfvars/03-data-platform-dev.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/tfvars/3-data-platform-dev.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/03-data-platform-dev.auto.tfvars.json"
+ name = "tfvars/3-data-platform-dev.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/03-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf
similarity index 90%
rename from fast/stages/03-data-platform/dev/variables.tf
rename to fast/stages/3-data-platform/dev/variables.tf
index 29dd1e45..74a5dbe1 100644
--- a/fast/stages/03-data-platform/dev/variables.tf
+++ b/fast/stages/3-data-platform/dev/variables.tf
@@ -15,7 +15,7 @@
# tfdoc:file:description Terraform Variables.
variable "automation" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Automation resources created by the bootstrap stage."
type = object({
outputs_bucket = string
@@ -23,12 +23,16 @@ variable "automation" {
}
variable "billing_account" {
- # tfdoc:variable:source 00-globals
- description = "Billing account id and organization id ('nnnnnnnn' or null)."
+ # tfdoc:variable:source 0-bootstrap
+ description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false."
type = object({
- id = string
- organization_id = number
+ id = string
+ is_org_level = optional(bool, true)
})
+ validation {
+ condition = var.billing_account.is_org_level != null
+ error_message = "Invalid `null` value for `billing_account.is_org_level`."
+ }
}
variable "composer_config" {
@@ -96,7 +100,7 @@ variable "data_force_destroy" {
}
variable "folder_ids" {
- # tfdoc:variable:source 01-resman
+ # tfdoc:variable:source 1-resman
description = "Folder to be used for the networking resources in folders/nnnn format."
type = object({
data-platform-dev = string
@@ -114,7 +118,7 @@ variable "groups" {
}
variable "host_project_ids" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Shared VPC project ids."
type = object({
dev-spoke-0 = string
@@ -195,7 +199,7 @@ variable "service_encryption_keys" {
}
variable "subnet_self_links" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Shared VPC subnet self links."
type = object({
dev-spoke-0 = map(string)
@@ -204,7 +208,7 @@ variable "subnet_self_links" {
}
variable "vpc_self_links" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Shared VPC self links."
type = object({
dev-spoke-0 = string
diff --git a/fast/stages/03-gke-multitenant/README.md b/fast/stages/3-gke-multitenant/README.md
similarity index 71%
rename from fast/stages/03-gke-multitenant/README.md
rename to fast/stages/3-gke-multitenant/README.md
index f08910c8..9f9d9498 100644
--- a/fast/stages/03-gke-multitenant/README.md
+++ b/fast/stages/3-gke-multitenant/README.md
@@ -2,7 +2,7 @@
This directory contains a stage that can be used to centralize management of GKE multinenant clusters.
-The Terraform code follows the same general approach used for the [project factory](../03-project-factory/) and [data platform](../03-data-platform/) stages, where a "fat module" contains the stage code and is used by thin code wrappers that localize it for each environment or specialized configuration:
+The Terraform code follows the same general approach used for the [project factory](../3-project-factory/) and [data platform](../3-data-platform/) stages, where a "fat module" contains the stage code and is used by thin code wrappers that localize it for each environment or specialized configuration:
The [`dev` folder](./dev/) contains an example setup for a generic development environment, and can be used as-is or cloned to implement other environments, or more specialized setups
diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/3-gke-multitenant/dev/README.md
similarity index 90%
rename from fast/stages/03-gke-multitenant/dev/README.md
rename to fast/stages/3-gke-multitenant/dev/README.md
index c446fbcb..4accf8e1 100644
--- a/fast/stages/03-gke-multitenant/dev/README.md
+++ b/fast/stages/3-gke-multitenant/dev/README.md
@@ -39,7 +39,7 @@ This stage creates a project containing and as many clusters and node pools as r
## How to run this stage
-This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run.
+This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../0-bootstrap), [`01-resman`](../../1-resman), 02-networking (either [VPN](../../2-networking-b-vpn) or [NVA](../../2-networking-c-nva)) and [`02-security`](../../2-security)) have been run.
It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below:
@@ -140,23 +140,23 @@ terraform apply
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap |
-| [billing_account](variables.tf#L29) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [folder_ids](variables.tf#L149) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman |
-| [host_project_ids](variables.tf#L164) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking |
-| [prefix](variables.tf#L213) | Prefix used for resources that need unique names. | string | ✓ | | |
-| [vpc_self_links](variables.tf#L225) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking |
-| [clusters](variables.tf#L38) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | |
-| [fleet_configmanagement_clusters](variables.tf#L86) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | |
-| [fleet_configmanagement_templates](variables.tf#L94) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | |
-| [fleet_features](variables.tf#L129) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | |
-| [fleet_workload_identity](variables.tf#L142) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | |
-| [group_iam](variables.tf#L157) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | |
-| [iam](variables.tf#L172) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | |
-| [labels](variables.tf#L179) | Project-level labels. | map(string) | | {} | |
-| [nodepools](variables.tf#L185) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | |
-| [outputs_location](variables.tf#L207) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
-| [project_services](variables.tf#L218) | Additional project services to enable. | list(string) | | [] | |
+| [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
+| [billing_account](variables.tf#L29) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [folder_ids](variables.tf#L153) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman |
+| [host_project_ids](variables.tf#L168) | Host project for the shared VPC. | object({…}) | ✓ | | 2-networking |
+| [prefix](variables.tf#L217) | Prefix used for resources that need unique names. | string | ✓ | | |
+| [vpc_self_links](variables.tf#L229) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking |
+| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | |
+| [fleet_configmanagement_clusters](variables.tf#L90) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | |
+| [fleet_configmanagement_templates](variables.tf#L98) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | |
+| [fleet_features](variables.tf#L133) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | |
+| [fleet_workload_identity](variables.tf#L146) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | |
+| [group_iam](variables.tf#L161) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | |
+| [iam](variables.tf#L176) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | |
+| [labels](variables.tf#L183) | Project-level labels. | map(string) | | {} | |
+| [nodepools](variables.tf#L189) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | |
+| [outputs_location](variables.tf#L211) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
+| [project_services](variables.tf#L222) | Additional project services to enable. | list(string) | | [] | |
## Outputs
diff --git a/fast/stages/03-gke-multitenant/dev/diagram.png b/fast/stages/3-gke-multitenant/dev/diagram.png
similarity index 100%
rename from fast/stages/03-gke-multitenant/dev/diagram.png
rename to fast/stages/3-gke-multitenant/dev/diagram.png
diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/3-gke-multitenant/dev/main.tf
similarity index 100%
rename from fast/stages/03-gke-multitenant/dev/main.tf
rename to fast/stages/3-gke-multitenant/dev/main.tf
diff --git a/fast/stages/03-gke-multitenant/dev/outputs.tf b/fast/stages/3-gke-multitenant/dev/outputs.tf
similarity index 96%
rename from fast/stages/03-gke-multitenant/dev/outputs.tf
rename to fast/stages/3-gke-multitenant/dev/outputs.tf
index 87b0ca73..3f231c68 100644
--- a/fast/stages/03-gke-multitenant/dev/outputs.tf
+++ b/fast/stages/3-gke-multitenant/dev/outputs.tf
@@ -42,13 +42,13 @@ locals {
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
- filename = "${pathexpand(var.outputs_location)}/tfvars/03-gke-dev.auto.tfvars.json"
+ filename = "${pathexpand(var.outputs_location)}/tfvars/3-gke-dev.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
- name = "tfvars/03-gke-dev.auto.tfvars.json"
+ name = "tfvars/3-gke-dev.auto.tfvars.json"
content = jsonencode(local.tfvars)
}
diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/3-gke-multitenant/dev/variables.tf
similarity index 92%
rename from fast/stages/03-gke-multitenant/dev/variables.tf
rename to fast/stages/3-gke-multitenant/dev/variables.tf
index 6be89126..2dbf5a6e 100644
--- a/fast/stages/03-gke-multitenant/dev/variables.tf
+++ b/fast/stages/3-gke-multitenant/dev/variables.tf
@@ -19,7 +19,7 @@
# cloud dns for gke?
variable "automation" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Automation resources created by the bootstrap stage."
type = object({
outputs_bucket = string
@@ -27,12 +27,16 @@ variable "automation" {
}
variable "billing_account" {
- # tfdoc:variable:source 00-bootstrap
- description = "Billing account id and organization id ('nnnnnnnn' or null)."
+ # tfdoc:variable:source 0-bootstrap
+ description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false."
type = object({
- id = string
- organization_id = number
+ id = string
+ is_org_level = optional(bool, true)
})
+ validation {
+ condition = var.billing_account.is_org_level != null
+ error_message = "Invalid `null` value for `billing_account.is_org_level`."
+ }
}
variable "clusters" {
@@ -147,7 +151,7 @@ variable "fleet_workload_identity" {
}
variable "folder_ids" {
- # tfdoc:variable:source 01-resman
+ # tfdoc:variable:source 1-resman
description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created."
type = object({
gke-dev = string
@@ -162,7 +166,7 @@ variable "group_iam" {
}
variable "host_project_ids" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Host project for the shared VPC."
type = object({
dev-spoke-0 = string
@@ -223,7 +227,7 @@ variable "project_services" {
}
variable "vpc_self_links" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Self link for the shared VPC."
type = object({
dev-spoke-0 = string
diff --git a/fast/stages/03-project-factory/README.md b/fast/stages/3-project-factory/README.md
similarity index 100%
rename from fast/stages/03-project-factory/README.md
rename to fast/stages/3-project-factory/README.md
diff --git a/fast/stages/03-project-factory/dev/README.md b/fast/stages/3-project-factory/dev/README.md
similarity index 85%
rename from fast/stages/03-project-factory/dev/README.md
rename to fast/stages/3-project-factory/dev/README.md
index 8fe213ce..2d95f918 100644
--- a/fast/stages/03-project-factory/dev/README.md
+++ b/fast/stages/3-project-factory/dev/README.md
@@ -28,7 +28,7 @@ The project factory takes care of the following activities:
## How to run this stage
-This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run.
+This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../0-bootstrap), [`01-resman`](../../1-resman), 02-networking (either [VPN](../../2-networking-b-vpn) or [NVA](../../2-networking-c-nva)) and [`02-security`](../../2-security)) have been run.
It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below:
@@ -108,13 +108,13 @@ terraform apply
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
-| [billing_account](variables.tf#L19) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap |
-| [prefix](variables.tf#L56) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap |
-| [data_dir](variables.tf#L28) | Relative path for the folder storing configuration data. | string | | "data/projects" | |
-| [defaults_file](variables.tf#L34) | Relative path for the file storing the project factory configuration. | string | | "data/defaults.yaml" | |
-| [environment_dns_zone](variables.tf#L40) | DNS zone suffix for environment. | string | | null | 02-networking |
-| [host_project_ids](variables.tf#L47) | Host project for the shared VPC. | object({…}) | | null | 02-networking |
-| [vpc_self_links](variables.tf#L67) | Self link for the shared VPC. | object({…}) | | null | 02-networking |
+| [billing_account](variables.tf#L19) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
+| [prefix](variables.tf#L60) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap |
+| [data_dir](variables.tf#L32) | Relative path for the folder storing configuration data. | string | | "data/projects" | |
+| [defaults_file](variables.tf#L38) | Relative path for the file storing the project factory configuration. | string | | "data/defaults.yaml" | |
+| [environment_dns_zone](variables.tf#L44) | DNS zone suffix for environment. | string | | null | 2-networking |
+| [host_project_ids](variables.tf#L51) | Host project for the shared VPC. | object({…}) | | null | 2-networking |
+| [vpc_self_links](variables.tf#L71) | Self link for the shared VPC. | object({…}) | | null | 2-networking |
## Outputs
diff --git a/fast/stages/03-project-factory/dev/data/defaults.yaml b/fast/stages/3-project-factory/dev/data/defaults.yaml
similarity index 100%
rename from fast/stages/03-project-factory/dev/data/defaults.yaml
rename to fast/stages/3-project-factory/dev/data/defaults.yaml
diff --git a/fast/stages/03-project-factory/dev/data/projects/project.yaml.sample b/fast/stages/3-project-factory/dev/data/projects/project.yaml.sample
similarity index 100%
rename from fast/stages/03-project-factory/dev/data/projects/project.yaml.sample
rename to fast/stages/3-project-factory/dev/data/projects/project.yaml.sample
diff --git a/fast/stages/03-project-factory/dev/diagram.png b/fast/stages/3-project-factory/dev/diagram.png
similarity index 100%
rename from fast/stages/03-project-factory/dev/diagram.png
rename to fast/stages/3-project-factory/dev/diagram.png
diff --git a/fast/stages/03-project-factory/dev/diagram.svg b/fast/stages/3-project-factory/dev/diagram.svg
similarity index 100%
rename from fast/stages/03-project-factory/dev/diagram.svg
rename to fast/stages/3-project-factory/dev/diagram.svg
diff --git a/fast/stages/03-project-factory/dev/main.tf b/fast/stages/3-project-factory/dev/main.tf
similarity index 100%
rename from fast/stages/03-project-factory/dev/main.tf
rename to fast/stages/3-project-factory/dev/main.tf
diff --git a/fast/stages/03-project-factory/dev/outputs.tf b/fast/stages/3-project-factory/dev/outputs.tf
similarity index 100%
rename from fast/stages/03-project-factory/dev/outputs.tf
rename to fast/stages/3-project-factory/dev/outputs.tf
diff --git a/fast/stages/03-project-factory/dev/variables.tf b/fast/stages/3-project-factory/dev/variables.tf
similarity index 76%
rename from fast/stages/03-project-factory/dev/variables.tf
rename to fast/stages/3-project-factory/dev/variables.tf
index 2993bfba..5ad49f77 100644
--- a/fast/stages/03-project-factory/dev/variables.tf
+++ b/fast/stages/3-project-factory/dev/variables.tf
@@ -17,12 +17,16 @@
#TODO: tfdoc annotations
variable "billing_account" {
- # tfdoc:variable:source 00-bootstrap
- description = "Billing account id and organization id ('nnnnnnnn' or null)."
+ # tfdoc:variable:source 0-bootstrap
+ description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false."
type = object({
- id = string
- organization_id = number
+ id = string
+ is_org_level = optional(bool, true)
})
+ validation {
+ condition = var.billing_account.is_org_level != null
+ error_message = "Invalid `null` value for `billing_account.is_org_level`."
+ }
}
variable "data_dir" {
@@ -38,14 +42,14 @@ variable "defaults_file" {
}
variable "environment_dns_zone" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "DNS zone suffix for environment."
type = string
default = null
}
variable "host_project_ids" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Host project for the shared VPC."
type = object({
dev-spoke-0 = string
@@ -54,7 +58,7 @@ variable "host_project_ids" {
}
variable "prefix" {
- # tfdoc:variable:source 00-bootstrap
+ # tfdoc:variable:source 0-bootstrap
description = "Prefix used for resources that need unique names. Use 9 characters or less."
type = string
@@ -65,7 +69,7 @@ variable "prefix" {
}
variable "vpc_self_links" {
- # tfdoc:variable:source 02-networking
+ # tfdoc:variable:source 2-networking
description = "Self link for the shared VPC."
type = object({
dev-spoke-0 = string
diff --git a/fast/stages/CLEANUP.md b/fast/stages/CLEANUP.md
index e5f1418f..3bc581f9 100644
--- a/fast/stages/CLEANUP.md
+++ b/fast/stages/CLEANUP.md
@@ -1,4 +1,5 @@
# FAST deployment clean up
+
If you want to destroy a previous FAST deployment in your organization, follow these steps.
Destruction must be done in reverse order, from stage 3 to stage 0
@@ -11,6 +12,7 @@ terraform destroy
```
## Stage 3 (GKE)
+
Terraform refuses to delete non-empty GCS buckets and BigQuery datasets, so they need to be removed manually from the state.
```bash
@@ -24,14 +26,15 @@ done
terraform destroy
```
-
## Stage 2 (Security)
+
```bash
cd $FAST_PWD/02-security/
terraform destroy
```
## Stage 2 (Networking)
+
```bash
cd $FAST_PWD/02-networking-XXX/
terraform destroy
@@ -43,7 +46,6 @@ A minor glitch can surface running `terraform destroy`, where the service projec
Stage 1 is a little more complicated because of the GCS buckets containing your terraform statefiles. By default, Terraform refuses to delete non-empty buckets, which is good to protect your terraform state, but it makes destruction a bit harder. Use the commands below to remove the GCS buckets from the state and then execute `terraform destroy`
-
```bash
cd $FAST_PWD/01-resman/
@@ -110,5 +112,6 @@ rm -i terraform.tfstate*
```
In case you want to deploy FAST stages again, the make sure to:
-* Modify the [prefix](00-bootstrap/variables.tf) variable to allow the deployment of resources that need unique names (eg, projects).
-* Modify the [custom_roles](00-bootstrap/variables.tf) variable to allow recently deleted custom roles to be created again.
+
+* Modify the [prefix](0-bootstrap/variables.tf) variable to allow the deployment of resources that need unique names (eg, projects).
+* Modify the [custom_roles](0-bootstrap/variables.tf) variable to allow recently deleted custom roles to be created again.
diff --git a/fast/stages/COMPANION.md b/fast/stages/COMPANION.md
index e0f6ec62..d5d7752f 100644
--- a/fast/stages/COMPANION.md
+++ b/fast/stages/COMPANION.md
@@ -1,32 +1,42 @@
# FAST deployment companion guide
-To deploy a GCP Landing Zone using FAST, your organization needs to meet a few prerequisites before starting. This guide serves as quick guide to prepare your GCP organization and also as cheat sheet with the commands and minimal configuration required to deploy FAST.
+To deploy a GCP Landing Zone using FAST, your organization needs to meet a few prerequisites before starting. This guide serves as quick guide to prepare your GCP organization and also as cheat sheet with the commands and minimal configuration required to deploy FAST.
The detailed explanation of each stage, their configuration, possible modifications and adaptations are included in the README of stage. This document only outlines the minimal configuration to get from an empty organization to a working FAST deployment.
**Warning! Executing FAST sets organization policies and authoritative role bindings in your GCP Organization. We recommend using FAST on a clean organization, or to fork and adapt FAST to support your existing Organization needs.**
## Prerequisites
+
1. FAST uses the recommended groups from the [GCP Enterprise Setup checklist](). Go to [Workspace / Cloud Identity](https://admin.google.com) and ensure all the following groups exist:
- - `gcp-billing-admins@`
- - `gcp-devops@`
- - `gcp-network-admins@`
- - `gcp-organization-admins@`
- - `gcp-security-admins@`
- - `gcp-support@`
+
+- `gcp-billing-admins@`
+- `gcp-devops@`
+- `gcp-network-admins@`
+- `gcp-organization-admins@`
+- `gcp-security-admins@`
+- `gcp-support@`
+
2. If you already executed FAST in your organization, make you [clean it up](CLEANUP.md) before continuing with the rest of this guide.
+
3. Grant your user “Organization Administrator” role in your organization and add it to the `gcp-organization-admins@` group.
+
4. Login with your user using gcloud.
+
```bash
gcloud auth login
gcloud auth application-default login
```
+
5. Clone the Fabric repository.
+
```bash
git clone https://github.com/GoogleCloudPlatform/cloud-foundation-fabric.git
cd cloud-foundation-fabric
```
+
6. Grant required roles to your user.
+
```bash
# set a variable to the fast folder
export FAST_PWD="$(pwd)/fast/stages"
@@ -49,9 +59,11 @@ gcloud organizations add-iam-policy-binding $FAST_ORG_ID \
--member user:$FAST_BU --role $role
done
```
-7. Configure Billing Account permissions.
+
+7. Configure Billing Account permissions.
If you are using a standalone billing account, the user applying this stage for the first time needs to be a Billing Administrator.
+
```bash
# find your billing account id with gcloud beta billing accounts list
# replace with your billing id!
@@ -60,9 +72,11 @@ export FAST_BA_ID=XXXXXX-YYYYYY-ZZZZZZ
gcloud beta billing accounts add-iam-policy-binding $FAST_BA_ID \
--member user:$FAST_BU --role roles/billing.admin
```
-If you are using a billing account in a different organization, please follow [these steps](00-bootstrap#billing-account-in-a-different-organization) instead.
+
+If you are using a billing account in a different organization, please follow [these steps](0-bootstrap#billing-account-in-a-different-organization) instead.
## Stage 0 (Bootstrap)
+
This initial stage will create common projects for IaC, Logging & Billing, and bootstrap IAM policies.
```bash
@@ -73,7 +87,9 @@ cd $FAST_PWD/00-bootstrap
# then edit to match your environment!
edit terraform.tfvars.sample
```
+
Here you have a terraform.tfvars example:
+
```hcl
# fetch the required id by running `gcloud beta billing accounts list`
billing_account={
@@ -99,7 +115,7 @@ terraform init
terraform apply -var bootstrap_user=$FAST_BU
# link the generated provider file
-ln -s ~/fast-config/providers/00-bootstrap* .
+ln -s ~/fast-config/providers/0-0-bootstrap* .
# re-run init and apply to remove user-level IAM
terraform init -migrate-state
@@ -108,22 +124,27 @@ terraform apply
```
## Stage 1 (Resource Management)
+
This stage performs two important tasks:
+
- Create the top-level hierarchy of folders, and the associated resources used later on to automate each part of the hierarchy (eg. Networking).
- Set organization policies on the organization, and any exception required on specific folders.
+
```bash
# move to the 01-resman directory
cd $FAST_PWD/01-resman
# Link providers and variables from previous stages
-ln -s ~/fast-config/providers/01-resman-providers.tf .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/providers/1-0-resman-providers.tf .
+ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
# Edit your terraform.tfvars to append Teams configuration (optional)
edit terraform.tfvars
```
+
In the following terraform.tfvars it is shown an example of configuration for teams provisioning:
+
```hcl
outputs_location = "~/fast-config"
@@ -140,6 +161,7 @@ team_folders = {
}
}
```
+
```bash
# run init and apply
terraform init
@@ -147,28 +169,34 @@ terraform apply
```
## Stage 2 (Networking)
+
In this stage, we will deploy one of the 3 available Hub&Spoke networking topologies:
+
1. VPC Peering
2. HA VPN
3. Multi-NIC appliances (NVA)
+
```bash
# move to the 02-networking-XXX directory (where XXX should be one of vpn|peering|nva)
cd $FAST_PWD/02-networking-XXX
# setup providers and variables from previous stages
-ln -s ~/fast-config/providers/02-networking-providers.tf .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
+ln -s ~/fast-config/providers/2-0-networking-providers.tf .
+ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/1-0-resman.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
# Create terraform.tfvars. output_location variable is required to generate networking stage output file
edit terraform.tfvars
```
+
In the following terraform.tfvars we configure output_location variable to generate networking stage output file:
+
```hcl
# path for automatic generation of configs
outputs_location = "~/fast-config"
```
+
```bash
# run init and apply
terraform init
@@ -176,21 +204,25 @@ terraform apply
```
## Stage 2 (Security)
+
This stage sets up security resources (KMS and VPC-SC) and configurations which impact the whole organization, or are shared across the hierarchy to other projects and teams.
+
```bash
# move to the 02-security directory
cd $FAST_PWD/02-security
# link providers and variables from previous stages
-ln -s ~/fast-config/providers/02-security-providers.tf .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
+ln -s ~/fast-config/providers/2-0-security-providers.tf .
+ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/1-0-resman.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
# Edit terraform.tfvars to include KMS and/or VPC-SC configuration
edit terraform.tfvars
```
-Some examples of terraform.tfvars configurations for KMS and VPC-SC can be found [here](02-security#customizations)
+
+Some examples of terraform.tfvars configurations for KMS and VPC-SC can be found [here](2-security#customizations)
+
```bash
# run init and apply
terraform init
@@ -198,15 +230,17 @@ terraform apply
```
## Stage 3 (Project Factory)
+
The Project Factory stage builds on top of your foundations to create and set up projects (and related resources) to be used for your workloads. It is organized in folders representing environments (e.g. "dev", "prod"), each implemented by a stand-alone terraform resource factory.
+
```bash
# Variable `outputs_location` is set to `~/fast-config`
-cd $FAST_PWD/03-project-factory/ENVIRONMENT
-ln -s ~/fast-config/providers/03-project-factory-ENVIRONMENT-providers.tf .
+cd $FAST_PWD/3-0-project-factory/ENVIRONMENT
+ln -s ~/fast-config/providers/3-0-project-factory-ENVIRONMENT-providers.tf .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/1-0-resman.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/2-0-networking.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
# Define your environment default values (eg for billing alerts and labels)
diff --git a/fast/stages/README.md b/fast/stages/README.md
index 9b41bf1c..acb14be6 100644
--- a/fast/stages/README.md
+++ b/fast/stages/README.md
@@ -1,4 +1,4 @@
-# Fast stages
+# FAST stages
Each of the folders contained here is a separate "stage", or Terraform root module.
@@ -9,7 +9,7 @@ When combined together, each stage is designed to leverage the previous stage's
This has two important consequences
- any stage can be swapped out and replaced by different code as long as it respects the contract by providing a predefined set of outputs and optionally accepting a predefined set of variables
-- data flow between stages can be partially automated (see [stage 00 documentation on output files](./00-bootstrap/README.md#output-files-and-cross-stage-variables)), reducing the effort and pain required to compile variables by hand
+- data flow between stages can be partially automated (see [stage 00 documentation on output files](./0-bootstrap/README.md#output-files-and-cross-stage-variables)), reducing the effort and pain required to compile variables by hand
One important assumption is that the flow of data is always forward looking, so no stage needs to depend on outputs generated further down the chain. This greatly simplifies both the logic and the implementation, and allows stages to be effectively independent.
@@ -19,28 +19,32 @@ Refer to each stage's documentation for a detailed description of its purpose, t
To destroy a previous FAST deployment follow the instructions detailed in [cleanup](CLEANUP.md).
-## Organizational level (00-01)
+## Organization (0 and 1)
-- [Bootstrap](00-bootstrap/README.md)
+- [Bootstrap](0-bootstrap/README.md)
Enables critical organization-level functionality that depends on broad permissions. It has two primary purposes. The first is to bootstrap the resources needed for automation of this and the following stages (service accounts, GCS buckets). And secondly, it applies the minimum amount of configuration needed at the organization level, to avoid the need of broad permissions later on, and to implement a minimum of security features like sinks and exports from the start.\
Exports: automation variables, organization-level custom roles
-- [Resource Management](01-resman/README.md)
+- [Resource Management](1-resman/README.md)
Creates the base resource hierarchy (folders) and the automation resources required later to delegate deployment of each part of the hierarchy to separate stages. This stage also configures organization-level policies and any exceptions needed by different branches of the resource hierarchy.\
Exports: folder ids, automation service account emails
-## Shared resources (02)
+## Multitenancy
-- [Security](02-security/README.md)
+Implemented via separate stages that configure separate FAST-enabled hierarchies for each tenant, check the [multitenant stages folder](../stages-multitenant/).
+
+## Shared resources (2)
+
+- [Security](2-security/README.md)
Manages centralized security configurations in a separate stage, and is typically owned by the security team. This stage implements VPC Security Controls via separate perimeters for environments and central services, and creates projects to host centralized KMS keys used by the whole organization. It's meant to be easily extended to include other security-related resources which are required, like Secret Manager.\
Exports: KMS key ids
-- Networking ([VPN](02-networking-vpn/README.md)/[NVA](02-networking-nva/README.md)/[Peering](02-networking-separate-envs/README.md)/[Separate environments](02-networking-separate-envs/README.md))
- Manages centralized network resources in a separate stage, and is typically owned by the networking team. This stage implements a hub-and-spoke design, and includes connectivity via VPN to on-premises, and YAML-based factories for firewall rules (hierarchical and VPC-level) and subnets. It's currently available in four flavors: [spokes connected via VPN](02-networking-vpn/README.md), [and spokes connected via appliances](02-networking-nva/README.md), [spokes connected via VPC peering](02-networking-peering/README.md), and [separated network environments](02-networking-separate-envs/README.md).\
+- Networking ([Peering](2-networking-a-peering/README.md)/[VPN](2-networking-b-vpn/README.md)/[NVA](2-networking-c-nva/README.md)/[Separate environments](2-networking-d-separate-envs/README.md))
+ Manages centralized network resources in a separate stage, and is typically owned by the networking team. This stage implements a hub-and-spoke design, and includes connectivity via VPN to on-premises, and YAML-based factories for firewall rules (hierarchical and VPC-level) and subnets. It's currently available in four flavors: [spokes connected via VPC peering](2-networking-a-peering/README.md), [spokes connected via VPN](2-networking-b-vpn/README.md), [and spokes connected via appliances](2-networking-c-nva/README.md), and [separated network environments](2-networking-d-separate-envs/README.md).\
Exports: host project ids and numbers, vpc self links
-## Environment-level resources (03)
+## Environment-level resources (3)
-- [Project Factory](03-project-factory/dev/)
+- [Project Factory](3-project-factory/dev/)
YAML-based fatory to create and configure application or team-level projects. Configuration includes VPC-level settings for Shared VPC, service-level configuration for CMEK encryption via centralized keys, and service account creation for workloads and applications. This stage is meant to be used once per environment.
-- [Data Platform](03-data-platform/dev/)
-- [GKE Multitenant](03-gke-multitenant/dev/)
+- [Data Platform](3-data-platform/dev/)
+- [GKE Multitenant](3-gke-multitenant/dev/)
- GCE Migration (in development)
diff --git a/modules/__experimental/net-neg/versions.tf b/modules/__experimental/net-neg/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/__experimental/net-neg/versions.tf
+++ b/modules/__experimental/net-neg/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/api-gateway/versions.tf b/modules/api-gateway/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/api-gateway/versions.tf
+++ b/modules/api-gateway/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/apigee/versions.tf b/modules/apigee/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/apigee/versions.tf
+++ b/modules/apigee/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/artifact-registry/versions.tf b/modules/artifact-registry/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/artifact-registry/versions.tf
+++ b/modules/artifact-registry/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/bigquery-dataset/versions.tf b/modules/bigquery-dataset/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/bigquery-dataset/versions.tf
+++ b/modules/bigquery-dataset/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/bigtable-instance/versions.tf b/modules/bigtable-instance/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/bigtable-instance/versions.tf
+++ b/modules/bigtable-instance/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/billing-budget/versions.tf b/modules/billing-budget/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/billing-budget/versions.tf
+++ b/modules/billing-budget/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/binauthz/versions.tf b/modules/binauthz/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/binauthz/versions.tf
+++ b/modules/binauthz/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/__need_fixing/onprem/versions.tf b/modules/cloud-config-container/__need_fixing/onprem/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/__need_fixing/onprem/versions.tf
+++ b/modules/cloud-config-container/__need_fixing/onprem/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/coredns/versions.tf b/modules/cloud-config-container/coredns/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/coredns/versions.tf
+++ b/modules/cloud-config-container/coredns/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/cos-generic-metadata/versions.tf b/modules/cloud-config-container/cos-generic-metadata/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/cos-generic-metadata/versions.tf
+++ b/modules/cloud-config-container/cos-generic-metadata/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/envoy-traffic-director/versions.tf b/modules/cloud-config-container/envoy-traffic-director/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/envoy-traffic-director/versions.tf
+++ b/modules/cloud-config-container/envoy-traffic-director/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/mysql/versions.tf b/modules/cloud-config-container/mysql/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/mysql/versions.tf
+++ b/modules/cloud-config-container/mysql/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/nginx-tls/versions.tf b/modules/cloud-config-container/nginx-tls/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/nginx-tls/versions.tf
+++ b/modules/cloud-config-container/nginx-tls/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/nginx/versions.tf b/modules/cloud-config-container/nginx/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/nginx/versions.tf
+++ b/modules/cloud-config-container/nginx/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/simple-nva/versions.tf b/modules/cloud-config-container/simple-nva/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/simple-nva/versions.tf
+++ b/modules/cloud-config-container/simple-nva/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-config-container/squid/versions.tf b/modules/cloud-config-container/squid/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-config-container/squid/versions.tf
+++ b/modules/cloud-config-container/squid/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-function/versions.tf b/modules/cloud-function/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-function/versions.tf
+++ b/modules/cloud-function/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-identity-group/versions.tf b/modules/cloud-identity-group/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-identity-group/versions.tf
+++ b/modules/cloud-identity-group/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloud-run/versions.tf b/modules/cloud-run/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloud-run/versions.tf
+++ b/modules/cloud-run/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/cloudsql-instance/versions.tf b/modules/cloudsql-instance/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/cloudsql-instance/versions.tf
+++ b/modules/cloudsql-instance/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/compute-mig/versions.tf b/modules/compute-mig/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/compute-mig/versions.tf
+++ b/modules/compute-mig/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md
index 2f715555..7cfaf068 100644
--- a/modules/compute-vm/README.md
+++ b/modules/compute-vm/README.md
@@ -8,6 +8,23 @@ This module can operate in two distinct modes:
In both modes, an optional service account can be created and assigned to either instances or template. If you need a managed instance group when using the module in template mode, refer to the [`compute-mig`](../compute-mig) module.
## Examples
+- [Instance using defaults](#instance-using-defaults)
+- [Service account management](#service-account-management)
+- [Disk management](#disk-management)
+ - [Disk sources](#disk-sources)
+ - [Disk types and options](#disk-types-and-options)
+- [Network interfaces](#network-interfaces)
+ - [Internal and external IPs](#internal-and-external-ips)
+ - [Using Alias IPs](#using-alias-ips)
+ - [Using gVNIC](#using-gvnic)
+- [Metadata](#metadata)
+- [IAM](#iam)
+- [Spot VM](#spot-vm)
+- [Confidential compute](#confidential-compute)
+- [Disk encryption with Cloud KMS](#disk-encryption-with-cloud-kms)
+- [Instance template](#instance-template)
+- [Instance group](#instance-group)
+
### Instance using defaults
@@ -25,47 +42,73 @@ module "simple-vm-example" {
}]
service_account_create = true
}
-# tftest modules=1 resources=2
-
+# tftest modules=1 resources=2 inventory=simple.yaml
```
-### Spot VM
+### Service account management
-[Spot VMs](https://cloud.google.com/compute/docs/instances/spot) are ephemeral compute instances suitable for batch jobs and fault-tolerant workloads. Spot VMs provide new features that [preemptible instances](https://cloud.google.com/compute/docs/instances/preemptible) do not support, such as the absence of a maximum runtime.
+VM service accounts can be managed in three different ways:
+- You can let the module create a service account for you by settting `service_account_create = true`
+- You can use an existing service account by setting `service_account_create = false` (the default value) and passing the full email address of the service account to the `service_account` variable. This is useful, for example, if you want to reuse the service account from another previously created instance, or if you want to create the service account manually with the `iam-service-account` module. In this case, you probably also want to set `service_account_scopes` to `cloud-platform`.
+- Lastly, you can use the default compute service account by setting `service_account_crate = false`. Please note that using the default compute service account is not recommended.
```hcl
-module "spot-vm-example" {
+module "vm-managed-sa-example" {
source = "./fabric/modules/compute-vm"
project_id = var.project_id
zone = "europe-west1-b"
- name = "test"
- options = {
- spot = true
- termination_action = "STOP"
- }
+ name = "test1"
network_interfaces = [{
network = var.vpc.self_link
subnetwork = var.subnet.self_link
}]
service_account_create = true
}
-# tftest modules=1 resources=2
+module "vm-managed-sa-example2" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "test2"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+ service_account = module.vm-managed-sa-example.service_account_email
+ service_account_scopes = ["cloud-platform"]
+}
+
+# not recommended
+module "vm-default-sa-example2" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "test3"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+ service_account_create = false
+}
+
+# tftest modules=3 resources=4 inventory=sas.yaml
```
-### Disk sources
+### Disk management
+
+#### Disk sources
Attached disks can be created and optionally initialized from a pre-existing source, or attached to VMs when pre-existing. The `source` and `source_type` attributes of the `attached_disks` variable allows several modes of operation:
-- `source_type = "image"` can be used with zonal disks in instances and templates, set `source` to the image name or link
-- `source_type = "snapshot"` can be used with instances only, set `source` to the snapshot name or link
-- `source_type = "attach"` can be used for both instances and templates to attach an existing disk, set source to the name (for zonal disks) or link (for regional disks) of the existing disk to attach; no disk will be created
+- `source_type = "image"` can be used with zonal disks in instances and templates, set `source` to the image name or self link
+- `source_type = "snapshot"` can be used with instances only, set `source` to the snapshot name or self link
+- `source_type = "attach"` can be used for both instances and templates to attach an existing disk, set source to the name (for zonal disks) or self link (for regional disks) of the existing disk to attach; no disk will be created
- `source_type = null` can be used where an empty disk is needed, `source` becomes irrelevant and can be left null
This is an example of attaching a pre-existing regional PD to a new instance:
```hcl
-module "simple-vm-example" {
+module "vm-disks-example" {
source = "./fabric/modules/compute-vm"
project_id = var.project_id
zone = "${var.region}-b"
@@ -91,7 +134,7 @@ module "simple-vm-example" {
And the same example for an instance template (where not using the full self link of the disk triggers recreation of the template)
```hcl
-module "simple-vm-example" {
+module "vm-disks-example" {
source = "./fabric/modules/compute-vm"
project_id = var.project_id
zone = "${var.region}-b"
@@ -115,39 +158,82 @@ module "simple-vm-example" {
# tftest modules=1 resources=2
```
-### Disk encryption with Cloud KMS
+#### Disk types and options
-This example shows how to control disk encryption via the the `encryption` variable, in this case the self link to a KMS CryptoKey that will be used to encrypt boot and attached disk. Managing the key with the `../kms` module is of course possible, but is not shown here.
+The `attached_disks` variable exposes an `option` attribute that can be used to fine tune the configuration of each disk. The following example shows a VM with multiple disks
```hcl
-module "kms-vm-example" {
+module "vm-disk-options-example" {
source = "./fabric/modules/compute-vm"
project_id = var.project_id
zone = "europe-west1-b"
- name = "kms-test"
+ name = "test"
network_interfaces = [{
network = var.vpc.self_link
subnetwork = var.subnet.self_link
}]
attached_disks = [
{
- name = "attached-disk"
- size = 10
+ name = "data1"
+ size = "10"
+ source_type = "image"
+ source = "image-1"
+ options = {
+ auto_delete = false
+ replica_zone = "europe-west1-c"
+ }
+ },
+ {
+ name = "data2"
+ size = "20"
+ source_type = "snapshot"
+ source = "snapshot-2"
+ options = {
+ type = "pd-ssd"
+ mode = "READ_ONLY"
+ }
}
]
service_account_create = true
- boot_disk = {
- image = "projects/debian-cloud/global/images/family/debian-10"
- }
- encryption = {
- encrypt_boot = true
- kms_key_self_link = var.kms_key.self_link
- }
}
-# tftest modules=1 resources=3
+# tftest modules=1 resources=4 inventory=disk-options.yaml
```
-### Using Alias IPs
+### Network interfaces
+
+#### Internal and external IPs
+
+By default VNs are create with an automatically assigned IP addresses, but you can change it through the `addreses` and `nat` attributes of the `network_interfaces` variable:
+
+```hcl
+module "vm-internal-ip" {
+ source = "./fabric/modules/compute-vm"
+ project_id = "my-project"
+ zone = "europe-west1-b"
+ name = "vm-internal-ip"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ addresses = { external = null, internal = "10.0.0.2" }
+ }]
+}
+
+module "vm-external-ip" {
+ source = "./fabric/modules/compute-vm"
+ project_id = "my-project"
+ zone = "europe-west1-b"
+ name = "vm-external-ip"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ nat = true
+ addresses = { external = "8.8.8.8", internal = null }
+ }]
+}
+# tftest modules=2 resources=2 inventory=ips.yaml
+```
+
+#### Using Alias IPs
This example shows how to add additional [Alias IPs](https://cloud.google.com/vpc/docs/alias-ip) to your VM.
@@ -164,12 +250,11 @@ module "vm-with-alias-ips" {
alias1 = "10.16.0.10/32"
}
}]
- service_account_create = true
}
-# tftest modules=1 resources=2
+# tftest modules=1 resources=1 inventory=alias-ips.yaml
```
-### Using gVNIC
+#### Using gVNIC
This example shows how to enable [gVNIC](https://cloud.google.com/compute/docs/networking/using-gvnic) on your VM by customizing a `cos` image. Given that gVNIC needs to be enabled as an instance configuration and as a guest os configuration, you'll need to supply a bootable disk with `guest_os_features=GVNIC`. `SEV_CAPABLE`, `UEFI_COMPATIBLE` and `VIRTIO_SCSI_MULTIQUEUE` are enabled implicitly in the `cos`, `rhel`, `centos` and other images.
@@ -210,9 +295,151 @@ module "vm-with-gvnic" {
}]
service_account_create = true
}
-# tftest modules=1 resources=3
+# tftest modules=1 resources=3 inventory=gvnic.yaml
```
+### Metadata
+
+You can define labels and custom metadata values. Metadata can be leveraged, for example, to define a custom startup script.
+
+```hcl
+module "vm-metadata-example" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "nginx-server"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+ labels = {
+ env = "dev"
+ system = "crm"
+ }
+ metadata = {
+ startup-script = <<-EOF
+ #! /bin/bash
+ apt-get update
+ apt-get install -y nginx
+ EOF
+ }
+ service_account_create = true
+}
+# tftest modules=1 resources=2 inventory=metadata.yaml
+```
+
+### IAM
+
+Like most modules, you can assign IAM roles to the instance using the `iam` variable.
+
+```hcl
+module "vm-iam-example" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "webserver"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+ iam = {
+ "roles/compute.instanceAdmin" = [
+ "group:webserver@example.com",
+ "group:admin@example.com"
+ ]
+ }
+}
+# tftest modules=1 resources=2 inventory=iam.yaml
+
+```
+
+### Spot VM
+
+[Spot VMs](https://cloud.google.com/compute/docs/instances/spot) are ephemeral compute instances suitable for batch jobs and fault-tolerant workloads. Spot VMs provide new features that [preemptible instances](https://cloud.google.com/compute/docs/instances/preemptible) do not support, such as the absence of a maximum runtime.
+
+```hcl
+module "spot-vm-example" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "test"
+ options = {
+ spot = true
+ termination_action = "STOP"
+ }
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+}
+# tftest modules=1 resources=1 inventory=spot.yaml
+```
+
+### Confidential compute
+
+You can enable confidential compute with the `confidential_compute` variable, which can be used for standalone instances or for instance templates.
+
+```hcl
+module "vm-confidential-example" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "confidential-vm"
+ confidential_compute = true
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+
+}
+
+module "template-confidential-example" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "confidential-template"
+ confidential_compute = true
+ create_template = true
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+}
+
+# tftest modules=2 resources=2 inventory=confidential.yaml
+```
+
+### Disk encryption with Cloud KMS
+
+This example shows how to control disk encryption via the the `encryption` variable, in this case the self link to a KMS CryptoKey that will be used to encrypt boot and attached disk. Managing the key with the `../kms` module is of course possible, but is not shown here.
+
+```hcl
+module "kms-vm-example" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ zone = "europe-west1-b"
+ name = "kms-test"
+ network_interfaces = [{
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }]
+ attached_disks = [{
+ name = "attached-disk"
+ size = 10
+ }]
+ service_account_create = true
+ boot_disk = {
+ image = "projects/debian-cloud/global/images/family/debian-10"
+ }
+ encryption = {
+ encrypt_boot = true
+ kms_key_self_link = var.kms_key.self_link
+ }
+}
+# tftest modules=1 resources=3 inventory=cmek.yaml
+```
+
+
### Instance template
This example shows how to use the module to manage an instance template that defines an additional attached disk for each instance, and overrides defaults for the boot disk image and service account.
@@ -239,7 +466,7 @@ module "cos-test" {
service_account = "vm-default@my-project.iam.gserviceaccount.com"
create_template = true
}
-# tftest modules=1 resources=1
+# tftest modules=1 resources=1 inventory=template.yaml
```
### Instance group
@@ -270,7 +497,7 @@ module "instance-group" {
}
group = { named_ports = {} }
}
-# tftest modules=1 resources=2
+# tftest modules=1 resources=2 inventory=group.yaml
```
diff --git a/modules/compute-vm/versions.tf b/modules/compute-vm/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/compute-vm/versions.tf
+++ b/modules/compute-vm/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/container-registry/versions.tf b/modules/container-registry/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/container-registry/versions.tf
+++ b/modules/container-registry/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/data-catalog-policy-tag/versions.tf b/modules/data-catalog-policy-tag/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/data-catalog-policy-tag/versions.tf
+++ b/modules/data-catalog-policy-tag/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/datafusion/versions.tf b/modules/datafusion/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/datafusion/versions.tf
+++ b/modules/datafusion/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/dns/versions.tf b/modules/dns/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/dns/versions.tf
+++ b/modules/dns/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/endpoints/versions.tf b/modules/endpoints/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/endpoints/versions.tf
+++ b/modules/endpoints/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/folder/versions.tf b/modules/folder/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/folder/versions.tf
+++ b/modules/folder/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/gcs/versions.tf b/modules/gcs/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/gcs/versions.tf
+++ b/modules/gcs/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md
index dabead4f..df7f14ca 100644
--- a/modules/gke-cluster/README.md
+++ b/modules/gke-cluster/README.md
@@ -103,25 +103,25 @@ module "cluster-autopilot" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [location](variables.tf#L118) | Cluster zone or region. | string | ✓ | |
-| [name](variables.tf#L175) | Cluster name. | string | ✓ | |
-| [project_id](variables.tf#L201) | Cluster project id. | string | ✓ | |
-| [vpc_config](variables.tf#L218) | VPC-level configuration. | object({…}) | ✓ | |
+| [location](variables.tf#L119) | Cluster zone or region. | string | ✓ | |
+| [name](variables.tf#L176) | Cluster name. | string | ✓ | |
+| [project_id](variables.tf#L202) | Cluster project id. | string | ✓ | |
+| [vpc_config](variables.tf#L219) | VPC-level configuration. | object({…}) | ✓ | |
| [cluster_autoscaling](variables.tf#L17) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | null |
| [description](variables.tf#L38) | Cluster description. | string | | null |
| [enable_addons](variables.tf#L44) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} |
-| [enable_features](variables.tf#L68) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {…} |
-| [issue_client_certificate](variables.tf#L106) | Enable issuing client certificate. | bool | | false |
-| [labels](variables.tf#L112) | Cluster resource labels. | map(string) | | null |
-| [logging_config](variables.tf#L123) | Logging configuration. | list(string) | | ["SYSTEM_COMPONENTS"] |
-| [maintenance_config](variables.tf#L129) | Maintenance window configuration. | object({…}) | | {…} |
-| [max_pods_per_node](variables.tf#L152) | Maximum number of pods per node in this cluster. | number | | 110 |
-| [min_master_version](variables.tf#L158) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null |
-| [monitoring_config](variables.tf#L164) | Monitoring components. | object({…}) | | {…} |
-| [node_locations](variables.tf#L180) | Zones in which the cluster's nodes are located. | list(string) | | [] |
-| [private_cluster_config](variables.tf#L187) | Private cluster configuration. | object({…}) | | null |
-| [release_channel](variables.tf#L206) | Release channel for GKE upgrades. | string | | null |
-| [tags](variables.tf#L212) | Network tags applied to nodes. | list(string) | | null |
+| [enable_features](variables.tf#L68) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {…} |
+| [issue_client_certificate](variables.tf#L107) | Enable issuing client certificate. | bool | | false |
+| [labels](variables.tf#L113) | Cluster resource labels. | map(string) | | null |
+| [logging_config](variables.tf#L124) | Logging configuration. | list(string) | | ["SYSTEM_COMPONENTS"] |
+| [maintenance_config](variables.tf#L130) | Maintenance window configuration. | object({…}) | | {…} |
+| [max_pods_per_node](variables.tf#L153) | Maximum number of pods per node in this cluster. | number | | 110 |
+| [min_master_version](variables.tf#L159) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null |
+| [monitoring_config](variables.tf#L165) | Monitoring components. | object({…}) | | {…} |
+| [node_locations](variables.tf#L181) | Zones in which the cluster's nodes are located. | list(string) | | [] |
+| [private_cluster_config](variables.tf#L188) | Private cluster configuration. | object({…}) | | null |
+| [release_channel](variables.tf#L207) | Release channel for GKE upgrades. | string | | null |
+| [tags](variables.tf#L213) | Network tags applied to nodes. | list(string) | | null |
## Outputs
diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf
index f55f46e8..83604c00 100644
--- a/modules/gke-cluster/main.tf
+++ b/modules/gke-cluster/main.tf
@@ -55,15 +55,18 @@ resource "google_container_cluster" "cluster" {
# the default nodepool is deleted here, use the gke-nodepool module instead
# default nodepool configuration based on a shielded_nodes variable
- node_config {
- dynamic "shielded_instance_config" {
- for_each = var.enable_features.shielded_nodes ? [""] : []
- content {
- enable_secure_boot = true
- enable_integrity_monitoring = true
+ dynamic "node_config" {
+ for_each = var.enable_features.autopilot ? [] : [""]
+ content {
+ dynamic "shielded_instance_config" {
+ for_each = var.enable_features.shielded_nodes ? [""] : []
+ content {
+ enable_secure_boot = true
+ enable_integrity_monitoring = true
+ }
}
+ tags = var.tags
}
- tags = var.tags
}
@@ -262,6 +265,13 @@ resource "google_container_cluster" "cluster" {
}
}
+ dynamic "mesh_certificates" {
+ for_each = var.enable_features.mesh_certificates != null ? [""] : []
+ content {
+ enable_certificates = var.enable_features.mesh_certificates
+ }
+ }
+
dynamic "monitoring_config" {
for_each = var.monitoring_config != null && !var.enable_features.autopilot ? [""] : []
content {
diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf
index f02ed504..ecfa11c9 100644
--- a/modules/gke-cluster/variables.tf
+++ b/modules/gke-cluster/variables.tf
@@ -84,6 +84,7 @@ variable "enable_features" {
groups_for_rbac = optional(string)
intranode_visibility = optional(bool, false)
l4_ilb_subsetting = optional(bool, false)
+ mesh_certificates = optional(bool)
pod_security_policy = optional(bool, false)
resource_usage_export = optional(object({
dataset = string
@@ -232,4 +233,4 @@ variable "vpc_config" {
master_authorized_ranges = optional(map(string))
})
nullable = false
-}
\ No newline at end of file
+}
diff --git a/modules/gke-cluster/versions.tf b/modules/gke-cluster/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/gke-cluster/versions.tf
+++ b/modules/gke-cluster/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/gke-hub/README.md b/modules/gke-hub/README.md
index 6afcd1c8..793a81ca 100644
--- a/modules/gke-hub/README.md
+++ b/modules/gke-hub/README.md
@@ -141,6 +141,13 @@ module "project" {
]
}
+resource "google_project_iam_member" "gkehub_fix" {
+ member = "serviceAccount:${module.project.service_accounts.robots.fleet}"
+ project = module.project.project_id
+ role = "roles/gkehub.serviceAgent"
+}
+
+
module "vpc" {
source = "./fabric/modules/net-vpc"
project_id = module.project.project_id
@@ -151,7 +158,7 @@ module "vpc" {
ip_cidr_range = "10.0.1.0/24"
name = "subnet-cluster-1"
region = "europe-west1"
- secondary_ip_range = {
+ secondary_ip_ranges = {
pods = "10.1.0.0/16"
services = "10.2.0.0/24"
}
@@ -160,16 +167,16 @@ module "vpc" {
ip_cidr_range = "10.0.2.0/24"
name = "subnet-cluster-2"
region = "europe-west4"
- secondary_ip_range = {
+ secondary_ip_ranges = {
pods = "10.3.0.0/16"
services = "10.4.0.0/24"
}
},
{
- ip_cidr_range = "10.0.0.0/28"
- name = "subnet-mgmt"
- region = "europe-west1"
- secondary_ip_range = null
+ ip_cidr_range = "10.0.0.0/28"
+ name = "subnet-mgmt"
+ region = "europe-west1"
+ secondary_ip_ranges = null
}
]
}
@@ -222,18 +229,24 @@ module "cluster_1" {
enable_private_endpoint = false
master_global_access = true
}
+
release_channel = "REGULAR"
labels = {
mesh_id = "proj-${module.project.number}"
}
+ enable_features = {
+ workload_identity = true
+ dataplane_v2 = true
+ }
}
module "cluster_1_nodepool" {
source = "./fabric/modules/gke-nodepool"
project_id = module.project.project_id
cluster_name = module.cluster_1.name
+ cluster_id = module.cluster_1.id
location = "europe-west1"
- name = "nodepool"
+ name = "cluster-1-nodepool"
node_count = { initial = 1 }
service_account = { create = true }
tags = ["cluster-1-node"]
@@ -261,14 +274,19 @@ module "cluster_2" {
labels = {
mesh_id = "proj-${module.project.number}"
}
+ enable_features = {
+ workload_identity = true
+ dataplane_v2 = true
+ }
}
module "cluster_2_nodepool" {
source = "./fabric/modules/gke-nodepool"
project_id = module.project.project_id
cluster_name = module.cluster_2.name
+ cluster_id = module.cluster_2.id
location = "europe-west4"
- name = "nodepool"
+ name = "cluster-2-nodepool"
node_count = { initial = 1 }
service_account = { create = true }
tags = ["cluster-2-node"]
@@ -277,6 +295,7 @@ module "cluster_2_nodepool" {
module "hub" {
source = "./fabric/modules/gke-hub"
project_id = module.project.project_id
+ depends_on = [google_project_iam_member.gkehub_fix]
clusters = {
cluster-1 = module.cluster_1.id
cluster-2 = module.cluster_2.id
@@ -295,7 +314,7 @@ module "hub" {
]
}
-# tftest modules=8 resources=30
+# tftest modules=8 resources=31
```
diff --git a/modules/gke-hub/versions.tf b/modules/gke-hub/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/gke-hub/versions.tf
+++ b/modules/gke-hub/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/gke-nodepool/versions.tf b/modules/gke-nodepool/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/gke-nodepool/versions.tf
+++ b/modules/gke-nodepool/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/iam-service-account/versions.tf b/modules/iam-service-account/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/iam-service-account/versions.tf
+++ b/modules/iam-service-account/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/kms/versions.tf b/modules/kms/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/kms/versions.tf
+++ b/modules/kms/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/logging-bucket/versions.tf b/modules/logging-bucket/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/logging-bucket/versions.tf
+++ b/modules/logging-bucket/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-address/versions.tf b/modules/net-address/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-address/versions.tf
+++ b/modules/net-address/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-cloudnat/versions.tf b/modules/net-cloudnat/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-cloudnat/versions.tf
+++ b/modules/net-cloudnat/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-glb/versions.tf b/modules/net-glb/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-glb/versions.tf
+++ b/modules/net-glb/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-ilb-l7/versions.tf b/modules/net-ilb-l7/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-ilb-l7/versions.tf
+++ b/modules/net-ilb-l7/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-ilb/versions.tf b/modules/net-ilb/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-ilb/versions.tf
+++ b/modules/net-ilb/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-interconnect-attachment-direct/versions.tf b/modules/net-interconnect-attachment-direct/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-interconnect-attachment-direct/versions.tf
+++ b/modules/net-interconnect-attachment-direct/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-vpc-firewall/versions.tf b/modules/net-vpc-firewall/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-vpc-firewall/versions.tf
+++ b/modules/net-vpc-firewall/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-vpc-peering/versions.tf b/modules/net-vpc-peering/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-vpc-peering/versions.tf
+++ b/modules/net-vpc-peering/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-vpc/versions.tf
+++ b/modules/net-vpc/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-vpn-dynamic/versions.tf b/modules/net-vpn-dynamic/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-vpn-dynamic/versions.tf
+++ b/modules/net-vpn-dynamic/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-vpn-ha/versions.tf b/modules/net-vpn-ha/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-vpn-ha/versions.tf
+++ b/modules/net-vpn-ha/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/net-vpn-static/versions.tf b/modules/net-vpn-static/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/net-vpn-static/versions.tf
+++ b/modules/net-vpn-static/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/organization/README.md b/modules/organization/README.md
index da44046a..b6caa3cd 100644
--- a/modules/organization/README.md
+++ b/modules/organization/README.md
@@ -462,13 +462,13 @@ module "org" {
| [iam_bindings_authoritative](variables.tf#L116) | IAM authoritative bindings, in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared. Bindings should also be authoritative when using authoritative audit config. Use with caution. | map(list(string)) | | null |
| [logging_exclusions](variables.tf#L122) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} |
| [logging_sinks](variables.tf#L129) | Logging sinks to create for the organization. | map(object({…})) | | {} |
-| [network_tags](variables.tf#L159) | Network tags by key name. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} |
-| [org_policies](variables.tf#L180) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} |
+| [network_tags](variables.tf#L159) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} |
+| [org_policies](variables.tf#L181) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} |
| [org_policies_data_path](variables.tf#L220) | Path containing org policies in YAML format. | string | | null |
| [org_policy_custom_constraints](variables.tf#L226) | Organization policiy custom constraints keyed by constraint name. | map(object({…})) | | {} |
| [org_policy_custom_constraints_data_path](variables.tf#L240) | Path containing org policy custom constraints in YAML format. | string | | null |
| [tag_bindings](variables.tf#L255) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null |
-| [tags](variables.tf#L261) | Tags by key name. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} |
+| [tags](variables.tf#L261) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} |
## Outputs
@@ -480,9 +480,9 @@ module "org" {
| [firewall_policy_id](outputs.tf#L40) | Map of firewall policy ids created in the organization. | |
| [network_tag_keys](outputs.tf#L45) | Tag key resources. | |
| [network_tag_values](outputs.tf#L54) | Tag value resources. | |
-| [organization_id](outputs.tf#L65) | Organization id dependent on module resources. | |
-| [sink_writer_identities](outputs.tf#L82) | Writer identities created for each sink. | |
-| [tag_keys](outputs.tf#L90) | Tag key resources. | |
-| [tag_values](outputs.tf#L99) | Tag value resources. | |
+| [organization_id](outputs.tf#L62) | Organization id dependent on module resources. | |
+| [sink_writer_identities](outputs.tf#L79) | Writer identities created for each sink. | |
+| [tag_keys](outputs.tf#L87) | Tag key resources. | |
+| [tag_values](outputs.tf#L96) | Tag value resources. | |
diff --git a/modules/organization/outputs.tf b/modules/organization/outputs.tf
index 40d84b47..2e594ee6 100644
--- a/modules/organization/outputs.tf
+++ b/modules/organization/outputs.tf
@@ -54,11 +54,8 @@ output "network_tag_keys" {
output "network_tag_values" {
description = "Tag value resources."
value = {
- for k, v in google_tags_tag_value.default
- : k => v if(
- google_tags_tag_key.default[split("/", k)[0]].purpose != null &&
- google_tags_tag_key.default[split("/", k)[0]].purpose != ""
- )
+ for k, v in google_tags_tag_value.default :
+ k => v if local.tag_values[k].tag_network
}
}
@@ -99,10 +96,7 @@ output "tag_keys" {
output "tag_values" {
description = "Tag value resources."
value = {
- for k, v in google_tags_tag_value.default
- : k => v if(
- google_tags_tag_key.default[split("/", k)[0]].purpose == null ||
- google_tags_tag_key.default[split("/", k)[0]].purpose == ""
- )
+ for k, v in google_tags_tag_value.default :
+ k => v if !local.tag_values[k].tag_network
}
}
diff --git a/modules/organization/tags.tf b/modules/organization/tags.tf
index 544b8989..b579479e 100644
--- a/modules/organization/tags.tf
+++ b/modules/organization/tags.tf
@@ -23,17 +23,21 @@ locals {
"Managed by the Terraform organization module."
)
key = "${tag}/${value}"
+ id = try(value_attrs.id, null)
name = value
roles = keys(coalesce(
value_attrs == null ? null : value_attrs.iam, {}
))
- tag = tag
+ tag = tag
+ tag_id = attrs.id
+ tag_network = try(attrs.network, null) != null
}
]
])
_tag_values_iam = flatten([
for key, value_attrs in local.tag_values : [
for role in value_attrs.roles : {
+ id = value_attrs.id
key = value_attrs.key
name = value_attrs.name
role = role
@@ -44,8 +48,9 @@ locals {
_tags_iam = flatten([
for tag, attrs in local.tags : [
for role in keys(coalesce(attrs.iam, {})) : {
- role = role
- tag = tag
+ role = role
+ tag = tag
+ tag_id = attrs.id
}
]
])
@@ -64,7 +69,7 @@ locals {
# keys
resource "google_tags_tag_key" "default" {
- for_each = local.tags
+ for_each = { for k, v in local.tags : k => v if v.id == null }
parent = var.organization_id
purpose = (
lookup(each.value, "network", null) == null ? null : "GCE_FIREWALL"
@@ -83,8 +88,12 @@ resource "google_tags_tag_key" "default" {
resource "google_tags_tag_key_iam_binding" "default" {
for_each = local.tags_iam
- tag_key = google_tags_tag_key.default[each.value.tag].id
- role = each.value.role
+ tag_key = (
+ each.value.tag_id == null
+ ? google_tags_tag_key.default[each.value.tag].id
+ : each.value.tag_id
+ )
+ role = each.value.role
members = coalesce(
local.tags[each.value.tag]["iam"][each.value.role], []
)
@@ -93,16 +102,24 @@ resource "google_tags_tag_key_iam_binding" "default" {
# values
resource "google_tags_tag_value" "default" {
- for_each = local.tag_values
- parent = google_tags_tag_key.default[each.value.tag].id
+ for_each = { for k, v in local.tag_values : k => v if v.id == null }
+ parent = (
+ each.value.tag_id == null
+ ? google_tags_tag_key.default[each.value.tag].id
+ : each.value.tag_id
+ )
short_name = each.value.name
description = each.value.description
}
resource "google_tags_tag_value_iam_binding" "default" {
- for_each = local.tag_values_iam
- tag_value = google_tags_tag_value.default[each.value.key].id
- role = each.value.role
+ for_each = local.tag_values_iam
+ tag_value = (
+ each.value.id == null
+ ? google_tags_tag_value.default[each.value.key].id
+ : each.value.id
+ )
+ role = each.value.role
members = coalesce(
local.tags[each.value.tag]["values"][each.value.name]["iam"][each.value.role],
[]
diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf
index 84c81ff5..ced5cad3 100644
--- a/modules/organization/variables.tf
+++ b/modules/organization/variables.tf
@@ -157,10 +157,11 @@ variable "logging_sinks" {
}
variable "network_tags" {
- description = "Network tags by key name. The `iam` attribute behaves like the similarly named one at module level."
+ description = "Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level."
type = map(object({
description = optional(string, "Managed by the Terraform organization module.")
iam = optional(map(list(string)), {})
+ id = optional(string)
network = string # project_id/vpc_name
values = optional(map(object({
description = optional(string, "Managed by the Terraform organization module.")
@@ -193,7 +194,6 @@ variable "org_policies" {
values = optional(list(string))
}))
enforce = optional(bool, true) # for boolean policies only.
-
# conditional values
rules = optional(list(object({
allow = optional(object({
@@ -259,13 +259,15 @@ variable "tag_bindings" {
}
variable "tags" {
- description = "Tags by key name. The `iam` attribute behaves like the similarly named one at module level."
+ description = "Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level."
type = map(object({
description = optional(string, "Managed by the Terraform organization module.")
iam = optional(map(list(string)), {})
+ id = optional(string)
values = optional(map(object({
description = optional(string, "Managed by the Terraform organization module.")
iam = optional(map(list(string)), {})
+ id = optional(string)
})), {})
}))
nullable = false
diff --git a/modules/organization/versions.tf b/modules/organization/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/organization/versions.tf
+++ b/modules/organization/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/project/versions.tf b/modules/project/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/project/versions.tf
+++ b/modules/project/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/projects-data-source/versions.tf b/modules/projects-data-source/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/projects-data-source/versions.tf
+++ b/modules/projects-data-source/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/pubsub/versions.tf b/modules/pubsub/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/pubsub/versions.tf
+++ b/modules/pubsub/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/secret-manager/versions.tf b/modules/secret-manager/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/secret-manager/versions.tf
+++ b/modules/secret-manager/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/service-directory/versions.tf b/modules/service-directory/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/service-directory/versions.tf
+++ b/modules/service-directory/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/source-repository/versions.tf b/modules/source-repository/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/source-repository/versions.tf
+++ b/modules/source-repository/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/modules/vpc-sc/versions.tf b/modules/vpc-sc/versions.tf
index 4900174a..08492c6f 100644
--- a/modules/vpc-sc/versions.tf
+++ b/modules/vpc-sc/versions.tf
@@ -17,11 +17,11 @@ terraform {
required_providers {
google = {
source = "hashicorp/google"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
- version = ">= 4.48.0" # tftest
+ version = ">= 4.50.0" # tftest
}
}
}
diff --git a/tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/__init__.py b/tests/blueprints/cloud_operations/terraform_enterprise_wif/__init__.py
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/__init__.py
rename to tests/blueprints/cloud_operations/terraform_enterprise_wif/__init__.py
diff --git a/tests/blueprints/factories/bigquery_factory/__init__.py b/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/__init__.py
similarity index 100%
rename from tests/blueprints/factories/bigquery_factory/__init__.py
rename to tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/__init__.py
diff --git a/tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/fixture/main.tf b/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/main.tf
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/fixture/main.tf
rename to tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/main.tf
diff --git a/tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/fixture/variables.tf b/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/variables.tf
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/fixture/variables.tf
rename to tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/variables.tf
diff --git a/tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/test_plan.py b/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/test_plan.py
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform-enterprise-wif/gcp-workload-identity-provider/test_plan.py
rename to tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/test_plan.py
diff --git a/tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/main.tf b/tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/main.tf
index 65cc20ae..3fee8af5 100644
--- a/tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/main.tf
+++ b/tests/blueprints/data_solutions/cmek_via_centralized_kms/fixture/main.tf
@@ -15,7 +15,10 @@
*/
module "test" {
- source = "../../../../../blueprints/data-solutions/cmek-via-centralized-kms/"
- billing_account = var.billing_account
- root_node = var.root_node
+ source = "../../../../../blueprints/data-solutions/cmek-via-centralized-kms/"
+ project_config = {
+ billing_account_id = "123456-123456-123456"
+ parent = "folders/12345678"
+ }
+ prefix = "prefix"
}
diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
index 93961b5e..785f4705 100644
--- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
+++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
@@ -21,5 +21,6 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner(FIXTURES_DIR)
- assert len(modules) == 38
- assert len(resources) == 286
+
+ assert len(modules) == 42
+ assert len(resources) == 296
diff --git a/tests/blueprints/data_solutions/shielded_folder/simple.tfvars b/tests/blueprints/data_solutions/shielded_folder/simple.tfvars
new file mode 100644
index 00000000..83e8b139
--- /dev/null
+++ b/tests/blueprints/data_solutions/shielded_folder/simple.tfvars
@@ -0,0 +1,20 @@
+access_policy_config = {
+ access_policy_create = {
+ parent = "organizations/1234567890123"
+ title = "ShieldedMVP"
+ }
+}
+folder_config = {
+ folder_create = {
+ display_name = "ShieldedMVP"
+ parent = "organizations/1234567890123"
+ }
+}
+organization = {
+ domain = "example.com"
+ id = "1122334455"
+}
+prefix = "prefix"
+project_config = {
+ billing_account_id = "123456-123456-123456"
+}
diff --git a/tests/blueprints/data_solutions/shielded_folder/simple.yaml b/tests/blueprints/data_solutions/shielded_folder/simple.yaml
new file mode 100644
index 00000000..244dcb97
--- /dev/null
+++ b/tests/blueprints/data_solutions/shielded_folder/simple.yaml
@@ -0,0 +1,51 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.folder.google_compute_firewall_policy.policy["prefix-fw-policy"]:
+ short_name: prefix-fw-policy
+ module.folder.google_folder.folder[0]:
+ display_name: ShieldedMVP
+ parent: organizations/1234567890123
+ module.log-export-project[0].google_project.project[0]:
+ billing_account: 123456-123456-123456
+ project_id: prefix-audit-logs
+ module.vpc-sc[0].google_access_context_manager_access_policy.default[0]:
+ parent: organizations/1122334455
+ title: shielded-folder
+ module.vpc-sc[0].google_access_context_manager_service_perimeter.regular["shielded"]:
+ description: null
+ perimeter_type: PERIMETER_TYPE_REGULAR
+ title: shielded
+
+counts:
+ google_access_context_manager_access_policy: 1
+ google_access_context_manager_service_perimeter: 1
+ google_bigquery_dataset: 1
+ google_bigquery_dataset_iam_member: 2
+ google_bigquery_default_service_account: 1
+ google_compute_firewall_policy: 1
+ google_compute_firewall_policy_rule: 4
+ google_folder: 2
+ google_folder_iam_binding: 2
+ google_logging_folder_sink: 2
+ google_org_policy_policy: 12
+ google_project: 1
+ google_project_iam_binding: 1
+ google_project_service: 4
+ google_project_service_identity: 1
+ google_projects: 1
+ google_storage_project_service_account: 1
+ modules: 5
+ resources: 38
diff --git a/tests/modules/gke_nodepool/__init__.py b/tests/blueprints/data_solutions/shielded_folder/tftest.yaml
similarity index 84%
rename from tests/modules/gke_nodepool/__init__.py
rename to tests/blueprints/data_solutions/shielded_folder/tftest.yaml
index 6d6d1266..3c0bcb8c 100644
--- a/tests/modules/gke_nodepool/__init__.py
+++ b/tests/blueprints/data_solutions/shielded_folder/tftest.yaml
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,3 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+module: blueprints/data-solutions/shielded-folder
+
+tests:
+ simple:
diff --git a/tests/fast/stages/s00_bootstrap/__init__.py b/tests/blueprints/data_solutions/vertex_mlops/__init__.py
similarity index 100%
rename from tests/fast/stages/s00_bootstrap/__init__.py
rename to tests/blueprints/data_solutions/vertex_mlops/__init__.py
diff --git a/tests/blueprints/data_solutions/vertex_mlops/fixture/main.tf b/tests/blueprints/data_solutions/vertex_mlops/fixture/main.tf
new file mode 100644
index 00000000..0b671f33
--- /dev/null
+++ b/tests/blueprints/data_solutions/vertex_mlops/fixture/main.tf
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "projects" {
+ source = "../../../../../blueprints/data-solutions/vertex-mlops/"
+ labels = {
+ "env" : "dev",
+ "team" : "ml"
+ }
+ bucket_name = "test-dev"
+ dataset_name = "test"
+ identity_pool_claims = "attribute.repository/ORGANIZATION/REPO"
+ notebooks = {
+ "myworkbench" : {
+ "owner" : "user@example.com",
+ "region" : "europe-west4",
+ "subnet" : "default",
+ }
+ }
+ prefix = "pref"
+ project_id = "test-dev"
+ project_create = {
+ billing_account_id = "000000-123456-123456"
+ parent = "folders/111111111111"
+ }
+}
diff --git a/tests/blueprints/factories/bigquery_factory/test_plan.py b/tests/blueprints/data_solutions/vertex_mlops/test_plan.py
similarity index 71%
rename from tests/blueprints/factories/bigquery_factory/test_plan.py
rename to tests/blueprints/data_solutions/vertex_mlops/test_plan.py
index 74705e42..eac30ad5 100644
--- a/tests/blueprints/factories/bigquery_factory/test_plan.py
+++ b/tests/blueprints/data_solutions/vertex_mlops/test_plan.py
@@ -11,9 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+import pytest
+
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
- modules, resources = e2e_plan_runner()
- assert len(modules) > 0
- assert len(resources) > 0
+ modules, resources = e2e_plan_runner(FIXTURES_DIR)
+ # TODO: to re-enable per-module resource count check print _, then test
+ assert len(modules) > 0 and len(resources) > 0
\ No newline at end of file
diff --git a/tests/blueprints/factories/bigquery_factory/examples/simple.yaml b/tests/blueprints/factories/bigquery_factory/examples/simple.yaml
new file mode 100644
index 00000000..d32492d6
--- /dev/null
+++ b/tests/blueprints/factories/bigquery_factory/examples/simple.yaml
@@ -0,0 +1,40 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.bq.module.bq["my_dataset"].google_bigquery_dataset.default:
+ dataset_id: my_dataset
+ project: project-id
+ module.bq.module.bq["my_dataset"].google_bigquery_table.default["countries"]:
+ dataset_id: my_dataset
+ friendly_name: countries
+ labels:
+ env: prod
+ project: project-id
+ schema: '[{"name":"country","type":"STRING"},{"name":"population","type":"INT64"}]'
+ table_id: countries
+ module.bq.module.bq["my_dataset"].google_bigquery_table.views["department"]:
+ dataset_id: my_dataset
+ friendly_name: department
+ labels:
+ env: prod
+ project: project-id
+ table_id: department
+ view:
+ - query: SELECT SUM(population) from my_dataset.countries
+ use_legacy_sql: false
+
+counts:
+ google_bigquery_dataset: 1
+ google_bigquery_table: 2
diff --git a/tests/fast/stages/s03_data_platform/test_plan.py b/tests/fast/stages/s03_data_platform/test_plan.py
deleted file mode 100644
index 0bb333e7..00000000
--- a/tests/fast/stages/s03_data_platform/test_plan.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def test_counts(plan_summary):
- "Test stage."
- summary = plan_summary("fast/stages/03-data-platform/dev/",
- tf_var_files=["common.tfvars"])
- assert summary.counts["modules"] > 0
- assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s03_gke_multitenant/test_plan.py b/tests/fast/stages/s03_gke_multitenant/test_plan.py
deleted file mode 100644
index 2d196ec4..00000000
--- a/tests/fast/stages/s03_gke_multitenant/test_plan.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def test_counts(plan_summary):
- "Test stage."
- summary = plan_summary("fast/stages/03-gke-multitenant/dev/",
- tf_var_files=["common.tfvars"])
- assert summary.counts["modules"] > 0
- assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s03_project_factory/common.tfvars b/tests/fast/stages/s03_project_factory/common.tfvars
deleted file mode 100644
index b65956b6..00000000
--- a/tests/fast/stages/s03_project_factory/common.tfvars
+++ /dev/null
@@ -1,11 +0,0 @@
-data_dir = "../../../../tests/fast/stages/s03_project_factory/data/projects/"
-defaults_file = "../../../../tests/fast/stages/s03_project_factory/data/defaults.yaml"
-prefix = "test"
-environment_dns_zone = "dev"
-billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
-}
-vpc_self_links = {
- dev-spoke-0 = "link"
-}
diff --git a/tests/fast/stages/s03_project_factory/test_plan.py b/tests/fast/stages/s03_project_factory/test_plan.py
deleted file mode 100644
index 3b284abb..00000000
--- a/tests/fast/stages/s03_project_factory/test_plan.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def test_counts(plan_summary):
- "Test stage."
- summary = plan_summary("fast/stages/03-project-factory/dev",
- tf_var_files=["common.tfvars"])
- assert summary.counts["modules"] > 0
- assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s01_resman/__init__.py b/tests/fast/stages/s0_bootstrap/__init__.py
similarity index 100%
rename from tests/fast/stages/s01_resman/__init__.py
rename to tests/fast/stages/s0_bootstrap/__init__.py
diff --git a/tests/fast/stages/s00_bootstrap/simple.tfvars b/tests/fast/stages/s0_bootstrap/simple.tfvars
similarity index 71%
rename from tests/fast/stages/s00_bootstrap/simple.tfvars
rename to tests/fast/stages/s0_bootstrap/simple.tfvars
index f8ef5735..5c389f53 100644
--- a/tests/fast/stages/s00_bootstrap/simple.tfvars
+++ b/tests/fast/stages/s0_bootstrap/simple.tfvars
@@ -4,8 +4,7 @@ organization = {
customer_id = "C00000000"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
prefix = "fast"
outputs_location = "/fast-config"
diff --git a/tests/fast/stages/s00_bootstrap/simple.yaml b/tests/fast/stages/s0_bootstrap/simple.yaml
similarity index 79%
rename from tests/fast/stages/s00_bootstrap/simple.yaml
rename to tests/fast/stages/s0_bootstrap/simple.yaml
index ed0d7738..63a64cb9 100644
--- a/tests/fast/stages/s00_bootstrap/simple.yaml
+++ b/tests/fast/stages/s0_bootstrap/simple.yaml
@@ -13,23 +13,22 @@
# limitations under the License.
counts:
- google_bigquery_dataset: 2
- google_bigquery_dataset_iam_member: 2
+ google_bigquery_dataset: 1
google_bigquery_default_service_account: 3
google_logging_organization_sink: 2
google_organization_iam_binding: 19
- google_organization_iam_custom_role: 2
+ google_organization_iam_custom_role: 3
google_organization_iam_member: 16
google_project: 3
google_project_iam_binding: 9
- google_project_iam_member: 1
+ google_project_iam_member: 3
google_project_service: 29
google_project_service_identity: 3
- google_service_account: 3
- google_service_account_iam_binding: 3
- google_storage_bucket: 4
- google_storage_bucket_iam_binding: 2
- google_storage_bucket_iam_member: 3
+ google_service_account: 2
+ google_service_account_iam_binding: 1
+ google_storage_bucket: 3
+ google_storage_bucket_iam_binding: 1
+ google_storage_bucket_iam_member: 2
google_storage_bucket_object: 5
google_storage_project_service_account: 3
local_file: 5
@@ -38,6 +37,7 @@ outputs:
custom_roles:
organization_iam_admin: organizations/123456789012/roles/organizationIamAdmin
service_project_network_admin: organizations/123456789012/roles/serviceProjectNetworkAdmin
+ tenant_network_admin: organizations/123456789012/roles/tenantNetworkAdmin
outputs_bucket: fast-prod-iac-core-outputs-0
project_ids:
automation: fast-prod-iac-core-0
@@ -45,5 +45,4 @@ outputs:
log-export: fast-prod-audit-logs-0
service_accounts:
bootstrap: fast-prod-bootstrap-0@fast-prod-iac-core-0.iam.gserviceaccount.com
- cicd: fast-prod-cicd-0@fast-prod-iac-core-0.iam.gserviceaccount.com
resman: fast-prod-resman-0@fast-prod-iac-core-0.iam.gserviceaccount.com
diff --git a/tests/fast/stages/s00_bootstrap/simple_projects.yaml b/tests/fast/stages/s0_bootstrap/simple_projects.yaml
similarity index 100%
rename from tests/fast/stages/s00_bootstrap/simple_projects.yaml
rename to tests/fast/stages/s0_bootstrap/simple_projects.yaml
diff --git a/tests/fast/stages/s00_bootstrap/simple_sas.yaml b/tests/fast/stages/s0_bootstrap/simple_sas.yaml
similarity index 82%
rename from tests/fast/stages/s00_bootstrap/simple_sas.yaml
rename to tests/fast/stages/s0_bootstrap/simple_sas.yaml
index ba84948d..0424e598 100644
--- a/tests/fast/stages/s00_bootstrap/simple_sas.yaml
+++ b/tests/fast/stages/s0_bootstrap/simple_sas.yaml
@@ -17,10 +17,6 @@ values:
account_id: fast-prod-bootstrap-0
display_name: Terraform organization bootstrap service account.
project: fast-prod-iac-core-0
- module.automation-tf-cicd-provisioning-sa.google_service_account.service_account[0]:
- account_id: fast-prod-cicd-0
- display_name: Terraform stage 1 CICD service account.
- project: fast-prod-iac-core-0
module.automation-tf-resman-sa.google_service_account.service_account[0]:
account_id: fast-prod-resman-0
display_name: Terraform stage 1 resman service account.
diff --git a/tests/fast/stages/s00_bootstrap/tftest.yaml b/tests/fast/stages/s0_bootstrap/tftest.yaml
similarity index 83%
rename from tests/fast/stages/s00_bootstrap/tftest.yaml
rename to tests/fast/stages/s0_bootstrap/tftest.yaml
index 4656859b..b53749ad 100644
--- a/tests/fast/stages/s00_bootstrap/tftest.yaml
+++ b/tests/fast/stages/s0_bootstrap/tftest.yaml
@@ -1,6 +1,6 @@
# skip boilerplate check
-module: fast/stages/00-bootstrap
+module: fast/stages/0-bootstrap
tests:
simple:
diff --git a/tests/fast/stages/s02_networking_nva/__init__.py b/tests/fast/stages/s1_resman/__init__.py
similarity index 100%
rename from tests/fast/stages/s02_networking_nva/__init__.py
rename to tests/fast/stages/s1_resman/__init__.py
diff --git a/tests/fast/stages/s01_resman/common.tfvars b/tests/fast/stages/s1_resman/common.tfvars
similarity index 91%
rename from tests/fast/stages/s01_resman/common.tfvars
rename to tests/fast/stages/s1_resman/common.tfvars
index f6d1d5ac..34c61351 100644
--- a/tests/fast/stages/s01_resman/common.tfvars
+++ b/tests/fast/stages/s1_resman/common.tfvars
@@ -6,8 +6,7 @@ automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
custom_roles = {
# organization_iam_admin = "organizations/123456789012/roles/organizationIamAdmin",
diff --git a/tests/fast/stages/s01_resman/test_plan.py b/tests/fast/stages/s1_resman/test_plan.py
similarity index 93%
rename from tests/fast/stages/s01_resman/test_plan.py
rename to tests/fast/stages/s1_resman/test_plan.py
index c8dce750..39bdfc11 100644
--- a/tests/fast/stages/s01_resman/test_plan.py
+++ b/tests/fast/stages/s1_resman/test_plan.py
@@ -15,7 +15,7 @@
def test_counts(plan_summary):
"Test stage."
- summary = plan_summary("fast/stages/01-resman",
+ summary = plan_summary("fast/stages/1-resman",
tf_var_files=["common.tfvars"])
assert summary.counts["modules"] > 0
assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s02_networking_peering/__init__.py b/tests/fast/stages/s2_networking_a_peering/__init__.py
similarity index 100%
rename from tests/fast/stages/s02_networking_peering/__init__.py
rename to tests/fast/stages/s2_networking_a_peering/__init__.py
diff --git a/tests/fast/stages/s02_networking_nva/common.tfvars b/tests/fast/stages/s2_networking_a_peering/common.tfvars
similarity index 81%
rename from tests/fast/stages/s02_networking_nva/common.tfvars
rename to tests/fast/stages/s2_networking_a_peering/common.tfvars
index acfc641f..6c2b0c03 100644
--- a/tests/fast/stages/s02_networking_nva/common.tfvars
+++ b/tests/fast/stages/s2_networking_a_peering/common.tfvars
@@ -1,10 +1,9 @@
-data_dir = "../../../fast/stages/02-networking-nva/data/"
+data_dir = "../../../fast/stages/2-networking-a-peering/data/"
automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
custom_roles = {
service_project_network_admin = "organizations/123456789012/roles/foo"
diff --git a/tests/fast/stages/s02_networking_peering/test_plan.py b/tests/fast/stages/s2_networking_a_peering/test_plan.py
similarity index 87%
rename from tests/fast/stages/s02_networking_peering/test_plan.py
rename to tests/fast/stages/s2_networking_a_peering/test_plan.py
index dff0fd47..09590d36 100644
--- a/tests/fast/stages/s02_networking_peering/test_plan.py
+++ b/tests/fast/stages/s2_networking_a_peering/test_plan.py
@@ -20,16 +20,16 @@ from deepdiff import DeepDiff
BASEDIR = Path(__file__).parent
FIXTURE_PEERING = BASEDIR / 'fixture'
-FIXTURE_VPN = BASEDIR.parent / 's02_networking_vpn/fixture'
+FIXTURE_VPN = BASEDIR.parent / 's2_networking_b_vpn/fixture'
STAGES = Path(__file__).parents[4] / 'fast/stages'
-STAGE_PEERING = STAGES / '02-networking-peering'
-STAGE_VPN = STAGES / '02-networking-vpn'
+STAGE_PEERING = STAGES / '2-networking-a-peering'
+STAGE_VPN = STAGES / '2-networking-b-vpn'
def test_counts(plan_summary):
"Test stage."
- summary = plan_summary("fast/stages/02-networking-peering",
+ summary = plan_summary("fast/stages/2-networking-a-peering",
tf_var_files=["common.tfvars"])
assert summary.counts["modules"] > 0
assert summary.counts["resources"] > 0
@@ -38,9 +38,9 @@ def test_counts(plan_summary):
def test_vpn_peering_parity(plan_summary):
'''Ensure VPN- and peering-based networking stages are identical except
for VPN and VPC peering resources'''
- summary_peering = plan_summary("fast/stages/02-networking-peering",
+ summary_peering = plan_summary("fast/stages/2-networking-a-peering",
tf_var_files=["common.tfvars"])
- summary_vpn = plan_summary("fast/stages/02-networking-vpn",
+ summary_vpn = plan_summary("fast/stages/2-networking-b-vpn",
tf_var_files=["common.tfvars"])
ddiff = DeepDiff(summary_vpn.values, summary_peering.values,
diff --git a/tests/fast/stages/s02_networking_separate_envs/__init__.py b/tests/fast/stages/s2_networking_b_vpn/__init__.py
similarity index 100%
rename from tests/fast/stages/s02_networking_separate_envs/__init__.py
rename to tests/fast/stages/s2_networking_b_vpn/__init__.py
diff --git a/tests/fast/stages/s02_networking_peering/common.tfvars b/tests/fast/stages/s2_networking_b_vpn/common.tfvars
similarity index 83%
rename from tests/fast/stages/s02_networking_peering/common.tfvars
rename to tests/fast/stages/s2_networking_b_vpn/common.tfvars
index 11b49d7c..66a7a609 100644
--- a/tests/fast/stages/s02_networking_peering/common.tfvars
+++ b/tests/fast/stages/s2_networking_b_vpn/common.tfvars
@@ -1,10 +1,9 @@
-data_dir = "../../../fast/stages/02-networking-peering/data/"
+data_dir = "../../../../../fast/stages/2-networking-b-vpn/data/"
automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
custom_roles = {
service_project_network_admin = "organizations/123456789012/roles/foo"
diff --git a/tests/fast/stages/s02_networking_vpn/fixture/main.tf b/tests/fast/stages/s2_networking_b_vpn/fixture/main.tf
similarity index 100%
rename from tests/fast/stages/s02_networking_vpn/fixture/main.tf
rename to tests/fast/stages/s2_networking_b_vpn/fixture/main.tf
diff --git a/tests/fast/stages/s02_networking_vpn/test_plan.py b/tests/fast/stages/s2_networking_b_vpn/test_plan.py
similarity index 92%
rename from tests/fast/stages/s02_networking_vpn/test_plan.py
rename to tests/fast/stages/s2_networking_b_vpn/test_plan.py
index cc62dbf7..8ac1bade 100644
--- a/tests/fast/stages/s02_networking_vpn/test_plan.py
+++ b/tests/fast/stages/s2_networking_b_vpn/test_plan.py
@@ -15,7 +15,7 @@
def test_counts(plan_summary):
"Test stage."
- summary = plan_summary("fast/stages/02-networking-vpn",
+ summary = plan_summary("fast/stages/2-networking-b-vpn",
tf_var_files=["common.tfvars"])
assert summary.counts["modules"] > 0
assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s02_networking_vpn/__init__.py b/tests/fast/stages/s2_networking_c_nva/__init__.py
similarity index 100%
rename from tests/fast/stages/s02_networking_vpn/__init__.py
rename to tests/fast/stages/s2_networking_c_nva/__init__.py
diff --git a/tests/fast/stages/s02_networking_vpn/common.tfvars b/tests/fast/stages/s2_networking_c_nva/common.tfvars
similarity index 72%
rename from tests/fast/stages/s02_networking_vpn/common.tfvars
rename to tests/fast/stages/s2_networking_c_nva/common.tfvars
index 7241594d..ad12b8d3 100644
--- a/tests/fast/stages/s02_networking_vpn/common.tfvars
+++ b/tests/fast/stages/s2_networking_c_nva/common.tfvars
@@ -1,10 +1,9 @@
-data_dir = "../../../../../fast/stages/02-networking-vpn/data/"
+data_dir = "../../../fast/stages/2-networking-c-nva/data/"
automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
custom_roles = {
service_project_network_admin = "organizations/123456789012/roles/foo"
@@ -14,11 +13,6 @@ folder_ids = {
networking-dev = null
networking-prod = null
}
-region_trigram = {
- europe-west1 = "ew1"
- europe-west3 = "ew3"
- europe-west8 = "ew8"
-}
service_accounts = {
data-platform-dev = "string"
data-platform-prod = "string"
diff --git a/tests/fast/stages/s02_networking_nva/test_plan.py b/tests/fast/stages/s2_networking_c_nva/test_plan.py
similarity index 92%
rename from tests/fast/stages/s02_networking_nva/test_plan.py
rename to tests/fast/stages/s2_networking_c_nva/test_plan.py
index 24964f7a..70c37bc3 100644
--- a/tests/fast/stages/s02_networking_nva/test_plan.py
+++ b/tests/fast/stages/s2_networking_c_nva/test_plan.py
@@ -15,7 +15,7 @@
def test_counts(plan_summary):
"Test stage."
- summary = plan_summary("fast/stages/02-networking-nva",
+ summary = plan_summary("fast/stages/2-networking-c-nva",
tf_var_files=["common.tfvars"])
assert summary.counts["modules"] > 0
assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s02_security/__init__.py b/tests/fast/stages/s2_networking_d_separate_envs/__init__.py
similarity index 100%
rename from tests/fast/stages/s02_security/__init__.py
rename to tests/fast/stages/s2_networking_d_separate_envs/__init__.py
diff --git a/tests/fast/stages/s02_networking_separate_envs/common.tfvars b/tests/fast/stages/s2_networking_d_separate_envs/common.tfvars
similarity index 78%
rename from tests/fast/stages/s02_networking_separate_envs/common.tfvars
rename to tests/fast/stages/s2_networking_d_separate_envs/common.tfvars
index c6b793fd..3ff0020a 100644
--- a/tests/fast/stages/s02_networking_separate_envs/common.tfvars
+++ b/tests/fast/stages/s2_networking_d_separate_envs/common.tfvars
@@ -1,10 +1,9 @@
-data_dir = "../../../../../fast/stages/02-networking-separate-envs/data/"
+data_dir = "../../../../../fast/stages/2-networking-d-separate-envs/data/"
automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
custom_roles = {
service_project_network_admin = "organizations/123456789012/roles/foo"
diff --git a/tests/fast/stages/s02_networking_separate_envs/test_plan.py b/tests/fast/stages/s2_networking_d_separate_envs/test_plan.py
similarity index 91%
rename from tests/fast/stages/s02_networking_separate_envs/test_plan.py
rename to tests/fast/stages/s2_networking_d_separate_envs/test_plan.py
index 89f6e257..51a9257a 100644
--- a/tests/fast/stages/s02_networking_separate_envs/test_plan.py
+++ b/tests/fast/stages/s2_networking_d_separate_envs/test_plan.py
@@ -15,7 +15,7 @@
def test_counts(plan_summary):
"Test stage."
- summary = plan_summary("fast/stages/02-networking-separate-envs",
+ summary = plan_summary("fast/stages/2-networking-d-separate-envs",
tf_var_files=["common.tfvars"])
assert summary.counts["modules"] > 0
assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s03_data_platform/__init__.py b/tests/fast/stages/s2_security/__init__.py
similarity index 100%
rename from tests/fast/stages/s03_data_platform/__init__.py
rename to tests/fast/stages/s2_security/__init__.py
diff --git a/tests/fast/stages/s02_security/common.tfvars b/tests/fast/stages/s2_security/common.tfvars
similarity index 91%
rename from tests/fast/stages/s02_security/common.tfvars
rename to tests/fast/stages/s2_security/common.tfvars
index b480a673..6fbb60b6 100644
--- a/tests/fast/stages/s02_security/common.tfvars
+++ b/tests/fast/stages/s2_security/common.tfvars
@@ -2,16 +2,15 @@ automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "000000-111111-222222"
- organization_id = 123456789012
+ id = "000000-111111-222222"
}
folder_ids = {
security = null
}
organization = {
- domain = "gcp-pso-italy.net"
- id = 856933387836
- customer_id = "C01lmug8b"
+ domain = "fast.example.com"
+ id = 123456789012
+ customer_id = "C00000000"
}
prefix = "fast"
kms_keys = {
diff --git a/tests/fast/stages/s02_security/test_plan.py b/tests/fast/stages/s2_security/test_plan.py
similarity index 93%
rename from tests/fast/stages/s02_security/test_plan.py
rename to tests/fast/stages/s2_security/test_plan.py
index 004f6b83..edf5622e 100644
--- a/tests/fast/stages/s02_security/test_plan.py
+++ b/tests/fast/stages/s2_security/test_plan.py
@@ -15,7 +15,7 @@
def test_counts(plan_summary):
"Test stage."
- summary = plan_summary("fast/stages/02-security",
+ summary = plan_summary("fast/stages/2-security",
tf_var_files=["common.tfvars"])
assert summary.counts["modules"] > 0
assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s03_gke_multitenant/__init__.py b/tests/fast/stages/s3_data_platform/__init__.py
similarity index 100%
rename from tests/fast/stages/s03_gke_multitenant/__init__.py
rename to tests/fast/stages/s3_data_platform/__init__.py
diff --git a/tests/fast/stages/s03_data_platform/common.tfvars b/tests/fast/stages/s3_data_platform/common.tfvars
similarity index 85%
rename from tests/fast/stages/s03_data_platform/common.tfvars
rename to tests/fast/stages/s3_data_platform/common.tfvars
index f5aada16..2ec41d37 100644
--- a/tests/fast/stages/s03_data_platform/common.tfvars
+++ b/tests/fast/stages/s3_data_platform/common.tfvars
@@ -2,8 +2,7 @@ automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "012345-67890A-BCDEF0",
- organization_id = 123456
+ id = "012345-67890A-BCDEF0",
}
folder_ids = {
data-platform-dev = "folders/12345678"
@@ -12,9 +11,9 @@ host_project_ids = {
dev-spoke-0 = "fast-dev-net-spoke-0"
}
organization = {
- domain = "example.com"
+ domain = "fast.example.com"
id = 123456789012
- customer_id = "A11aaaaa1"
+ customer_id = "C00000000"
}
prefix = "fast"
subnet_self_links = {
diff --git a/tests/fast/stages/s3_data_platform/test_plan.py b/tests/fast/stages/s3_data_platform/test_plan.py
new file mode 100644
index 00000000..ad7fa3d2
--- /dev/null
+++ b/tests/fast/stages/s3_data_platform/test_plan.py
@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def test_counts(plan_summary):
+ "Test stage."
+ summary = plan_summary("fast/stages/3-data-platform/dev/",
+ tf_var_files=["common.tfvars"])
+ assert summary.counts["modules"] > 0
+ assert summary.counts["resources"] > 0
diff --git a/tests/fast/stages/s03_project_factory/__init__.py b/tests/fast/stages/s3_gke_multitenant/__init__.py
similarity index 100%
rename from tests/fast/stages/s03_project_factory/__init__.py
rename to tests/fast/stages/s3_gke_multitenant/__init__.py
diff --git a/tests/fast/stages/s03_gke_multitenant/common.tfvars b/tests/fast/stages/s3_gke_multitenant/common.tfvars
similarity index 92%
rename from tests/fast/stages/s03_gke_multitenant/common.tfvars
rename to tests/fast/stages/s3_gke_multitenant/common.tfvars
index d22db1d2..1cafdd9a 100644
--- a/tests/fast/stages/s03_gke_multitenant/common.tfvars
+++ b/tests/fast/stages/s3_gke_multitenant/common.tfvars
@@ -2,8 +2,7 @@ automation = {
outputs_bucket = "test"
}
billing_account = {
- id = "012345-67890A-BCDEF0",
- organization_id = 123456
+ id = "012345-67890A-BCDEF0",
}
clusters = {
mycluster = {
diff --git a/tests/fast/stages/s3_gke_multitenant/test_plan.py b/tests/fast/stages/s3_gke_multitenant/test_plan.py
new file mode 100644
index 00000000..c517cb93
--- /dev/null
+++ b/tests/fast/stages/s3_gke_multitenant/test_plan.py
@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def test_counts(plan_summary):
+ "Test stage."
+ summary = plan_summary("fast/stages/3-gke-multitenant/dev/",
+ tf_var_files=["common.tfvars"])
+ assert summary.counts["modules"] > 0
+ assert summary.counts["resources"] > 0
diff --git a/tests/modules/api_gateway/__init__.py b/tests/fast/stages/s3_project_factory/__init__.py
similarity index 100%
rename from tests/modules/api_gateway/__init__.py
rename to tests/fast/stages/s3_project_factory/__init__.py
diff --git a/tests/fast/stages/s3_project_factory/common.tfvars b/tests/fast/stages/s3_project_factory/common.tfvars
new file mode 100644
index 00000000..d3f8c6f9
--- /dev/null
+++ b/tests/fast/stages/s3_project_factory/common.tfvars
@@ -0,0 +1,10 @@
+data_dir = "../../../../tests/fast/stages/s3_project_factory/data/projects/"
+defaults_file = "../../../../tests/fast/stages/s3_project_factory/data/defaults.yaml"
+prefix = "test"
+environment_dns_zone = "dev"
+billing_account = {
+ id = "000000-111111-222222"
+}
+vpc_self_links = {
+ dev-spoke-0 = "link"
+}
diff --git a/tests/fast/stages/s03_project_factory/data/defaults.yaml b/tests/fast/stages/s3_project_factory/data/defaults.yaml
similarity index 100%
rename from tests/fast/stages/s03_project_factory/data/defaults.yaml
rename to tests/fast/stages/s3_project_factory/data/defaults.yaml
diff --git a/tests/fast/stages/s03_project_factory/data/projects/project.yaml b/tests/fast/stages/s3_project_factory/data/projects/project.yaml
similarity index 100%
rename from tests/fast/stages/s03_project_factory/data/projects/project.yaml
rename to tests/fast/stages/s3_project_factory/data/projects/project.yaml
diff --git a/tests/fast/stages/s3_project_factory/test_plan.py b/tests/fast/stages/s3_project_factory/test_plan.py
new file mode 100644
index 00000000..fa293da8
--- /dev/null
+++ b/tests/fast/stages/s3_project_factory/test_plan.py
@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def test_counts(plan_summary):
+ "Test stage."
+ summary = plan_summary("fast/stages/3-project-factory/dev",
+ tf_var_files=["common.tfvars"])
+ assert summary.counts["modules"] > 0
+ assert summary.counts["resources"] > 0
diff --git a/tests/modules/compute_vm/__init__.py b/tests/fast/stages_multitenant/__init__.py
similarity index 100%
rename from tests/modules/compute_vm/__init__.py
rename to tests/fast/stages_multitenant/__init__.py
diff --git a/tests/modules/dns/__init__.py b/tests/fast/stages_multitenant/s0_bootstrap_tenant/__init__.py
similarity index 100%
rename from tests/modules/dns/__init__.py
rename to tests/fast/stages_multitenant/s0_bootstrap_tenant/__init__.py
diff --git a/tests/fast/stages_multitenant/s0_bootstrap_tenant/simple.tfvars b/tests/fast/stages_multitenant/s0_bootstrap_tenant/simple.tfvars
new file mode 100644
index 00000000..52ca76a3
--- /dev/null
+++ b/tests/fast/stages_multitenant/s0_bootstrap_tenant/simple.tfvars
@@ -0,0 +1,61 @@
+automation = {
+ federated_identity_pool = null
+ federated_identity_providers = null
+ project_id = "fast-prod-automation"
+ project_number = 123456
+ outputs_bucket = "test"
+}
+billing_account = {
+ id = "000000-111111-222222"
+}
+custom_roles = {
+ # organization_iam_admin = "organizations/123456789012/roles/organizationIamAdmin",
+ service_project_network_admin = "organizations/123456789012/roles/xpnServiceAdmin"
+ tenant_network_admin = "organizations/123456789012/roles/TenantNetworkAdmin"
+}
+groups = {
+ gcp-billing-admins = "gcp-billing-admins",
+ gcp-devops = "gcp-devops",
+ gcp-network-admins = "gcp-network-admins",
+ gcp-organization-admins = "gcp-organization-admins",
+ gcp-security-admins = "gcp-security-admins",
+ gcp-support = "gcp-support"
+}
+organization = {
+ domain = "fast.example.com"
+ id = 123456789012
+ customer_id = "C00000000"
+}
+prefix = "fast2"
+tag_keys = {
+ context = "tagKeys/1234567890"
+ environment = "tagKeys/4567890123"
+ tenant = "tagKeys/7890123456"
+}
+tag_names = {
+ context = "context"
+ environment = "environment"
+ tenant = "tenant"
+}
+tag_values = {
+ "context/data" : "tagValues/1234567890",
+ "context/gke" : "tagValues/1234567890",
+ "context/networking" : "tagValues/1234567890",
+ "context/sandbox" : "tagValues/1234567890",
+ "context/security" : "tagValues/1234567890",
+ "context/teams" : "tagValues/1234567890",
+ "environment/development" : "tagValues/1234567890",
+ "environment/production" : "tagValues/1234567890"
+}
+tenant_config = {
+ groups = {
+ gcp-admins = "gcp-tn01-admins"
+ }
+ descriptive_name = "Tenant 01"
+ locations = {
+ gcs = "europe-west8"
+ logging = "europe-west8"
+ }
+ short_name = "tn01"
+}
+test_principal = "foo-prod-resman-0@foo-prod-iac-core-0.iam.gserviceaccount.com"
diff --git a/tests/blueprints/factories/bigquery_factory/fixture/views/view_a.yaml b/tests/fast/stages_multitenant/s0_bootstrap_tenant/simple.yaml
similarity index 50%
rename from tests/blueprints/factories/bigquery_factory/fixture/views/view_a.yaml
rename to tests/fast/stages_multitenant/s0_bootstrap_tenant/simple.yaml
index 23c41b98..e5ccc4fd 100644
--- a/tests/blueprints/factories/bigquery_factory/fixture/views/view_a.yaml
+++ b/tests/fast/stages_multitenant/s0_bootstrap_tenant/simple.yaml
@@ -12,6 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-dataset: dataset_b
-view: view_a
-query: "SELECT CURRENT_DATE() LIMIT 1"
+counts:
+ google_bigquery_default_service_account: 2
+ google_folder: 2
+ google_folder_iam_binding: 5
+ google_organization_iam_member: 39
+ google_project: 2
+ google_project_iam_binding: 8
+ google_project_service: 26
+ google_project_service_identity: 3
+ google_service_account: 11
+ google_storage_bucket: 2
+ google_storage_bucket_iam_binding: 1
+ google_storage_bucket_iam_member: 1
+ google_storage_bucket_object: 2
+ google_storage_project_service_account: 2
+ google_tags_tag_binding: 1
+ google_tags_tag_value: 1
+ modules: 19
+ resources: 129
diff --git a/tests/fast/stages_multitenant/s0_bootstrap_tenant/tftest.yaml b/tests/fast/stages_multitenant/s0_bootstrap_tenant/tftest.yaml
new file mode 100644
index 00000000..c2fa9fa8
--- /dev/null
+++ b/tests/fast/stages_multitenant/s0_bootstrap_tenant/tftest.yaml
@@ -0,0 +1,10 @@
+# skip boilerplate check
+
+module: fast/stages-multitenant/0-bootstrap-tenant
+
+tests:
+ simple:
+ tfvars:
+ - simple.tfvars
+ inventory:
+ - simple.yaml
diff --git a/tests/modules/gcs/__init__.py b/tests/fast/stages_multitenant/s1_resman_tenant/__init__.py
similarity index 100%
rename from tests/modules/gcs/__init__.py
rename to tests/fast/stages_multitenant/s1_resman_tenant/__init__.py
diff --git a/tests/fast/stages_multitenant/s1_resman_tenant/simple.tfvars b/tests/fast/stages_multitenant/s1_resman_tenant/simple.tfvars
new file mode 100644
index 00000000..33cf4619
--- /dev/null
+++ b/tests/fast/stages_multitenant/s1_resman_tenant/simple.tfvars
@@ -0,0 +1,70 @@
+automation = {
+ federated_identity_pools = null
+ federated_identity_providers = null
+ project_id = "tn0-prod-automation-0"
+ project_number = 123456
+ outputs_bucket = "tn0-prod-automation-0"
+ service_accounts = {
+ networking = "foo-tn0-net-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ resman = "foo-tn0-resman-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ security = "foo-tn0-sec-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ dp-dev = "foo-tn0-dp-dev-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ dp-prod = "foo-tn0-dp-prod-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ gke-dev = "foo-tn0-gke-dev-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ gke-prod = "foo-tn0-gke-prod-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ pf-dev = "foo-tn0-pf-dev-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ pf-prod = "foo-tn0-pf-prod-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ sandbox = "foo-tn0-sandbox-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ teams = "foo-tn0-teams-0@foo-tn0-prod-iac-core-0.iam.gserviceaccount.com"
+ }
+}
+billing_account = {
+ id = "000000-111111-222222"
+}
+custom_roles = {
+ # organization_iam_admin = "organizations/123456789012/roles/organizationIamAdmin",
+ service_project_network_admin = "organizations/123456789012/roles/xpnServiceAdmin"
+}
+fast_features = {
+ data_platform = true
+ gke = true
+ project_factory = true
+ sandbox = true
+ teams = true
+}
+groups = {
+ gcp-devops = "gcp-devops",
+ gcp-network-admins = "gcp-network-admins",
+ gcp-security-admins = "gcp-security-admins",
+}
+organization = {
+ domain = "fast.example.com"
+ id = 123456789012
+ customer_id = "C00000000"
+}
+prefix = "foo-tn0"
+root_node = "folders/1234567890"
+short_name = "tn0"
+tags = {
+ keys = {
+ context = "tagKeys/1234567890"
+ environment = "tagKeys/4567890123"
+ tenant = "tagKeys/7890123456"
+ }
+ names = {
+ context = "context"
+ environment = "environment"
+ tenant = "tenant"
+ }
+ values = {
+ "context/data" : "tagValues/1234567890",
+ "context/gke" : "tagValues/1234567890",
+ "context/networking" : "tagValues/1234567890",
+ "context/sandbox" : "tagValues/1234567890",
+ "context/security" : "tagValues/1234567890",
+ "context/teams" : "tagValues/1234567890",
+ "environment/development" : "tagValues/1234567890",
+ "environment/production" : "tagValues/1234567890"
+ }
+}
+test_skip_data_sources = true
diff --git a/tests/blueprints/factories/bigquery_factory/fixture/tables/table_a.yaml b/tests/fast/stages_multitenant/s1_resman_tenant/simple.yaml
similarity index 59%
rename from tests/blueprints/factories/bigquery_factory/fixture/tables/table_a.yaml
rename to tests/fast/stages_multitenant/s1_resman_tenant/simple.yaml
index 05adbcb0..44c07c62 100644
--- a/tests/blueprints/factories/bigquery_factory/fixture/tables/table_a.yaml
+++ b/tests/fast/stages_multitenant/s1_resman_tenant/simple.yaml
@@ -12,6 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-dataset: dataset_a
-table: table_a
-schema: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}]
+counts:
+ google_folder: 13
+ google_folder_iam_binding: 42
+ google_folder_iam_member: 3
+ google_org_policy_policy: 2
+ google_service_account: 9
+ google_service_account_iam_binding: 8
+ google_storage_bucket: 10
+ google_storage_bucket_iam_binding: 10
+ google_storage_bucket_iam_member: 9
+ google_storage_bucket_object: 11
+ google_tags_tag_binding: 12
+ modules: 32
+ resources: 129
diff --git a/tests/fast/stages_multitenant/s1_resman_tenant/tftest.yaml b/tests/fast/stages_multitenant/s1_resman_tenant/tftest.yaml
new file mode 100644
index 00000000..2e107e08
--- /dev/null
+++ b/tests/fast/stages_multitenant/s1_resman_tenant/tftest.yaml
@@ -0,0 +1,10 @@
+# skip boilerplate check
+
+module: fast/stages-multitenant/1-resman-tenant
+
+tests:
+ simple:
+ tfvars:
+ - simple.tfvars
+ inventory:
+ - simple.yaml
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 788f8178..11a397de 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -31,7 +31,7 @@ PlanSummary = collections.namedtuple('PlanSummary', 'values counts outputs')
@contextlib.contextmanager
def _prepare_root_module(path):
"""Context manager to prepare a terraform module to be tested.
-
+
If the TFTEST_COPY environment variable is set, `path` is copied to
a temporary directory and a few terraform files (e.g.
terraform.tfvars) are delete to ensure a clean test environment.
@@ -49,6 +49,7 @@ def _prepare_root_module(path):
# deployment with links to configs)
ignore_patterns = shutil.ignore_patterns('*.auto.tfvars',
'*.auto.tfvars.json',
+ '[0-9]-*-providers.tf',
'terraform.tfstate*',
'terraform.tfvars', '.terraform')
@@ -180,19 +181,19 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
expected_values = inventory['values']
for address, expected_value in expected_values.items():
assert address in summary.values, \
- f'{address} is not a valid address in the plan'
+ f'{address} is not a valid address in the plan'
for k, v in expected_value.items():
assert k in summary.values[address], \
- f'{k} not found at {address}'
+ f'{k} not found at {address}'
plan_value = summary.values[address][k]
assert plan_value == v, \
- f'{k} at {address} failed. Got `{plan_value}`, expected `{v}`'
+ f'{k} at {address} failed. Got `{plan_value}`, expected `{v}`'
if 'counts' in inventory:
expected_counts = inventory['counts']
for type_, expected_count in expected_counts.items():
assert type_ in summary.counts, \
- f'module does not create any resources of type `{type_}`'
+ f'module does not create any resources of type `{type_}`'
plan_count = summary.counts[type_]
assert plan_count == expected_count, \
f'count of {type_} resources failed. Got {plan_count}, expected {expected_count}'
@@ -201,7 +202,7 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
expected_outputs = inventory['outputs']
for output_name, expected_output in expected_outputs.items():
assert output_name in summary.outputs, \
- f'module does not output `{output_name}`'
+ f'module does not output `{output_name}`'
output = summary.outputs[output_name]
# assert 'value' in output, \
# f'output `{output_name}` does not have a value (is it sensitive or dynamic?)'
diff --git a/tests/modules/compute_vm/examples/alias-ips.yaml b/tests/modules/compute_vm/examples/alias-ips.yaml
new file mode 100644
index 00000000..016f9660
--- /dev/null
+++ b/tests/modules/compute_vm/examples/alias-ips.yaml
@@ -0,0 +1,36 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.vm-with-alias-ips.google_compute_instance.default[0]:
+ name: test
+ network_interface:
+ - access_config: []
+ alias_ip_range:
+ - ip_cidr_range: 10.16.0.10/32
+ subnetwork_range_name: alias1
+ ipv6_access_config: []
+ network: projects/xxx/global/networks/aaa
+ nic_type: null
+ queue_count: null
+ subnetwork: subnet_self_link
+ project: my-project
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 1
+ modules: 1
+ resources: 1
+
+outputs: {}
diff --git a/tests/modules/compute_vm/examples/cmek.yaml b/tests/modules/compute_vm/examples/cmek.yaml
new file mode 100644
index 00000000..cf390fde
--- /dev/null
+++ b/tests/modules/compute_vm/examples/cmek.yaml
@@ -0,0 +1,57 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.kms-vm-example.google_compute_disk.disks["attached-disk"]:
+ disk_encryption_key:
+ - kms_key_self_link: kms_key_self_link
+ kms_key_service_account: null
+ raw_key: null
+ labels:
+ disk_name: attached-disk
+ disk_type: pd-balanced
+ name: kms-test-attached-disk
+ project: project-id
+ size: 10
+ type: pd-balanced
+ zone: europe-west1-b
+ module.kms-vm-example.google_compute_instance.default[0]:
+ attached_disk:
+ - device_name: attached-disk
+ disk_encryption_key_raw: null
+ mode: READ_WRITE
+ source: kms-test-attached-disk
+ boot_disk:
+ - auto_delete: true
+ disk_encryption_key_raw: null
+ initialize_params:
+ - image: projects/debian-cloud/global/images/family/debian-10
+ size: 10
+ type: pd-balanced
+ kms_key_self_link: kms_key_self_link
+ mode: READ_WRITE
+ name: kms-test
+ zone: europe-west1-b
+ module.kms-vm-example.google_service_account.service_account[0]:
+ account_id: tf-vm-kms-test
+ description: null
+ disabled: false
+ display_name: Terraform VM kms-test.
+ project: project-id
+ timeouts: null
+
+counts:
+ google_compute_disk: 1
+ google_compute_instance: 1
+ google_service_account: 1
diff --git a/tests/modules/compute_vm/examples/confidential.yaml b/tests/modules/compute_vm/examples/confidential.yaml
new file mode 100644
index 00000000..e842d4cb
--- /dev/null
+++ b/tests/modules/compute_vm/examples/confidential.yaml
@@ -0,0 +1,31 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.template-confidential-example.google_compute_instance_template.default[0]:
+ confidential_instance_config:
+ - enable_confidential_compute: true
+ name_prefix: confidential-template-
+ project: project-id
+ region: europe-west1
+ module.vm-confidential-example.google_compute_instance.default[0]:
+ confidential_instance_config:
+ - enable_confidential_compute: true
+ name: confidential-vm
+ project: project-id
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 1
+ google_compute_instance_template: 1
diff --git a/tests/modules/compute_vm/examples/disk-options.yaml b/tests/modules/compute_vm/examples/disk-options.yaml
new file mode 100644
index 00000000..91c11b41
--- /dev/null
+++ b/tests/modules/compute_vm/examples/disk-options.yaml
@@ -0,0 +1,59 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.vm-disk-options-example.google_compute_disk.disks["data2"]:
+ name: test-data2
+ project: project-id
+ size: 20
+ snapshot: snapshot-2
+ type: pd-ssd
+ zone: europe-west1-b
+ module.vm-disk-options-example.google_compute_instance.default[0]:
+ attached_disk:
+ - device_name: data2
+ disk_encryption_key_raw: null
+ mode: READ_ONLY
+ source: test-data2
+ - device_name: data1
+ disk_encryption_key_raw: null
+ mode: READ_WRITE
+ source: test-data1
+ boot_disk:
+ - auto_delete: true
+ disk_encryption_key_raw: null
+ initialize_params:
+ - image: projects/debian-cloud/global/images/family/debian-11
+ size: 10
+ type: pd-balanced
+ mode: READ_WRITE
+ description: Managed by the compute-vm Terraform module.
+ name: test
+ project: project-id
+ zone: europe-west1-b
+ module.vm-disk-options-example.google_compute_region_disk.disks["data1"]:
+ name: test-data1
+ project: project-id
+ region: europe-west1
+ replica_zones:
+ - europe-west1-b
+ - europe-west1-c
+ size: 10
+ type: pd-balanced
+
+counts:
+ google_compute_disk: 1
+ google_compute_instance: 1
+ google_compute_region_disk: 1
+ google_service_account: 1
diff --git a/tests/modules/compute_vm/examples/group.yaml b/tests/modules/compute_vm/examples/group.yaml
new file mode 100644
index 00000000..c28c4764
--- /dev/null
+++ b/tests/modules/compute_vm/examples/group.yaml
@@ -0,0 +1,27 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.instance-group.google_compute_instance.default[0]: {}
+ module.instance-group.google_compute_instance_group.unmanaged[0]:
+ name: ilb-test
+ named_port: []
+ network: projects/xxx/global/networks/aaa
+ project: my-project
+ timeouts: null
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 1
+ google_compute_instance_group: 1
diff --git a/tests/modules/compute_vm/examples/gvnic.yaml b/tests/modules/compute_vm/examples/gvnic.yaml
new file mode 100644
index 00000000..da95de9e
--- /dev/null
+++ b/tests/modules/compute_vm/examples/gvnic.yaml
@@ -0,0 +1,43 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ google_compute_image.cos-gvnic:
+ guest_os_features:
+ - type: GVNIC
+ - type: SEV_CAPABLE
+ - type: UEFI_COMPATIBLE
+ - type: VIRTIO_SCSI_MULTIQUEUE
+ name: my-image
+ project: my-project
+ source_image: https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-89-16108-534-18
+ module.vm-with-gvnic.google_compute_instance.default[0]:
+ name: test
+ network_interface:
+ - access_config: []
+ alias_ip_range: []
+ ipv6_access_config: []
+ network: projects/xxx/global/networks/aaa
+ nic_type: GVNIC
+ queue_count: null
+ subnetwork: subnet_self_link
+ project: my-project
+ zone: europe-west1-b
+
+counts:
+ google_compute_image: 1
+ google_compute_instance: 1
+ google_service_account: 1
+ modules: 1
+ resources: 3
diff --git a/tests/modules/compute_vm/examples/iam.yaml b/tests/modules/compute_vm/examples/iam.yaml
new file mode 100644
index 00000000..254d266d
--- /dev/null
+++ b/tests/modules/compute_vm/examples/iam.yaml
@@ -0,0 +1,34 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.vm-iam-example.google_compute_instance.default[0]:
+ name: webserver
+ module.vm-iam-example.google_compute_instance_iam_binding.default["roles/compute.instanceAdmin"]:
+ condition: []
+ instance_name: webserver
+ members:
+ - group:admin@example.com
+ - group:webserver@example.com
+ project: project-id
+ role: roles/compute.instanceAdmin
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 1
+ google_compute_instance_iam_binding: 1
+ modules: 1
+ resources: 2
+
+outputs: {}
diff --git a/tests/modules/compute_vm/examples/ips.yaml b/tests/modules/compute_vm/examples/ips.yaml
new file mode 100644
index 00000000..65931abb
--- /dev/null
+++ b/tests/modules/compute_vm/examples/ips.yaml
@@ -0,0 +1,45 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.vm-external-ip.google_compute_instance.default[0]:
+ name: vm-external-ip
+ network_interface:
+ - access_config:
+ - nat_ip: 8.8.8.8
+ public_ptr_domain_name: null
+ alias_ip_range: []
+ ipv6_access_config: []
+ network: projects/xxx/global/networks/aaa
+ nic_type: null
+ queue_count: null
+ subnetwork: subnet_self_link
+ project: my-project
+ zone: europe-west1-b
+ module.vm-internal-ip.google_compute_instance.default[0]:
+ name: vm-internal-ip
+ network_interface:
+ - access_config: []
+ alias_ip_range: []
+ ipv6_access_config: []
+ network: projects/xxx/global/networks/aaa
+ network_ip: 10.0.0.2
+ nic_type: null
+ queue_count: null
+ subnetwork: subnet_self_link
+ project: my-project
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 2
diff --git a/tests/modules/compute_vm/examples/metadata.yaml b/tests/modules/compute_vm/examples/metadata.yaml
new file mode 100644
index 00000000..fbe0d06f
--- /dev/null
+++ b/tests/modules/compute_vm/examples/metadata.yaml
@@ -0,0 +1,32 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.vm-metadata-example.google_compute_instance.default[0]:
+ metadata:
+ startup-script: |
+ #! /bin/bash
+ apt-get update
+ apt-get install -y nginx
+ name: nginx-server
+ project: project-id
+ zone: europe-west1-b
+ labels:
+ env: dev
+ system: crm
+ module.vm-metadata-example.google_service_account.service_account[0]: {}
+
+counts:
+ google_compute_instance: 1
+ google_service_account: 1
diff --git a/tests/modules/compute_vm/examples/sas.yaml b/tests/modules/compute_vm/examples/sas.yaml
new file mode 100644
index 00000000..96a94831
--- /dev/null
+++ b/tests/modules/compute_vm/examples/sas.yaml
@@ -0,0 +1,49 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.vm-default-sa-example2.google_compute_instance.default[0]:
+ name: test3
+ project: project-id
+ service_account:
+ - scopes:
+ - https://www.googleapis.com/auth/devstorage.read_only
+ - https://www.googleapis.com/auth/logging.write
+ - https://www.googleapis.com/auth/monitoring.write
+ zone: europe-west1-b
+ module.vm-managed-sa-example.google_compute_instance.default[0]:
+ name: test1
+ project: project-id
+ service_account:
+ - scopes:
+ - https://www.googleapis.com/auth/cloud-platform
+ - https://www.googleapis.com/auth/userinfo.email
+ zone: europe-west1-b
+ module.vm-managed-sa-example.google_service_account.service_account[0]:
+ account_id: tf-vm-test1
+ display_name: Terraform VM test1.
+ project: project-id
+ module.vm-managed-sa-example2.google_compute_instance.default[0]:
+ name: test2
+ project: project-id
+ service_account:
+ - scopes:
+ - https://www.googleapis.com/auth/cloud-platform
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 3
+ google_service_account: 1
+ modules: 3
+ resources: 4
diff --git a/tests/modules/compute_vm/examples/simple.yaml b/tests/modules/compute_vm/examples/simple.yaml
new file mode 100644
index 00000000..6754efaa
--- /dev/null
+++ b/tests/modules/compute_vm/examples/simple.yaml
@@ -0,0 +1,72 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.simple-vm-example.google_compute_instance.default[0]:
+ advanced_machine_features: []
+ allow_stopping_for_update: true
+ attached_disk: []
+ boot_disk:
+ - auto_delete: true
+ disk_encryption_key_raw: null
+ initialize_params:
+ - image: projects/debian-cloud/global/images/family/debian-11
+ size: 10
+ type: pd-balanced
+ mode: READ_WRITE
+ can_ip_forward: false
+ deletion_protection: false
+ description: Managed by the compute-vm Terraform module.
+ enable_display: false
+ hostname: null
+ labels: null
+ machine_type: f1-micro
+ metadata: null
+ metadata_startup_script: null
+ name: test
+ network_interface:
+ - access_config: []
+ alias_ip_range: []
+ ipv6_access_config: []
+ network: projects/xxx/global/networks/aaa
+ nic_type: null
+ queue_count: null
+ subnetwork: subnet_self_link
+ project: project-id
+ scheduling:
+ - automatic_restart: true
+ instance_termination_action: null
+ max_run_duration: []
+ min_node_cpus: null
+ node_affinities: []
+ on_host_maintenance: MIGRATE
+ preemptible: false
+ provisioning_model: STANDARD
+ scratch_disk: []
+ service_account:
+ - scopes:
+ - https://www.googleapis.com/auth/cloud-platform
+ - https://www.googleapis.com/auth/userinfo.email
+ shielded_instance_config: []
+ tags: null
+ zone: europe-west1-b
+ module.simple-vm-example.google_service_account.service_account[0]:
+ account_id: tf-vm-test
+ display_name: Terraform VM test.
+ project: project-id
+
+
+counts:
+ google_compute_instance: 1
+ google_service_account: 1
diff --git a/tests/modules/compute_vm/examples/spot.yaml b/tests/modules/compute_vm/examples/spot.yaml
new file mode 100644
index 00000000..c15852db
--- /dev/null
+++ b/tests/modules/compute_vm/examples/spot.yaml
@@ -0,0 +1,31 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.spot-vm-example.google_compute_instance.default[0]:
+ name: test
+ project: project-id
+ scheduling:
+ - automatic_restart: false
+ instance_termination_action: STOP
+ max_run_duration: []
+ min_node_cpus: null
+ node_affinities: []
+ on_host_maintenance: TERMINATE
+ preemptible: true
+ provisioning_model: SPOT
+ zone: europe-west1-b
+
+counts:
+ google_compute_instance: 1
diff --git a/tests/modules/compute_vm/examples/template.yaml b/tests/modules/compute_vm/examples/template.yaml
new file mode 100644
index 00000000..1f1888bf
--- /dev/null
+++ b/tests/modules/compute_vm/examples/template.yaml
@@ -0,0 +1,65 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.cos-test.google_compute_instance_template.default[0]:
+ disk:
+ - auto_delete: true
+ boot: true
+ disk_encryption_key: []
+ disk_name: null
+ disk_size_gb: 10
+ disk_type: pd-balanced
+ labels: null
+ resource_policies: null
+ source: null
+ source_image: projects/cos-cloud/global/images/family/cos-stable
+ source_image_encryption_key: []
+ source_snapshot: null
+ source_snapshot_encryption_key: []
+ - auto_delete: true
+ device_name: disk-1
+ disk_encryption_key: []
+ disk_name: disk-1
+ disk_size_gb: 10
+ disk_type: pd-balanced
+ labels: null
+ mode: READ_WRITE
+ resource_policies: null
+ source: null
+ source_image_encryption_key: []
+ source_snapshot: null
+ source_snapshot_encryption_key: []
+ type: PERSISTENT
+ name_prefix: test-
+ network_interface:
+ - access_config: []
+ alias_ip_range: []
+ ipv6_access_config: []
+ network: projects/xxx/global/networks/aaa
+ network_ip: null
+ nic_type: null
+ queue_count: null
+ subnetwork: subnet_self_link
+ project: my-project
+ region: europe-west1
+ service_account:
+ - email: vm-default@my-project.iam.gserviceaccount.com
+ scopes:
+ - https://www.googleapis.com/auth/devstorage.read_only
+ - https://www.googleapis.com/auth/logging.write
+ - https://www.googleapis.com/auth/monitoring.write
+
+counts:
+ google_compute_instance_template: 1
diff --git a/tests/modules/compute_vm/fixture/main.tf b/tests/modules/compute_vm/fixture/main.tf
deleted file mode 100644
index 5815f25f..00000000
--- a/tests/modules/compute_vm/fixture/main.tf
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-module "test" {
- source = "../../../../modules/compute-vm"
- project_id = "my-project"
- zone = "europe-west1-b"
- name = "test"
- attached_disks = var.attached_disks
- attached_disk_defaults = var.attached_disk_defaults
- create_template = var.create_template
- confidential_compute = var.confidential_compute
- group = var.group
- iam = var.iam
- metadata = var.metadata
- network_interfaces = var.network_interfaces
- service_account_create = var.service_account_create
-}
diff --git a/tests/modules/compute_vm/fixture/variables.tf b/tests/modules/compute_vm/fixture/variables.tf
deleted file mode 100644
index 02d839f6..00000000
--- a/tests/modules/compute_vm/fixture/variables.tf
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-variable "attached_disks" {
- description = "Additional disks, if options is null defaults will be used in its place. Source type is one of 'image' (zonal disks in vms and template), 'snapshot' (vm), 'existing', and null."
- type = any
- default = []
-}
-
-variable "attached_disk_defaults" {
- description = "Defaults for attached disks options."
- type = any
- default = {
- auto_delete = true
- mode = "READ_WRITE"
- replica_zone = null
- type = "pd-balanced"
- }
-}
-
-variable "confidential_compute" {
- type = bool
- default = false
-}
-
-variable "create_template" {
- type = bool
- default = false
-}
-
-variable "group" {
- type = any
- default = null
-}
-
-variable "iam" {
- type = map(set(string))
- default = {}
-}
-
-variable "metadata" {
- type = map(string)
- default = {}
-}
-
-variable "network_interfaces" {
- type = any
- default = [{
- network = "https://www.googleapis.com/compute/v1/projects/my-project/global/networks/default",
- subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/default-default",
- }]
-}
-
-variable "service_account_create" {
- type = bool
- default = false
-}
diff --git a/tests/modules/compute_vm/test_plan.py b/tests/modules/compute_vm/test_plan.py
deleted file mode 100644
index 701891c5..00000000
--- a/tests/modules/compute_vm/test_plan.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-def test_defaults(plan_runner):
- _, resources = plan_runner()
- assert len(resources) == 1
- assert resources[0]['type'] == 'google_compute_instance'
-
-
-def test_service_account(plan_runner):
- _, resources = plan_runner(service_account_create='true')
- assert len(resources) == 2
- assert set(r['type'] for r in resources) == set([
- 'google_compute_instance', 'google_service_account'
- ])
-
-
-def test_template(plan_runner):
- _, resources = plan_runner(create_template='true')
- assert len(resources) == 1
- assert resources[0]['type'] == 'google_compute_instance_template'
- assert resources[0]['values']['name_prefix'] == 'test-'
-
-
-def test_group(plan_runner):
- _, resources = plan_runner(group='{named_ports={}}')
- assert len(resources) == 2
- assert set(r['type'] for r in resources) == set([
- 'google_compute_instance_group', 'google_compute_instance'
- ])
-
-
-def test_iam(plan_runner):
- iam = (
- '{"roles/compute.instanceAdmin" = ["user:a@a.com", "user:b@a.com"],'
- '"roles/iam.serviceAccountUser" = ["user:a@a.com"]}'
- )
- _, resources = plan_runner(iam=iam)
- assert len(resources) == 3
- assert set(r['type'] for r in resources) == set([
- 'google_compute_instance', 'google_compute_instance_iam_binding'])
- iam_bindings = dict(
- (r['index'], r['values']['members']) for r in resources if r['type']
- == 'google_compute_instance_iam_binding'
- )
- assert iam_bindings == {
- 'roles/compute.instanceAdmin': ['user:a@a.com', 'user:b@a.com'],
- 'roles/iam.serviceAccountUser': ['user:a@a.com'],
- }
-
-
-def test_confidential_compute(plan_runner):
- _, resources = plan_runner(confidential_compute='true')
- assert len(resources) == 1
- assert resources[0]['values']['confidential_instance_config'] == [
- {'enable_confidential_compute': True}]
- assert resources[0]['values']['scheduling'][0]['on_host_maintenance'] == 'TERMINATE'
-
-
-def test_confidential_compute_template(plan_runner):
- _, resources = plan_runner(confidential_compute='true',
- create_template='true')
- assert len(resources) == 1
- assert resources[0]['values']['confidential_instance_config'] == [
- {'enable_confidential_compute': True}]
- assert resources[0]['values']['scheduling'][0]['on_host_maintenance'] == 'TERMINATE'
diff --git a/tests/modules/compute_vm/test_plan_disks.py b/tests/modules/compute_vm/test_plan_disks.py
deleted file mode 100644
index 153c072f..00000000
--- a/tests/modules/compute_vm/test_plan_disks.py
+++ /dev/null
@@ -1,165 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def test_types(plan_runner):
- _disks = '''[{
- name = "data1"
- size = "10"
- source_type = "image"
- source = "image-1"
- options = null
- }, {
- name = "data2"
- size = "20"
- source_type = "snapshot"
- source = "snapshot-2"
- options = null
- }, {
- name = "data3"
- size = null
- source_type = "attach"
- source = "disk-3"
- options = null
- }]
- '''
- _, resources = plan_runner(attached_disks=_disks)
- assert len(resources) == 3
- disks = {
- r['values']['name']: r['values']
- for r in resources if r['type'] == 'google_compute_disk'
- }
- assert disks['test-data1']['size'] == 10
- assert disks['test-data2']['size'] == 20
- assert disks['test-data1']['image'] == 'image-1'
- assert disks['test-data1']['snapshot'] is None
- assert disks['test-data2']['snapshot'] == 'snapshot-2'
- assert disks['test-data2']['image'] is None
- instance = [
- r['values'] for r in resources
- if r['type'] == 'google_compute_instance'
- ][0]
- instance_disks = {
- d['source']: d['device_name']
- for d in instance['attached_disk']
- }
- assert instance_disks == {
- 'test-data1': 'data1',
- 'test-data2': 'data2',
- 'disk-3': 'data3'
- }
-
-
-def test_options(plan_runner):
- _disks = '''[{
- name = "data1"
- size = "10"
- source_type = "image"
- source = "image-1"
- options = {
- mode = null, replica_zone = null, type = "pd-standard"
- }
- }, {
- name = "data2"
- size = "20"
- source_type = null
- source = null
- options = {
- mode = null, replica_zone = "europe-west1-c", type = "pd-ssd"
- }
- }]
- '''
- _, resources = plan_runner(attached_disks=_disks)
- assert len(resources) == 3
- disks_z = [
- r['values'] for r in resources if r['type'] == 'google_compute_disk'
- ]
- disks_r = [
- r['values'] for r in resources
- if r['type'] == 'google_compute_region_disk'
- ]
- assert len(disks_z) == len(disks_r) == 1
- instance = [
- r['values'] for r in resources
- if r['type'] == 'google_compute_instance'
- ][0]
- instance_disks = [d['device_name'] for d in instance['attached_disk']]
- assert instance_disks == ['data1', 'data2']
-
-
-def test_template(plan_runner):
- _disks = '''[{
- name = "data1"
- size = "10"
- source_type = "image"
- source = "image-1"
- options = {
- mode = null, replica_zone = null, type = "pd-standard"
- }
- }, {
- name = "data2"
- size = "20"
- source_type = null
- source = null
- options = {
- mode = null, replica_zone = "europe-west1-c", type = "pd-ssd"
- }
- }]
- '''
- _, resources = plan_runner(attached_disks=_disks, create_template="true")
- assert len(resources) == 1
- template = [
- r['values'] for r in resources
- if r['type'] == 'google_compute_instance_template'
- ][0]
- assert len(template['disk']) == 3
-
-
-def test_auto_delete(plan_runner):
- _disks = '''[{
- name = "data1"
- size = "10"
- options = {
- auto_delete = true, mode = "READ_WRITE"
- }
- }, {
- name = "data2"
- size = "20"
- options = {
- auto_delete = false, mode = "READ_WRITE"
- },
- }, {
- name = "data3"
- size = "20"
- options = {
- mode = "READ_ONLY"
- }
- }]
- '''
- _, resources = plan_runner(attached_disks=_disks, create_template="true")
- assert len(resources) == 1
- template = [
- r['values'] for r in resources
- if r['type'] == 'google_compute_instance_template'
- ][0]
- additional_disks = [
- d for d in template['disk'] if 'boot' not in d or d['boot'] != True
- ]
- assert len(additional_disks) == 3
- disk_data1 = [d for d in additional_disks if d['disk_name'] == 'data1']
- disk_data2 = [d for d in additional_disks if d['disk_name'] == 'data2']
- disk_data3 = [d for d in additional_disks if d['disk_name'] == 'data3']
- assert len(disk_data1) == 1 and disk_data1[0]['auto_delete'] == True
- assert len(disk_data2) == 1 and disk_data2[0]['auto_delete'] == False
- assert len(disk_data3) == 1 and disk_data3[0]['auto_delete'] == False
diff --git a/tests/modules/compute_vm/test_plan_interfaces.py b/tests/modules/compute_vm/test_plan_interfaces.py
deleted file mode 100644
index e88c087b..00000000
--- a/tests/modules/compute_vm/test_plan_interfaces.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-def test_address(plan_runner):
- nics = '''[{
- network = "https://www.googleapis.com/compute/v1/projects/my-project/global/networks/default",
- subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/default-default",
- nat = false,
- addresses = {external=null, internal="10.0.0.2"}
- }]
- '''
- _, resources = plan_runner(network_interfaces=nics)
- assert len(resources) == 1
- n = resources[0]['values']['network_interface'][0]
- assert n['network_ip'] == "10.0.0.2"
- assert n['access_config'] == []
-
-
-def test_nat_address(plan_runner):
- nics = '''[{
- network = "https://www.googleapis.com/compute/v1/projects/my-project/global/networks/default",
- subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/default-default",
- nat = true,
- addresses = {external="8.8.8.8", internal=null}
- }]
- '''
- _, resources = plan_runner(network_interfaces=nics)
- assert len(resources) == 1
- n = resources[0]['values']['network_interface'][0]
- assert 'network_ip' not in n
- assert n['access_config'][0]['nat_ip'] == '8.8.8.8'
diff --git a/tests/modules/gke_cluster/__init__.py b/tests/modules/gke_cluster/__init__.py
deleted file mode 100644
index 6d6d1266..00000000
--- a/tests/modules/gke_cluster/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/tests/modules/iam_service_account/__init__.py b/tests/modules/iam_service_account/__init__.py
deleted file mode 100644
index 6d6d1266..00000000
--- a/tests/modules/iam_service_account/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/tests/modules/net_glb/__init__.py b/tests/modules/net_glb/__init__.py
deleted file mode 100644
index 6d6d1266..00000000
--- a/tests/modules/net_glb/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/tests/modules/organization/tags.tfvars b/tests/modules/organization/tags.tfvars
new file mode 100644
index 00000000..2a4dcb42
--- /dev/null
+++ b/tests/modules/organization/tags.tfvars
@@ -0,0 +1,59 @@
+network_tags = {
+ net_environment = {
+ network = "foobar"
+ }
+}
+tags = {
+ foo = {}
+ bar = {
+ description = null
+ iam = null
+ values = null
+ }
+ baz = {
+ id = "tagKeys/1234567890"
+ values = {
+ one = null
+ two = null
+ }
+ }
+ foobar = {
+ description = "Foobar tag."
+ iam = {
+ "roles/resourcemanager.tagAdmin" = [
+ "user:user1@example.com", "user:user2@example.com"
+ ]
+ }
+ values = {
+ one = null
+ two = {
+ description = "Foobar 2."
+ iam = {
+ "roles/resourcemanager.tagViewer" = [
+ "user:user3@example.com"
+ ]
+ }
+ }
+ three = {
+ description = "Foobar 3."
+ iam = {
+ "roles/resourcemanager.tagViewer" = [
+ "user:user3@example.com"
+ ]
+ "roles/resourcemanager.tagAdmin" = [
+ "user:user4@example.com"
+ ]
+ }
+ }
+ four = {
+ description = "Foobar 4."
+ id = "tagValues/1234567890"
+ iam = {
+ "roles/resourcemanager.tagViewer" = [
+ "user:user4@example.com"
+ ]
+ }
+ }
+ }
+ }
+}
diff --git a/tests/modules/organization/tags.yaml b/tests/modules/organization/tags.yaml
new file mode 100644
index 00000000..3e5524d4
--- /dev/null
+++ b/tests/modules/organization/tags.yaml
@@ -0,0 +1,76 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ google_tags_tag_key.default["bar"]:
+ description: Managed by the Terraform organization module.
+ parent: organizations/1234567890
+ purpose: null
+ purpose_data: null
+ short_name: bar
+ google_tags_tag_key.default["foo"]:
+ description: Managed by the Terraform organization module.
+ parent: organizations/1234567890
+ purpose: null
+ purpose_data: null
+ short_name: foo
+ google_tags_tag_key.default["foobar"]:
+ description: Foobar tag.
+ parent: organizations/1234567890
+ purpose: null
+ purpose_data: null
+ short_name: foobar
+ google_tags_tag_key.default["net_environment"]:
+ description: Managed by the Terraform organization module.
+ parent: organizations/1234567890
+ purpose: GCE_FIREWALL
+ purpose_data:
+ network: foobar
+ short_name: net_environment
+ ? google_tags_tag_key_iam_binding.default["foobar:roles/resourcemanager.tagAdmin"]
+ : condition: []
+ members:
+ - user:user1@example.com
+ - user:user2@example.com
+ role: roles/resourcemanager.tagAdmin
+ google_tags_tag_value.default["foobar/one"]:
+ description: Managed by the Terraform organization module.
+ short_name: one
+ google_tags_tag_value.default["foobar/three"]:
+ description: Foobar 3.
+ short_name: three
+ google_tags_tag_value.default["foobar/two"]:
+ description: Foobar 2.
+ short_name: two
+ ? google_tags_tag_value_iam_binding.default["foobar/three:roles/resourcemanager.tagAdmin"]
+ : condition: []
+ members:
+ - user:user4@example.com
+ role: roles/resourcemanager.tagAdmin
+ ? google_tags_tag_value_iam_binding.default["foobar/three:roles/resourcemanager.tagViewer"]
+ : condition: []
+ members:
+ - user:user3@example.com
+ role: roles/resourcemanager.tagViewer
+ ? google_tags_tag_value_iam_binding.default["foobar/two:roles/resourcemanager.tagViewer"]
+ : condition: []
+ members:
+ - user:user3@example.com
+ role: roles/resourcemanager.tagViewer
+
+counts:
+ google_tags_tag_key: 4
+ google_tags_tag_key_iam_binding: 1
+ google_tags_tag_value: 5
+ google_tags_tag_value_iam_binding: 4
diff --git a/tests/modules/organization/tftest.yaml b/tests/modules/organization/tftest.yaml
index 7466614e..7568b732 100644
--- a/tests/modules/organization/tftest.yaml
+++ b/tests/modules/organization/tftest.yaml
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -23,3 +23,4 @@ tests:
org_policies_boolean:
org_policies_custom_constraints:
firewall_policies_factory_combined:
+ tags:
diff --git a/tools/check_links.py b/tools/check_links.py
index 77dc6173..1e2759df 100755
--- a/tools/check_links.py
+++ b/tools/check_links.py
@@ -86,7 +86,7 @@ def main(dirs, external):
state = '✓' if all(l.valid for l in doc.links) else '✗'
print(f'[{state}] {doc.relpath} ({len(doc.links)})')
if state == '✗':
- error = [f'{dir_name}{doc.relpath}']
+ error = [f'{dir_name}/{doc.relpath}']
for l in doc.links:
if not l.valid:
error.append(f' - {l.dest}')
diff --git a/tools/plan_summary.py b/tools/plan_summary.py
index 78c5f939..ae52c86c 100755
--- a/tools/plan_summary.py
+++ b/tools/plan_summary.py
@@ -15,16 +15,19 @@
# limitations under the License.
import click
+import os
import sys
import tempfile
import yaml
from pathlib import Path
-BASEDIR = Path(__file__).parents[1]
-sys.path.append(str(BASEDIR / 'tests'))
-
-import fixtures
+try:
+ import fixtures
+except ImportError:
+ BASEDIR = Path(__file__).parents[1]
+ sys.path.append(str(BASEDIR / 'tests'))
+ import fixtures
@click.command()