Merge branch 'master' into autopilot
This commit is contained in:
Ludovico Magnocavallo
GitHub
commit
662a9b185c
|
|
@ -0,0 +1,55 @@
|
|||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: fabric-tests
|
||||
description: Set up Fabric testing environment
|
||||
inputs:
|
||||
PYTHON_VERSION:
|
||||
required: true
|
||||
TERRAFORM_VERSION:
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Config auth
|
||||
shell: bash
|
||||
run: |
|
||||
echo '{"type": "service_account", "project_id": "test-only"}' \
|
||||
| tee -a $GOOGLE_APPLICATION_CREDENTIALS
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ inputs.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'tests/requirements.txt'
|
||||
- name: Set up Terraform
|
||||
uses: hashicorp/setup-terraform@v2
|
||||
with:
|
||||
terraform_version: ${{ inputs.TERRAFORM_VERSION }}
|
||||
terraform_wrapper: false
|
||||
- name: Configure provider cache
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'plugin_cache_dir = "/home/runner/.terraform.d/plugin-cache"' \
|
||||
| tee -a /home/runner/.terraformrc
|
||||
echo 'disable_checkpoint = true' \
|
||||
| tee -a /home/runner/.terraformrc
|
||||
mkdir -p /home/runner/.terraform.d/plugin-cache
|
||||
# avoid conflicts with user-installed providers on local machines
|
||||
- name: Pin provider versions
|
||||
shell: bash
|
||||
run: |
|
||||
for f in $(find . -name versions.tf); do
|
||||
sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
|
||||
done
|
||||
|
|
@ -31,72 +31,51 @@ env:
|
|||
TF_VERSION: 1.3.9
|
||||
|
||||
jobs:
|
||||
examples:
|
||||
examples-blueprints:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Config auth
|
||||
run: |
|
||||
echo '{"type": "service_account", "project_id": "test-only"}' \
|
||||
| tee -a $GOOGLE_APPLICATION_CREDENTIALS
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
- name: Call composite action fabric-tests
|
||||
uses: ./.github/actions/fabric-tests
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'tests/requirements.txt'
|
||||
PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
|
||||
TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
|
||||
|
||||
- name: Set up Terraform
|
||||
uses: hashicorp/setup-terraform@v2
|
||||
with:
|
||||
terraform_version: ${{ env.TF_VERSION }}
|
||||
terraform_wrapper: false
|
||||
|
||||
# avoid conflicts with user-installed providers on local machines
|
||||
- name: Pin provider versions
|
||||
- name: Run tests on documentation examples
|
||||
id: pytest
|
||||
run: |
|
||||
for f in $(find . -name versions.tf); do
|
||||
sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
|
||||
done
|
||||
pip install -r tests/requirements.txt
|
||||
pytest -vv -k blueprints/ tests/examples
|
||||
|
||||
examples-modules:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Call composite action fabric-tests
|
||||
uses: ./.github/actions/fabric-tests
|
||||
with:
|
||||
PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
|
||||
TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
|
||||
|
||||
- name: Run tests on documentation examples
|
||||
id: pytest
|
||||
run: |
|
||||
mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
|
||||
pip install -r tests/requirements.txt
|
||||
pytest -vv tests/examples
|
||||
pytest -vv -k modules/ tests/examples
|
||||
|
||||
blueprints:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Config auth
|
||||
run: |
|
||||
echo '{"type": "service_account", "project_id": "test-only"}' \
|
||||
| tee -a $GOOGLE_APPLICATION_CREDENTIALS
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
- name: Call composite action fabric-tests
|
||||
uses: ./.github/actions/fabric-tests
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'tests/requirements.txt'
|
||||
|
||||
- name: Set up Terraform
|
||||
uses: hashicorp/setup-terraform@v2
|
||||
with:
|
||||
terraform_version: ${{ env.TF_VERSION }}
|
||||
terraform_wrapper: false
|
||||
|
||||
# avoid conflicts with user-installed providers on local machines
|
||||
- name: Pin provider versions
|
||||
run: |
|
||||
for f in $(find . -name versions.tf); do
|
||||
sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
|
||||
done
|
||||
PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
|
||||
TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
|
||||
|
||||
- name: Run tests environments
|
||||
id: pytest
|
||||
|
|
@ -110,30 +89,11 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Config auth
|
||||
run: |
|
||||
echo '{"type": "service_account", "project_id": "test-only"}' \
|
||||
| tee -a $GOOGLE_APPLICATION_CREDENTIALS
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
- name: Call composite action fabric-tests
|
||||
uses: ./.github/actions/fabric-tests
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'tests/requirements.txt'
|
||||
|
||||
- name: Set up Terraform
|
||||
uses: hashicorp/setup-terraform@v2
|
||||
with:
|
||||
terraform_version: ${{ env.TF_VERSION }}
|
||||
terraform_wrapper: false
|
||||
|
||||
# avoid conflicts with user-installed providers on local machines
|
||||
- name: Pin provider versions
|
||||
run: |
|
||||
for f in $(find . -name versions.tf); do
|
||||
sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
|
||||
done
|
||||
PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
|
||||
TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
|
||||
|
||||
- name: Run tests modules
|
||||
id: pytest
|
||||
|
|
@ -147,30 +107,11 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Config auth
|
||||
run: |
|
||||
echo '{"type": "service_account", "project_id": "test-only"}' \
|
||||
| tee -a $GOOGLE_APPLICATION_CREDENTIALS
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
- name: Call composite action fabric-tests
|
||||
uses: ./.github/actions/fabric-tests
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'tests/requirements.txt'
|
||||
|
||||
- name: Set up Terraform
|
||||
uses: hashicorp/setup-terraform@v2
|
||||
with:
|
||||
terraform_version: ${{ env.TF_VERSION }}
|
||||
terraform_wrapper: false
|
||||
|
||||
# avoid conflicts with user-installed providers on local machines
|
||||
- name: Pin provider versions
|
||||
run: |
|
||||
for f in $(find . -name versions.tf); do
|
||||
sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
|
||||
done
|
||||
PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
|
||||
TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
|
||||
|
||||
- name: Run tests on FAST stages
|
||||
id: pytest
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ All notable changes to this project will be documented in this file.
|
|||
|
||||
### FAST
|
||||
|
||||
- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
|
||||
- [[#1209](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1209)] Billing exclusion support for FAST mt resman ([ludoo](https://github.com/ludoo)) <!-- 2023-03-03 16:23:37+00:00 -->
|
||||
- [[#1207](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1207)] Allow preventing creation of billing IAM roles in FAST, add instructions on delayed billing association ([ludoo](https://github.com/ludoo)) <!-- 2023-03-03 08:24:42+00:00 -->
|
||||
- [[#1184](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1184)] **incompatible change:** Allow multiple peer gateways in VPN HA module ([ludoo](https://github.com/ludoo)) <!-- 2023-02-27 10:19:00+00:00 -->
|
||||
|
|
@ -63,6 +64,7 @@ All notable changes to this project will be documented in this file.
|
|||
|
||||
### MODULES
|
||||
|
||||
- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
|
||||
- [[#1206](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1206)] Dataproc module. Fix output. ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-02 12:59:19+00:00 -->
|
||||
- [[#1205](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1205)] Fix issue with GKE cluster notifications topic & static output for pubsub module ([rosmo](https://github.com/rosmo)) <!-- 2023-03-02 10:43:40+00:00 -->
|
||||
- [[#1204](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1204)] Fix url_redirect issue on net-glb module ([erabusi](https://github.com/erabusi)) <!-- 2023-03-02 06:51:40+00:00 -->
|
||||
|
|
@ -100,6 +102,7 @@ All notable changes to this project will be documented in this file.
|
|||
|
||||
### TOOLS
|
||||
|
||||
- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
|
||||
- [[#1209](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1209)] Billing exclusion support for FAST mt resman ([ludoo](https://github.com/ludoo)) <!-- 2023-03-03 16:23:37+00:00 -->
|
||||
- [[#1208](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1208)] Fix outdated go deps, dependabot alerts ([averbuks](https://github.com/averbuks)) <!-- 2023-03-03 06:15:09+00:00 -->
|
||||
- [[#1182](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1182)] Bump actions versions ([juliocc](https://github.com/juliocc)) <!-- 2023-02-25 16:27:20+00:00 -->
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Currently available blueprints:
|
|||
|
||||
- **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/)
|
||||
- **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation)
|
||||
- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder)
|
||||
- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml)
|
||||
- **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory)
|
||||
- **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [GKE Autopilot](./gke/autopilot)
|
||||
- **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke)
|
||||
|
|
|
|||
|
|
@ -69,3 +69,9 @@ This [blueprint](./vertex-mlops/) implements the infrastructure required to have
|
|||
This [blueprint](./shielded-folder/) implements an opinionated folder configuration according to GCP best practices. Configurations implemented on the folder would be beneficial to host workloads inheriting constraints from the folder they belong to.
|
||||
|
||||
<br clear="left">
|
||||
|
||||
### BigQuery ML and Vertex AI Pipeline
|
||||
|
||||
<a href="./bq-ml/" title="BigQuery ML and Vertex AI Pipeline"><img src="./bq-ml/images/diagram.png" align="left" width="280px"></a>
|
||||
This [blueprint](./bq-ml/) provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML.
|
||||
<br clear="left">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
# BigQuery ML and Vertex AI Pipeline
|
||||
|
||||
This blueprint provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML.
|
||||
|
||||
This is the high-level diagram:
|
||||
|
||||
|
||||
|
||||
It also includes the IAM wiring needed to make such scenarios work. Regional resources are used in this example, but the same logic applies to 'dual regional', 'multi regional', or 'global' resources.
|
||||
|
||||
The example is designed to match real-world use cases with a minimum amount of resources and be used as a starting point for your scenario.
|
||||
|
||||
## Managed resources and services
|
||||
|
||||
This sample creates several distinct groups of resources:
|
||||
|
||||
- Networking
|
||||
- VPC network
|
||||
- Subnet
|
||||
- Firewall rules for SSH access via IAP and open communication within the VPC
|
||||
- Cloud Nat
|
||||
- IAM
|
||||
- Vertex AI workbench service account
|
||||
- Vertex AI pipeline service account
|
||||
- Storage
|
||||
- GCS bucket
|
||||
- Bigquery dataset
|
||||
|
||||
## Customization
|
||||
|
||||
### Virtual Private Cloud (VPC) design
|
||||
|
||||
As is often the case in real-world configurations, this blueprint accepts an existing Shared-VPC via the `vpc_config` variable as input.
|
||||
|
||||
### Customer Managed Encryption Keys
|
||||
|
||||
As is often the case in real-world configurations, this blueprint accepts as input existing Cloud KMS keys to encrypt resources via the `service_encryption_keys` variable.
|
||||
|
||||
## Demo
|
||||
|
||||
In the [`demo`](./demo/) folder, you can find an example of creating a Vertex AI pipeline from a publicly available dataset and deploying the model to be used from a Vertex AI managed endpoint or from within Bigquery.
|
||||
|
||||
To run the demo:
|
||||
|
||||
- Connect to the Vertex AI workbench instance
|
||||
- Clone this repository
|
||||
- Run the and run [`demo/bmql_pipeline.ipynb`](demo/bmql_pipeline.ipynb) Jupyter Notebook.
|
||||
|
||||
## Files
|
||||
|
||||
| name | description | modules | resources |
|
||||
|---|---|---|---|
|
||||
| [datastorage.tf](./datastorage.tf) | Datastorage resources. | <code>bigquery-dataset</code> · <code>gcs</code> | |
|
||||
| [main.tf](./main.tf) | Core resources. | <code>project</code> | |
|
||||
| [outputs.tf](./outputs.tf) | Output variables. | | |
|
||||
| [variables.tf](./variables.tf) | Terraform variables. | | |
|
||||
| [versions.tf](./versions.tf) | Version pins. | | |
|
||||
| [vertex.tf](./vertex.tf) | Vertex resources. | <code>iam-service-account</code> | <code>google_notebooks_instance</code> · <code>google_vertex_ai_metadata_store</code> |
|
||||
| [vpc.tf](./vpc.tf) | VPC resources. | <code>net-cloudnat</code> · <code>net-vpc</code> · <code>net-vpc-firewall</code> | <code>google_project_iam_member</code> |
|
||||
|
||||
<!-- BEGIN TFDOC -->
|
||||
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [prefix](variables.tf#L23) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L41) | Project id references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
||||
| [location](variables.tf#L17) | The location where resources will be deployed. | <code>string</code> | | <code>"US"</code> |
|
||||
| [project_create](variables.tf#L32) | Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L46) | The region where resources will be deployed. | <code>string</code> | | <code>"us-central1"</code> |
|
||||
| [service_encryption_keys](variables.tf#L52) | Cloud KMS to use to encrypt different services. The key location should match the service region. | <code title="object({ bq = string compute = string storage = string })">object({…})</code> | | <code>null</code> |
|
||||
| [vpc_config](variables.tf#L62) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | <code title="object({ host_project = string network_self_link = string subnet_self_link = string })">object({…})</code> | | <code>null</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
| name | description | sensitive |
|
||||
|---|---|:---:|
|
||||
| [bucket](outputs.tf#L17) | GCS Bucket URL. | |
|
||||
| [dataset](outputs.tf#L22) | GCS Bucket URL. | |
|
||||
| [notebook](outputs.tf#L27) | Vertex AI notebook details. | |
|
||||
| [project](outputs.tf#L35) | Project id. | |
|
||||
| [service-account-vertex](outputs.tf#L40) | Service account to be used for Vertex AI pipelines. | |
|
||||
| [vertex-ai-metadata-store](outputs.tf#L45) | Vertex AI Metadata Store ID. | |
|
||||
| [vpc](outputs.tf#L50) | VPC Network. | |
|
||||
|
||||
<!-- END TFDOC -->
|
||||
## Test
|
||||
|
||||
```hcl
|
||||
module "test" {
|
||||
source = "./fabric/blueprints/data-solutions/bq-ml/"
|
||||
project_create = {
|
||||
billing_account_id = "123456-123456-123456"
|
||||
parent = "folders/12345678"
|
||||
}
|
||||
project_id = "project-1"
|
||||
prefix = "prefix"
|
||||
}
|
||||
|
||||
# tftest modules=9 resources=46
|
||||
```
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Datastorage resources.
|
||||
|
||||
module "bucket" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.project.project_id
|
||||
prefix = var.prefix
|
||||
location = var.location
|
||||
name = "data"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key
|
||||
}
|
||||
|
||||
module "dataset" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}_data"
|
||||
encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key
|
||||
location = "US"
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# BigQuery ML and Vertex AI Pipeline Demo
|
||||
|
||||
This demo shows how to combine BigQuery ML (BQML) and Vertex AI to create a ML pipeline leveraging the infrastructure created in the blueprint.
|
||||
|
||||
More in details, this tutorial will focus on the following three steps:
|
||||
|
||||
- define a Vertex AI pipeline to create features, train and evaluate BQML models
|
||||
- serve a BQ model through an API powered by Vertex AI Endpoint
|
||||
- create batch prediction via BigQuery
|
||||
|
||||
In this tutorial we will also see how to make explainable predictions, in order to understand what are the most important features that most influence the algorithm outputs.
|
||||
|
||||
# Dataset
|
||||
|
||||
This tutorial uses a fictitious e-commerce dataset collecting programmatically generated data from the fictitious e-commerce store called The Look. The dataset is publicy available on BigQuery at this location `bigquery-public-data.thelook_ecommerce`.
|
||||
|
||||
# Goal
|
||||
|
||||
The goal of this tutorial is to train a classification ML model using BigQuery ML and predict if a new web session is going to convert.
|
||||
|
||||
The tutorial focuses more on how to combine Vertex AI and BigQuery ML to create a model that can be used both for near-real time and batch predictions rather than the design of the model itself.
|
||||
|
||||
# Main components
|
||||
|
||||
In this tutorial we will make use of the following main components:
|
||||
- Big Query:
|
||||
- standard: to create a view which contains the model features and the target variable
|
||||
- ML: to train, evaluate and make batch predictions
|
||||
- Vertex AI:
|
||||
- Pipeline: to define a configurable and re-usable set of steps to train and evaluate a BQML model
|
||||
- Experiment: to keep track of all the trainings done via the Pipeline
|
||||
- Model Registry: to keep track of the trained versions of a specific model
|
||||
- Endpoint: to serve the model via API
|
||||
- Workbench: to run this demo
|
||||
|
||||
# How to get started
|
||||
|
||||
1. Access the Vertex AI Workbench
|
||||
2. clone this repository
|
||||
2. run the [`bmql_pipeline.ipynb`](bmql_pipeline.ipynb) Jupyter Notebook
|
||||
|
|
@ -0,0 +1,459 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Copyright 2023 Google LLC**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2023 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Install python requirements and import packages"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -r requirements.txt"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import kfp\n",
|
||||
"import google_cloud_pipeline_components.v1.bigquery as bqop\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform as aip\n",
|
||||
"from google.cloud import bigquery"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Set your env variables"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPERIMENT_NAME = 'bqml-experiment'\n",
|
||||
"ENDPOINT_DISPLAY_NAME = 'bqml-endpoint'\n",
|
||||
"DATASET = \"{}_data\".format(PREFIX.replace(\"-\",\"_\")) \n",
|
||||
"LOCATION = 'US'\n",
|
||||
"MODEL_NAME = 'bqml-model'\n",
|
||||
"PIPELINE_NAME = 'bqml-vertex-pipeline'\n",
|
||||
"PIPELINE_ROOT = f\"gs://{PREFIX}-data\"\n",
|
||||
"PREFIX = 'your-prefix'\n",
|
||||
"PROJECT_ID = 'your-project-id'\n",
|
||||
"REGION = 'us-central1'\n",
|
||||
"SERVICE_ACCOUNT = f\"vertex-sa@{PROJECT_ID}.iam.gserviceaccount.com\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Vertex AI Pipeline Definition\n",
|
||||
"\n",
|
||||
"Let's first define the queries for the features and target creation and the query to train the model\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# this query creates the features for our model and the target value we would like to predict\n",
|
||||
"\n",
|
||||
"features_query = \"\"\"\n",
|
||||
"CREATE VIEW if NOT EXISTS `{project_id}.{dataset}.ecommerce_abt` AS\n",
|
||||
"WITH abt AS (\n",
|
||||
" SELECT user_id,\n",
|
||||
" session_id,\n",
|
||||
" city,\n",
|
||||
" postal_code,\n",
|
||||
" browser,\n",
|
||||
" traffic_source,\n",
|
||||
" min(created_at) AS session_starting_ts,\n",
|
||||
" sum(CASE WHEN event_type = 'purchase' THEN 1 ELSE 0 END) has_purchased\n",
|
||||
" FROM `bigquery-public-data.thelook_ecommerce.events` \n",
|
||||
" GROUP BY user_id,\n",
|
||||
" session_id,\n",
|
||||
" city,\n",
|
||||
" postal_code,\n",
|
||||
" browser,\n",
|
||||
" traffic_source\n",
|
||||
"), previous_orders AS (\n",
|
||||
" SELECT user_id,\n",
|
||||
" array_agg (struct(created_at AS order_creations_ts,\n",
|
||||
" o.order_id,\n",
|
||||
" o.status,\n",
|
||||
" oi.order_cost)) as user_orders\n",
|
||||
" FROM `bigquery-public-data.thelook_ecommerce.orders` o\n",
|
||||
" JOIN (SELECT order_id,\n",
|
||||
" sum(sale_price) order_cost \n",
|
||||
" FROM `bigquery-public-data.thelook_ecommerce.order_items`\n",
|
||||
" GROUP BY 1) oi\n",
|
||||
" ON o.order_id = oi.order_id\n",
|
||||
" GROUP BY 1\n",
|
||||
")\n",
|
||||
"SELECT abt.*,\n",
|
||||
" CASE WHEN extract(DAYOFWEEK FROM session_starting_ts) IN (1,7)\n",
|
||||
" THEN 'WEEKEND' \n",
|
||||
" ELSE 'WEEKDAY'\n",
|
||||
" END AS day_of_week,\n",
|
||||
" extract(HOUR FROM session_starting_ts) hour_of_day,\n",
|
||||
" (SELECT count(DISTINCT uo.order_id) \n",
|
||||
" FROM unnest(user_orders) uo \n",
|
||||
" WHERE uo.order_creations_ts < session_starting_ts \n",
|
||||
" AND status IN ('Shipped', 'Complete', 'Processing')) AS number_of_successful_orders,\n",
|
||||
" IFNULL((SELECT sum(DISTINCT uo.order_cost) \n",
|
||||
" FROM unnest(user_orders) uo \n",
|
||||
" WHERE uo.order_creations_ts < session_starting_ts \n",
|
||||
" AND status IN ('Shipped', 'Complete', 'Processing')), 0) AS sum_previous_orders,\n",
|
||||
" (SELECT count(DISTINCT uo.order_id) \n",
|
||||
" FROM unnest(user_orders) uo \n",
|
||||
" WHERE uo.order_creations_ts < session_starting_ts \n",
|
||||
" AND status IN ('Cancelled', 'Returned')) AS number_of_unsuccessful_orders\n",
|
||||
"FROM abt \n",
|
||||
"LEFT JOIN previous_orders pso \n",
|
||||
"ON abt.user_id = pso.user_id\n",
|
||||
"\"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# this query create the train job on BQ ML\n",
|
||||
"train_query = \"\"\"\n",
|
||||
"CREATE OR REPLACE MODEL `{project_id}.{dataset}.{model_name}`\n",
|
||||
"OPTIONS(MODEL_TYPE='{model_type}',\n",
|
||||
" INPUT_LABEL_COLS=['has_purchased'],\n",
|
||||
" ENABLE_GLOBAL_EXPLAIN=TRUE,\n",
|
||||
" MODEL_REGISTRY='VERTEX_AI',\n",
|
||||
" DATA_SPLIT_METHOD = 'RANDOM',\n",
|
||||
" DATA_SPLIT_EVAL_FRACTION = {split_fraction}\n",
|
||||
" ) AS \n",
|
||||
"SELECT * EXCEPT (session_id, session_starting_ts, user_id) \n",
|
||||
"FROM `{project_id}.{dataset}.ecommerce_abt`\n",
|
||||
"WHERE extract(ISOYEAR FROM session_starting_ts) = 2022\n",
|
||||
"\"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In the following code block, we are defining our Vertex AI pipeline. It is made up of three main steps:\n",
|
||||
"1. Create a BigQuery dataset that will contain the BigQuery ML models\n",
|
||||
"2. Train the BigQuery ML model, in this case, a logistic regression\n",
|
||||
"3. Evaluate the BigQuery ML model with the standard evaluation metrics\n",
|
||||
"\n",
|
||||
"The pipeline takes as input the following variables:\n",
|
||||
"- ```dataset```: name of the dataset where the artifacts will be stored\n",
|
||||
"- ```evaluate_job_conf```: bq dict configuration to define where to store evaluation metrics\n",
|
||||
"- ```location```: BigQuery location\n",
|
||||
"- ```model_name```: the display name of the BigQuery ML model\n",
|
||||
"- ```project_id```: the project id where the GCP resources will be created\n",
|
||||
"- ```split_fraction```: the percentage of data that will be used as an evaluation dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"@kfp.dsl.pipeline(name='bqml-pipeline', pipeline_root=PIPELINE_ROOT)\n",
|
||||
"def pipeline(\n",
|
||||
" model_name: str,\n",
|
||||
" split_fraction: float,\n",
|
||||
" evaluate_job_conf: dict, \n",
|
||||
" dataset: str = DATASET,\n",
|
||||
" project_id: str = PROJECT_ID,\n",
|
||||
" location: str = LOCATION,\n",
|
||||
" ):\n",
|
||||
"\n",
|
||||
" create_dataset = bqop.BigqueryQueryJobOp(\n",
|
||||
" project=project_id,\n",
|
||||
" location=location,\n",
|
||||
" query=f'CREATE SCHEMA IF NOT EXISTS {dataset}'\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" create_features_view = bqop.BigqueryQueryJobOp(\n",
|
||||
" project=project_id,\n",
|
||||
" location=location,\n",
|
||||
" query=features_query.format(dataset=dataset, project_id=project_id),\n",
|
||||
" #job_configuration_query = {\"writeDisposition\": \"WRITE_TRUNCATE\"} #, \"destinationTable\":{\"projectId\":project_id,\"datasetId\":dataset,\"tableId\":\"ecommerce_abt_table\"}} #{\"destinationTable\":{\"projectId\":\"project_id\",\"datasetId\":dataset,\"tableId\":\"ecommerce_abt_table\"}}, #\"writeDisposition\": \"WRITE_TRUNCATE\", \n",
|
||||
"\n",
|
||||
" ).after(create_dataset)\n",
|
||||
"\n",
|
||||
" create_bqml_model = bqop.BigqueryCreateModelJobOp(\n",
|
||||
" project=project_id,\n",
|
||||
" location=location,\n",
|
||||
" query=train_query.format(model_type = 'LOGISTIC_REG'\n",
|
||||
" , project_id = project_id\n",
|
||||
" , dataset = dataset\n",
|
||||
" , model_name = model_name\n",
|
||||
" , split_fraction=split_fraction)\n",
|
||||
" ).after(create_features_view)\n",
|
||||
"\n",
|
||||
" evaluate_bqml_model = bqop.BigqueryEvaluateModelJobOp(\n",
|
||||
" project=project_id,\n",
|
||||
" location=location,\n",
|
||||
" model=create_bqml_model.outputs[\"model\"],\n",
|
||||
" job_configuration_query=evaluate_job_conf\n",
|
||||
" ).after(create_bqml_model)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# this is to compile our pipeline and generate the json description file\n",
|
||||
"kfp.v2.compiler.Compiler().compile(pipeline_func=pipeline,\n",
|
||||
" package_path=f'{PIPELINE_NAME}.json') "
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Create Experiment\n",
|
||||
"\n",
|
||||
"We will create an experiment to keep track of our training and tasks on a specific issue or problem."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"my_experiment = aip.Experiment.get_or_create(\n",
|
||||
" experiment_name=EXPERIMENT_NAME,\n",
|
||||
" description='This is a new experiment to keep track of bqml trainings',\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Running the same training Vertex AI pipeline with different parameters\n",
|
||||
"\n",
|
||||
"One of the main tasks during the training phase is to compare different models or to try the same model with different inputs. We can leverage the power of Vertex AI Pipelines to submit the same steps with different training parameters. Thanks to the experiments artifact, it is possible to easily keep track of all the tests that have been done. This simplifies the process of selecting the best model to deploy.\n",
|
||||
"\n",
|
||||
"In this demo case, we will run the same training pipeline while changing the split data percentage between training and test data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# this configuration is needed in order to persist the evaluation metrics on big query\n",
|
||||
"job_configuration_query = {\"destinationTable\": {\"projectId\": PROJECT_ID, \"datasetId\": DATASET}, \"writeDisposition\": \"WRITE_TRUNCATE\"}\n",
|
||||
"\n",
|
||||
"for split_fraction in [0.1, 0.2]:\n",
|
||||
" job_configuration_query['destinationTable']['tableId'] = MODEL_NAME+'-fraction-{}-eval_table'.format(int(split_fraction*100))\n",
|
||||
" pipeline = aip.PipelineJob(\n",
|
||||
" parameter_values = {'split_fraction':split_fraction, 'model_name': MODEL_NAME+'-fraction-{}'.format(int(split_fraction*100)), 'evaluate_job_conf': job_configuration_query },\n",
|
||||
" display_name=PIPELINE_NAME,\n",
|
||||
" template_path=f'{PIPELINE_NAME}.json',\n",
|
||||
" pipeline_root=PIPELINE_ROOT,\n",
|
||||
" enable_caching=True\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=my_experiment)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Deploy the model on a Vertex AI endpoint\n",
|
||||
"\n",
|
||||
"Thanks to the integration of Vertex AI Endpoint, creating a live endpoint to serve the model we prefer is very straightforward."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get the model from the Model Registry \n",
|
||||
"model = aip.Model(model_name=f'{MODEL_NAME}-fraction-10')\n",
|
||||
"\n",
|
||||
"# let's create a Vertex Endpoint where we will deploy the ML model\n",
|
||||
"endpoint = aip.Endpoint.create(\n",
|
||||
" display_name=ENDPOINT_DISPLAY_NAME,\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# deploy the BigQuery ML model on Vertex Endpoint\n",
|
||||
"# have a coffe - this step can take up 10/15 minutes to finish\n",
|
||||
"model.deploy(endpoint=endpoint, deployed_model_display_name='bqml-deployed-model')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Let's get a prediction from new data\n",
|
||||
"inference_test = {\n",
|
||||
" 'postal_code': '97700-000',\n",
|
||||
" 'number_of_successful_orders': 0,\n",
|
||||
" 'city': 'Santiago',\n",
|
||||
" 'sum_previous_orders': 1,\n",
|
||||
" 'number_of_unsuccessful_orders': 0,\n",
|
||||
" 'day_of_week': 'WEEKDAY',\n",
|
||||
" 'traffic_source': 'Facebook',\n",
|
||||
" 'browser': 'Firefox',\n",
|
||||
" 'hour_of_day': 20\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"my_prediction = endpoint.predict([inference_test])\n",
|
||||
"\n",
|
||||
"my_prediction"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# batch prediction on BigQuery\n",
|
||||
"\n",
|
||||
"explain_predict_query = \"\"\"\n",
|
||||
"SELECT *\n",
|
||||
"FROM ML.EXPLAIN_PREDICT(MODEL `{project_id}.{dataset}.{model_name}`,\n",
|
||||
" (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) \n",
|
||||
" FROM `{project_id}.{dataset}.ecommerce_abt`\n",
|
||||
" WHERE extract(ISOYEAR FROM session_starting_ts) = 2023),\n",
|
||||
" STRUCT(5 AS top_k_features, 0.5 AS threshold))\n",
|
||||
"LIMIT 100\n",
|
||||
"\"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# batch prediction on BigQuery\n",
|
||||
"\n",
|
||||
"with open(\"sql/explain_predict.sql\") as file:\n",
|
||||
" explain_predict_query = file.read()\n",
|
||||
"\n",
|
||||
"client = bigquery_client = bigquery.Client(location=LOCATION, project=PROJECT_ID)\n",
|
||||
"batch_predictions = bigquery_client.query(\n",
|
||||
" explain_predict_query.format(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" dataset=DATASET,\n",
|
||||
" model_name=f'{MODEL_NAME}-fraction-10')\n",
|
||||
" ).to_dataframe()\n",
|
||||
"\n",
|
||||
"batch_predictions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Conclusions\n",
|
||||
"\n",
|
||||
"Thanks to this tutorial we were able to:\n",
|
||||
"- Define a re-usable Vertex AI pipeline to train and evaluate BQ ML models\n",
|
||||
"- Use a Vertex AI Experiment to keep track of multiple trainings for the same model with different paramenters (in this case a different split for train/test data)\n",
|
||||
"- Deploy the preferred model on a Vertex AI managed Endpoint in order to serve the model for real-time use cases via API\n",
|
||||
"- Make batch prediction via Big Query and see what are the top 5 features which influenced the algorithm output"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.8.9"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
kfp==1.8.19
|
||||
google-cloud-pipeline-components==1.0.39
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
|
|
@ -0,0 +1,65 @@
|
|||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Core resources.
|
||||
|
||||
locals {
|
||||
service_encryption_keys = var.service_encryption_keys
|
||||
shared_vpc_project = try(var.vpc_config.host_project, null)
|
||||
subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.vpc_config.subnet_self_link
|
||||
: values(module.vpc.0.subnet_self_links)[0]
|
||||
)
|
||||
use_shared_vpc = var.vpc_config != null
|
||||
vpc = (
|
||||
local.use_shared_vpc
|
||||
? var.vpc_config.network_self_link
|
||||
: module.vpc.0.self_link
|
||||
)
|
||||
}
|
||||
|
||||
module "project" {
|
||||
source = "../../../modules/project"
|
||||
name = var.project_id
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = var.project_create != null
|
||||
prefix = var.project_create == null ? null : var.prefix
|
||||
services = [
|
||||
"aiplatform.googleapis.com",
|
||||
"bigquery.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"ml.googleapis.com",
|
||||
"notebooks.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"stackdriver.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
]
|
||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||
attach = true
|
||||
host_project = local.shared_vpc_project
|
||||
}
|
||||
service_encryption_key_ids = {
|
||||
compute = [try(local.service_encryption_keys.compute, null)]
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
service_config = {
|
||||
disable_on_destroy = false, disable_dependent_services = false
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Output variables.
|
||||
|
||||
output "bucket" {
|
||||
description = "GCS Bucket URL."
|
||||
value = module.bucket.url
|
||||
}
|
||||
|
||||
output "dataset" {
|
||||
description = "GCS Bucket URL."
|
||||
value = module.dataset.id
|
||||
}
|
||||
|
||||
output "notebook" {
|
||||
description = "Vertex AI notebook details."
|
||||
value = {
|
||||
name = resource.google_notebooks_instance.playground.name
|
||||
id = resource.google_notebooks_instance.playground.id
|
||||
}
|
||||
}
|
||||
|
||||
output "project" {
|
||||
description = "Project id."
|
||||
value = module.project.project_id
|
||||
}
|
||||
|
||||
output "service-account-vertex" {
|
||||
description = "Service account to be used for Vertex AI pipelines."
|
||||
value = module.service-account-vertex.email
|
||||
}
|
||||
|
||||
output "vertex-ai-metadata-store" {
|
||||
description = "Vertex AI Metadata Store ID."
|
||||
value = google_vertex_ai_metadata_store.store.id
|
||||
}
|
||||
|
||||
output "vpc" {
|
||||
description = "VPC Network."
|
||||
value = local.vpc
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Terraform variables.
|
||||
|
||||
variable "location" {
|
||||
description = "The location where resources will be deployed."
|
||||
type = string
|
||||
default = "US"
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
description = "Prefix used for resource names."
|
||||
type = string
|
||||
validation {
|
||||
condition = var.prefix != ""
|
||||
error_message = "Prefix cannot be empty."
|
||||
}
|
||||
}
|
||||
|
||||
variable "project_create" {
|
||||
description = "Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id."
|
||||
type = object({
|
||||
billing_account_id = string
|
||||
parent = string
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Project id references existing project if `project_create` is null."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
description = "The region where resources will be deployed."
|
||||
type = string
|
||||
default = "us-central1"
|
||||
}
|
||||
|
||||
variable "service_encryption_keys" {
|
||||
description = "Cloud KMS to use to encrypt different services. The key location should match the service region."
|
||||
type = object({
|
||||
bq = string
|
||||
compute = string
|
||||
storage = string
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "vpc_config" {
|
||||
description = "Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values."
|
||||
type = object({
|
||||
host_project = string
|
||||
network_self_link = string
|
||||
subnet_self_link = string
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
terraform {
|
||||
required_version = ">= 1.3.1"
|
||||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = ">= 4.55.0" # tftest
|
||||
}
|
||||
google-beta = {
|
||||
source = "hashicorp/google-beta"
|
||||
version = ">= 4.55.0" # tftest
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Vertex resources.
|
||||
|
||||
resource "google_vertex_ai_metadata_store" "store" {
|
||||
provider = google-beta
|
||||
project = module.project.project_id
|
||||
name = "default" #"${var.prefix}-metadata-store"
|
||||
description = "Vertex Ai Metadata Store"
|
||||
region = var.region
|
||||
#TODO Check/Implement P4SA logic for IAM role
|
||||
# encryption_spec {
|
||||
# kms_key_name = var.service_encryption_keys.ai_metadata_store
|
||||
# }
|
||||
}
|
||||
|
||||
module "service-account-notebook" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.project.project_id
|
||||
name = "notebook-sa"
|
||||
iam_project_roles = {
|
||||
(module.project.project_id) = [
|
||||
"roles/bigquery.admin",
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/bigquery.user",
|
||||
"roles/dialogflow.client",
|
||||
"roles/storage.admin",
|
||||
"roles/aiplatform.user",
|
||||
"roles/iam.serviceAccountUser"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "service-account-vertex" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.project.project_id
|
||||
name = "vertex-sa"
|
||||
iam_project_roles = {
|
||||
(module.project.project_id) = [
|
||||
"roles/bigquery.admin",
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/bigquery.user",
|
||||
"roles/dialogflow.client",
|
||||
"roles/storage.admin",
|
||||
"roles/aiplatform.user"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_notebooks_instance" "playground" {
|
||||
name = "${var.prefix}-notebook"
|
||||
location = format("%s-%s", var.region, "b")
|
||||
machine_type = "e2-medium"
|
||||
project = module.project.project_id
|
||||
|
||||
container_image {
|
||||
repository = "gcr.io/deeplearning-platform-release/base-cpu"
|
||||
tag = "latest"
|
||||
}
|
||||
|
||||
install_gpu_driver = true
|
||||
boot_disk_type = "PD_SSD"
|
||||
boot_disk_size_gb = 110
|
||||
disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null
|
||||
kms_key = try(local.service_encryption_keys.compute, null)
|
||||
|
||||
no_public_ip = true
|
||||
no_proxy_access = false
|
||||
|
||||
network = local.vpc
|
||||
subnet = local.subnet
|
||||
|
||||
service_account = module.service-account-notebook.email
|
||||
|
||||
# Enable Secure Boot
|
||||
shielded_instance_config {
|
||||
enable_secure_boot = true
|
||||
}
|
||||
|
||||
# Remove once terraform-provider-google/issues/9164 is fixed
|
||||
lifecycle {
|
||||
ignore_changes = [disk_encryption, kms_key]
|
||||
}
|
||||
|
||||
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
|
||||
# tags = ["ssh"]
|
||||
depends_on = [
|
||||
google_project_iam_member.shared_vpc,
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description VPC resources.
|
||||
|
||||
module "vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
name = "${var.prefix}-vpc"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.0.0.0/20"
|
||||
name = "${var.prefix}-subnet"
|
||||
region = var.region
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
module "vpc-firewall" {
|
||||
source = "../../../modules/net-vpc-firewall"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
network = module.vpc.0.name
|
||||
default_rules_config = {
|
||||
admin_ranges = ["10.0.0.0/20"]
|
||||
}
|
||||
ingress_rules = {
|
||||
#TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed
|
||||
("${var.prefix}-iap") = {
|
||||
description = "Enable SSH from IAP on Notebooks."
|
||||
source_ranges = ["35.235.240.0/20"]
|
||||
targets = ["notebook-instance"]
|
||||
rules = [{ protocol = "tcp", ports = [22] }]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module "cloudnat" {
|
||||
source = "../../../modules/net-cloudnat"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
region = var.region
|
||||
router_network = module.vpc.0.name
|
||||
}
|
||||
|
||||
resource "google_project_iam_member" "shared_vpc" {
|
||||
count = local.use_shared_vpc ? 1 : 0
|
||||
project = var.vpc_config.host_project
|
||||
role = "roles/compute.networkUser"
|
||||
member = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
|
||||
}
|
||||
|
|
@ -116,8 +116,27 @@ Initial population depends on a modules repository being configured in the `modu
|
|||
|
||||
### Commit configuration
|
||||
|
||||
Finally, a `commit_config` variable is optional: it can be used to configure author, email and message used in commits for initial population of files, its defaults are probably fine for most use cases.
|
||||
An optional variable `commit_config` can be used to configure the author, email, and message used in commits for the initial population of files. Its defaults are probably fine for most use cases.
|
||||
|
||||
### Pull Request configuration
|
||||
|
||||
An optional variable `pull_request_config` can be used to configure the title, body, head_ref, and base_ref of the pull request created for the initial population or update of files. By default, no pull request is created. To create a pull request, set the `create` attribute to `true`. `base_ref` defaults to `main` and `head_ref` to the head branch name. If the head branch does not exist, it will be created from the base_ref branch.
|
||||
|
||||
```hcl
|
||||
pull_request_config = {
|
||||
create = true
|
||||
title = "FAST: initial loading or update"
|
||||
body = ""
|
||||
base_ref = "main"
|
||||
head_ref = "fast-loader"
|
||||
}
|
||||
# tftest skip
|
||||
```
|
||||
|
||||
To start using a pull request workflow, if the initial loading was created without a pull request in the past, please use the following command to delete the actual branch files from the Terraform state to keep it in the current state:
|
||||
```bash
|
||||
terraform state list | grep github_repository_file | awk '{print "terraform state rm '\''"$1"'\''"}'
|
||||
```
|
||||
<!-- TFDOC OPTS files:1 -->
|
||||
<!-- BEGIN TFDOC -->
|
||||
|
||||
|
|
@ -126,7 +145,7 @@ Finally, a `commit_config` variable is optional: it can be used to configure aut
|
|||
| name | description | resources |
|
||||
|---|---|---|
|
||||
| [cicd-versions.tf](./cicd-versions.tf) | Provider version. | |
|
||||
| [main.tf](./main.tf) | Module-level locals and resources. | <code>github_actions_secret</code> · <code>github_repository</code> · <code>github_repository_deploy_key</code> · <code>github_repository_file</code> · <code>tls_private_key</code> |
|
||||
| [main.tf](./main.tf) | Module-level locals and resources. | <code>github_actions_secret</code> · <code>github_branch</code> · <code>github_repository</code> · <code>github_repository_deploy_key</code> · <code>github_repository_file</code> · <code>github_repository_pull_request</code> · <code>tls_private_key</code> |
|
||||
| [outputs.tf](./outputs.tf) | Module outputs. | |
|
||||
| [providers.tf](./providers.tf) | Provider configuration. | |
|
||||
| [variables.tf](./variables.tf) | Module variables. | |
|
||||
|
|
@ -138,7 +157,8 @@ Finally, a `commit_config` variable is optional: it can be used to configure aut
|
|||
| [organization](variables.tf#L51) | GitHub organization. | <code>string</code> | ✓ | |
|
||||
| [commmit_config](variables.tf#L17) | Configure commit metadata. | <code title="object({ author = optional(string, "FAST loader") email = optional(string, "fast-loader@fast.gcp.tf") message = optional(string, "FAST initial loading") })">object({…})</code> | | <code>{}</code> |
|
||||
| [modules_config](variables.tf#L28) | Configure access to repository module via key, and replacement for modules sources in stage repositories. | <code title="object({ repository_name = string source_ref = optional(string) module_prefix = optional(string, "") key_config = optional(object({ create_key = optional(bool, false) create_secrets = optional(bool, false) keypair_path = optional(string) }), {}) })">object({…})</code> | | <code>null</code> |
|
||||
| [repositories](variables.tf#L56) | Repositories to create. | <code title="map(object({ create_options = optional(object({ allow = optional(object({ auto_merge = optional(bool) merge_commit = optional(bool) rebase_merge = optional(bool) squash_merge = optional(bool) })) auto_init = optional(bool) description = optional(string) features = optional(object({ issues = optional(bool) projects = optional(bool) wiki = optional(bool) })) templates = optional(object({ gitignore = optional(string, "Terraform") license = optional(string) repository = optional(object({ name = string owner = string })) }), {}) visibility = optional(string, "private") })) populate_from = optional(string) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [pull_request_config](variables.tf#L56) | Configure pull request metadata. | <code title="object({ create = optional(bool, false) title = optional(string, "FAST: initial loading or update") body = optional(string, "") base_ref = optional(string, "main") head_ref = optional(string, "fast-loader") })">object({…})</code> | | <code>{}</code> |
|
||||
| [repositories](variables.tf#L69) | Repositories to create. | <code title="map(object({ create_options = optional(object({ allow = optional(object({ auto_merge = optional(bool) merge_commit = optional(bool) rebase_merge = optional(bool) squash_merge = optional(bool) })) auto_init = optional(bool) description = optional(string) features = optional(object({ issues = optional(bool) projects = optional(bool) wiki = optional(bool) })) templates = optional(object({ gitignore = optional(string, "Terraform") license = optional(string) repository = optional(object({ name = string owner = string })) }), {}) visibility = optional(string, "private") })) populate_from = optional(string) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -136,10 +136,21 @@ resource "github_actions_secret" "default" {
|
|||
)
|
||||
}
|
||||
|
||||
resource "github_branch" "default" {
|
||||
for_each = (
|
||||
try(var.pull_request_config.create, null) == true
|
||||
? local.repositories
|
||||
: {}
|
||||
)
|
||||
repository = each.key
|
||||
branch = var.pull_request_config.head_ref
|
||||
source_branch = var.pull_request_config.base_ref
|
||||
}
|
||||
|
||||
resource "github_repository_file" "default" {
|
||||
for_each = local.modules_repo == null ? {} : local.repository_files
|
||||
repository = local.repositories[each.value.repository]
|
||||
branch = "main"
|
||||
branch = try(var.pull_request_config.head_ref, "main")
|
||||
file = each.value.name
|
||||
content = (
|
||||
endswith(each.value.name, ".tf") && local.modules_repo != null
|
||||
|
|
@ -154,4 +165,23 @@ resource "github_repository_file" "default" {
|
|||
commit_author = var.commmit_config.author
|
||||
commit_email = var.commmit_config.email
|
||||
overwrite_on_create = true
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = [
|
||||
content,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
resource "github_repository_pull_request" "default" {
|
||||
for_each = (
|
||||
try(var.pull_request_config.create, null) == true
|
||||
? local.repositories
|
||||
: {}
|
||||
)
|
||||
base_repository = each.key
|
||||
title = var.pull_request_config.title
|
||||
body = var.pull_request_config.body
|
||||
base_ref = var.pull_request_config.base_ref
|
||||
head_ref = var.pull_request_config.head_ref
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,6 +53,19 @@ variable "organization" {
|
|||
type = string
|
||||
}
|
||||
|
||||
variable "pull_request_config" {
|
||||
description = "Configure pull request metadata."
|
||||
type = object({
|
||||
create = optional(bool, false)
|
||||
title = optional(string, "FAST: initial loading or update")
|
||||
body = optional(string, "")
|
||||
base_ref = optional(string, "main")
|
||||
head_ref = optional(string, "fast-loader")
|
||||
})
|
||||
default = {}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "repositories" {
|
||||
description = "Repositories to create."
|
||||
type = map(object({
|
||||
|
|
|
|||
|
|
@ -347,20 +347,19 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
|
|||
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | <code title="object({ id = string is_org_level = optional(bool, true) })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L92) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code title="object({ networking = string networking-dev = string networking-prod = string })">object({…})</code> | ✓ | | <code>1-resman</code> |
|
||||
| [organization](variables.tf#L126) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L142) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [organization](variables.tf#L102) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev = "10.128.32.0/19" gcp_landing = "10.128.0.0/19" gcp_prod = "10.128.64.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||
| [custom_roles](variables.tf#L55) | Custom roles defined at the org level, in key => id format. | <code title="object({ service_project_network_admin = string })">object({…})</code> | | <code>null</code> | <code>0-bootstrap</code> |
|
||||
| [dns](variables.tf#L64) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
||||
| [factories_config](variables.tf#L72) | Configuration for network resource factories. | <code title="object({ data_dir = optional(string, "data") firewall_policy_name = optional(string, "factory") })">object({…})</code> | | <code title="{ data_dir = "data" }">{…}</code> | |
|
||||
| [l7ilb_subnets](variables.tf#L102) | Subnets used for L7 ILBs. | <code title="object({ dev = optional(list(object({ ip_cidr_range = string region = string })), []) prod = optional(list(object({ ip_cidr_range = string region = string })), []) })">object({…})</code> | | <code title="{ dev = [ { ip_cidr_range = "10.128.60.0/24", region = "primary" }, { ip_cidr_range = "10.128.61.0/24", region = "secondary" } ] prod = [ { ip_cidr_range = "10.128.92.0/24", region = "primary" }, { ip_cidr_range = "10.128.93.0/24", region = "secondary" } ] }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L136) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [peering_configs](variables-peerings.tf#L19) | Peering configurations. | <code title="map(object({ export_local_custom_routes = bool export_peer_custom_routes = bool }))">map(object({…}))</code> | | <code title="{ dev = { export_local_custom_routes = true export_peer_custom_routes = true } prod = { export_local_custom_routes = true export_peer_custom_routes = true } }">{…}</code> | |
|
||||
| [psa_ranges](variables.tf#L153) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L190) | Region definitions. | <code title="object({ primary = string secondary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" secondary = "europe-west4" }">{…}</code> | |
|
||||
| [router_onprem_configs](variables.tf#L202) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-primary = { asn = "65533" adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L220) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string gke-dev = string gke-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L234) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L166) | Region definitions. | <code title="object({ primary = string secondary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" secondary = "europe-west4" }">{…}</code> | |
|
||||
| [router_onprem_configs](variables.tf#L178) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-primary = { asn = "65533" adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L196) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string gke-dev = string gke-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L210) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
region: europe-west1
|
||||
description: Default subnet for dev Data Platform
|
||||
ip_cidr_range: 10.128.48.0/24
|
||||
secondary_ip_range:
|
||||
secondary_ip_ranges:
|
||||
pods: 100.128.48.0/20
|
||||
services: 100.255.48.0/24
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
region: europe-west1
|
||||
description: Default subnet for prod gke nodes
|
||||
ip_cidr_range: 10.64.0.0/24
|
||||
secondary_ip_range:
|
||||
secondary_ip_ranges:
|
||||
pods: 100.64.0.0/16
|
||||
services: 192.168.1.0/24
|
||||
|
|
|
|||
|
|
@ -16,19 +16,6 @@
|
|||
|
||||
# tfdoc:file:description Dev spoke VPC and related resources.
|
||||
|
||||
locals {
|
||||
_l7ilb_subnets_dev = [
|
||||
for v in var.l7ilb_subnets.dev : merge(v, {
|
||||
active = true
|
||||
region = lookup(var.regions, v.region, v.region)
|
||||
})]
|
||||
l7ilb_subnets_dev = [
|
||||
for v in local._l7ilb_subnets_dev : merge(v, {
|
||||
name = "dev-l7ilb-${local.region_shortnames[v.region]}"
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
module "dev-spoke-project" {
|
||||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account.id
|
||||
|
|
@ -57,13 +44,12 @@ module "dev-spoke-project" {
|
|||
}
|
||||
|
||||
module "dev-spoke-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.dev-spoke-project.project_id
|
||||
name = "dev-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets_dev
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.dev-spoke-project.project_id
|
||||
name = "dev-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
# set explicit routes for googleapis in case the default route is deleted
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -16,19 +16,6 @@
|
|||
|
||||
# tfdoc:file:description Production spoke VPC and related resources.
|
||||
|
||||
locals {
|
||||
_l7ilb_subnets_prod = [
|
||||
for v in var.l7ilb_subnets.prod : merge(v, {
|
||||
active = true
|
||||
region = lookup(var.regions, v.region, v.region)
|
||||
})]
|
||||
l7ilb_subnets_prod = [
|
||||
for v in local._l7ilb_subnets_prod : merge(v, {
|
||||
name = "prod-l7ilb-${local.region_shortnames[v.region]}"
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
module "prod-spoke-project" {
|
||||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account.id
|
||||
|
|
@ -57,13 +44,12 @@ module "prod-spoke-project" {
|
|||
}
|
||||
|
||||
module "prod-spoke-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.prod-spoke-project.project_id
|
||||
name = "prod-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets_prod
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.prod-spoke-project.project_id
|
||||
name = "prod-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
# set explicit routes for googleapis in case the default route is deleted
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -99,30 +99,6 @@ variable "folder_ids" {
|
|||
})
|
||||
}
|
||||
|
||||
variable "l7ilb_subnets" {
|
||||
description = "Subnets used for L7 ILBs."
|
||||
type = object({
|
||||
dev = optional(list(object({
|
||||
ip_cidr_range = string
|
||||
region = string
|
||||
})), [])
|
||||
prod = optional(list(object({
|
||||
ip_cidr_range = string
|
||||
region = string
|
||||
})), [])
|
||||
})
|
||||
default = {
|
||||
dev = [
|
||||
{ ip_cidr_range = "10.128.60.0/24", region = "primary" },
|
||||
{ ip_cidr_range = "10.128.61.0/24", region = "secondary" }
|
||||
]
|
||||
prod = [
|
||||
{ ip_cidr_range = "10.128.92.0/24", region = "primary" },
|
||||
{ ip_cidr_range = "10.128.93.0/24", region = "secondary" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
variable "organization" {
|
||||
# tfdoc:variable:source 0-bootstrap
|
||||
description = "Organization details."
|
||||
|
|
|
|||
|
|
@ -372,20 +372,19 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
|
|||
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | <code title="object({ id = string is_org_level = optional(bool, true) })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L92) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code title="object({ networking = string networking-dev = string networking-prod = string })">object({…})</code> | ✓ | | <code>1-resman</code> |
|
||||
| [organization](variables.tf#L126) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L142) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [organization](variables.tf#L102) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev = "10.128.32.0/19" gcp_landing = "10.128.0.0/19" gcp_prod = "10.128.64.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||
| [custom_roles](variables.tf#L55) | Custom roles defined at the org level, in key => id format. | <code title="object({ service_project_network_admin = string })">object({…})</code> | | <code>null</code> | <code>0-bootstrap</code> |
|
||||
| [dns](variables.tf#L64) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
||||
| [factories_config](variables.tf#L72) | Configuration for network resource factories. | <code title="object({ data_dir = optional(string, "data") firewall_policy_name = optional(string, "factory") })">object({…})</code> | | <code title="{ data_dir = "data" }">{…}</code> | |
|
||||
| [l7ilb_subnets](variables.tf#L102) | Subnets used for L7 ILBs. | <code title="object({ dev = optional(list(object({ ip_cidr_range = string region = string })), []) prod = optional(list(object({ ip_cidr_range = string region = string })), []) })">object({…})</code> | | <code title="{ dev = [ { ip_cidr_range = "10.128.60.0/24", region = "primary" }, { ip_cidr_range = "10.128.61.0/24", region = "secondary" } ] prod = [ { ip_cidr_range = "10.128.92.0/24", region = "primary" }, { ip_cidr_range = "10.128.93.0/24", region = "secondary" } ] }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L136) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [psa_ranges](variables.tf#L153) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L190) | Region definitions. | <code title="object({ primary = string secondary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" secondary = "europe-west4" }">{…}</code> | |
|
||||
| [router_onprem_configs](variables.tf#L202) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-primary = { asn = "65533" adv = null } }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L166) | Region definitions. | <code title="object({ primary = string secondary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" secondary = "europe-west4" }">{…}</code> | |
|
||||
| [router_onprem_configs](variables.tf#L178) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-primary = { asn = "65533" adv = null } }">{…}</code> | |
|
||||
| [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-primary = { asn = "64512", adv = null } landing-secondary = { asn = "64512", adv = null } spoke-dev-primary = { asn = "64513", adv = null } spoke-dev-secondary = { asn = "64513", adv = null } spoke-prod-primary = { asn = "64514", adv = null } spoke-prod-secondary = { asn = "64514", adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L220) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string gke-dev = string gke-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L234) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L196) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string gke-dev = string gke-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L210) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | <code title="map(object({ default = bool custom = list(string) }))">map(object({…}))</code> | | <code title="{ landing-primary = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } landing-secondary = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } dev-primary = { default = false custom = ["gcp_dev"] } prod-primary = { default = false custom = ["gcp_prod"] } prod-secondary = { default = false custom = ["gcp_prod"] } }">{…}</code> | |
|
||||
|
||||
## Outputs
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
region: europe-west1
|
||||
description: Default subnet for dev Data Platform
|
||||
ip_cidr_range: 10.128.48.0/24
|
||||
secondary_ip_range:
|
||||
secondary_ip_ranges:
|
||||
pods: 100.128.48.0/20
|
||||
services: 100.255.48.0/24
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
region: europe-west1
|
||||
description: Default subnet for prod gke nodes
|
||||
ip_cidr_range: 10.64.0.0/24
|
||||
secondary_ip_range:
|
||||
secondary_ip_ranges:
|
||||
pods: 100.64.0.0/16
|
||||
services: 192.168.1.0/24
|
||||
|
|
|
|||
|
|
@ -16,19 +16,6 @@
|
|||
|
||||
# tfdoc:file:description Dev spoke VPC and related resources.
|
||||
|
||||
locals {
|
||||
_l7ilb_subnets_dev = [
|
||||
for v in var.l7ilb_subnets.dev : merge(v, {
|
||||
active = true
|
||||
region = lookup(var.regions, v.region, v.region)
|
||||
})]
|
||||
l7ilb_subnets_dev = [
|
||||
for v in local._l7ilb_subnets_dev : merge(v, {
|
||||
name = "dev-l7ilb-${local.region_shortnames[v.region]}"
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
module "dev-spoke-project" {
|
||||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account.id
|
||||
|
|
@ -57,13 +44,12 @@ module "dev-spoke-project" {
|
|||
}
|
||||
|
||||
module "dev-spoke-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.dev-spoke-project.project_id
|
||||
name = "dev-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets_dev
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.dev-spoke-project.project_id
|
||||
name = "dev-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
# set explicit routes for googleapis in case the default route is deleted
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -16,19 +16,6 @@
|
|||
|
||||
# tfdoc:file:description Production spoke VPC and related resources.
|
||||
|
||||
locals {
|
||||
_l7ilb_subnets_prod = [
|
||||
for v in var.l7ilb_subnets.prod : merge(v, {
|
||||
active = true
|
||||
region = lookup(var.regions, v.region, v.region)
|
||||
})]
|
||||
l7ilb_subnets_prod = [
|
||||
for v in local._l7ilb_subnets_prod : merge(v, {
|
||||
name = "prod-l7ilb-${local.region_shortnames[v.region]}"
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
module "prod-spoke-project" {
|
||||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account.id
|
||||
|
|
@ -57,13 +44,12 @@ module "prod-spoke-project" {
|
|||
}
|
||||
|
||||
module "prod-spoke-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.prod-spoke-project.project_id
|
||||
name = "prod-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets_prod
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.prod-spoke-project.project_id
|
||||
name = "prod-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
# set explicit routes for googleapis in case the default route is deleted
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -99,30 +99,6 @@ variable "folder_ids" {
|
|||
})
|
||||
}
|
||||
|
||||
variable "l7ilb_subnets" {
|
||||
description = "Subnets used for L7 ILBs."
|
||||
type = object({
|
||||
dev = optional(list(object({
|
||||
ip_cidr_range = string
|
||||
region = string
|
||||
})), [])
|
||||
prod = optional(list(object({
|
||||
ip_cidr_range = string
|
||||
region = string
|
||||
})), [])
|
||||
})
|
||||
default = {
|
||||
dev = [
|
||||
{ ip_cidr_range = "10.128.60.0/24", region = "primary" },
|
||||
{ ip_cidr_range = "10.128.61.0/24", region = "secondary" }
|
||||
]
|
||||
prod = [
|
||||
{ ip_cidr_range = "10.128.92.0/24", region = "primary" },
|
||||
{ ip_cidr_range = "10.128.93.0/24", region = "secondary" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
variable "organization" {
|
||||
# tfdoc:variable:source 0-bootstrap
|
||||
description = "Organization details."
|
||||
|
|
|
|||
|
|
@ -421,20 +421,19 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
|
|||
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | <code title="object({ id = string is_org_level = optional(bool, true) })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L97) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code title="object({ networking = string networking-dev = string networking-prod = string })">object({…})</code> | ✓ | | <code>1-resman</code> |
|
||||
| [organization](variables.tf#L133) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L149) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [organization](variables.tf#L115) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L131) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev_primary = "10.128.128.0/19" gcp_dev_secondary = "10.128.160.0/19" gcp_landing_trusted_primary = "10.128.64.0/19" gcp_landing_trusted_secondary = "10.128.96.0/19" gcp_landing_untrusted_primary = "10.128.0.0/19" gcp_landing_untrusted_secondary = "10.128.32.0/19" gcp_prod_primary = "10.128.192.0/19" gcp_prod_secondary = "10.128.224.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||
| [custom_roles](variables.tf#L60) | Custom roles defined at the org level, in key => id format. | <code title="object({ service_project_network_admin = string })">object({…})</code> | | <code>null</code> | <code>0-bootstrap</code> |
|
||||
| [dns](variables.tf#L69) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
||||
| [factories_config](variables.tf#L77) | Configuration for network resource factories. | <code title="object({ data_dir = optional(string, "data") firewall_policy_name = optional(string, "factory") })">object({…})</code> | | <code title="{ data_dir = "data" }">{…}</code> | |
|
||||
| [l7ilb_subnets](variables.tf#L107) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ dev = [ { ip_cidr_range = "10.128.159.0/24", region = "primary" }, { ip_cidr_range = "10.128.191.0/24", region = "secondary" } ] prod = [ { ip_cidr_range = "10.128.223.0/24", region = "primary" }, { ip_cidr_range = "10.128.255.0/24", region = "secondary" } ] }">{…}</code> | |
|
||||
| [onprem_cidr](variables.tf#L125) | Onprem addresses in name => range format. | <code>map(string)</code> | | <code title="{ main = "10.0.0.0/24" }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L143) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [psa_ranges](variables.tf#L160) | IP ranges used for Private Service Access (e.g. CloudSQL). Ranges is in name => range format. | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L181) | Region definitions. | <code title="object({ primary = string secondary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" secondary = "europe-west4" }">{…}</code> | |
|
||||
| [router_configs](variables.tf#L193) | Configurations for CRs and onprem routers. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-trusted-primary = { asn = "64512" adv = null } landing-trusted-secondary = { asn = "64512" adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L216) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string gke-dev = string gke-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L230) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-trusted-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } landing-trusted-secondary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [onprem_cidr](variables.tf#L107) | Onprem addresses in name => range format. | <code>map(string)</code> | | <code title="{ main = "10.0.0.0/24" }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L125) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [psa_ranges](variables.tf#L142) | IP ranges used for Private Service Access (e.g. CloudSQL). Ranges is in name => range format. | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L163) | Region definitions. | <code title="object({ primary = string secondary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" secondary = "europe-west4" }">{…}</code> | |
|
||||
| [router_configs](variables.tf#L175) | Configurations for CRs and onprem routers. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-trusted-primary = { asn = "64512" adv = null } landing-trusted-secondary = { asn = "64512" adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L198) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string gke-dev = string gke-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L212) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-trusted-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } landing-trusted-secondary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -18,12 +18,6 @@
|
|||
|
||||
locals {
|
||||
custom_roles = coalesce(var.custom_roles, {})
|
||||
l7ilb_subnets = { for env, v in var.l7ilb_subnets : env => [
|
||||
for s in v : merge(s, {
|
||||
active = true
|
||||
name = "${env}-l7ilb-${s.region}"
|
||||
})]
|
||||
}
|
||||
# combine all regions from variables and subnets
|
||||
regions = distinct(concat(
|
||||
values(var.regions),
|
||||
|
|
|
|||
|
|
@ -16,19 +16,6 @@
|
|||
|
||||
# tfdoc:file:description Dev spoke VPC and related resources.
|
||||
|
||||
locals {
|
||||
_l7ilb_subnets_dev = [
|
||||
for v in var.l7ilb_subnets.dev : merge(v, {
|
||||
active = true
|
||||
region = lookup(var.regions, v.region, v.region)
|
||||
})]
|
||||
l7ilb_subnets_dev = [
|
||||
for v in local._l7ilb_subnets_dev : merge(v, {
|
||||
name = "dev-l7ilb-${local.region_shortnames[v.region]}"
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
module "dev-spoke-project" {
|
||||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account.id
|
||||
|
|
@ -63,7 +50,6 @@ module "dev-spoke-vpc" {
|
|||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
delete_default_routes_on_create = true
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets_dev
|
||||
# Set explicit routes for googleapis; send everything else to NVAs
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -16,19 +16,6 @@
|
|||
|
||||
# tfdoc:file:description Production spoke VPC and related resources.
|
||||
|
||||
locals {
|
||||
_l7ilb_subnets_prod = [
|
||||
for v in var.l7ilb_subnets.prod : merge(v, {
|
||||
active = true
|
||||
region = lookup(var.regions, v.region, v.region)
|
||||
})]
|
||||
l7ilb_subnets_prod = [
|
||||
for v in local._l7ilb_subnets_prod : merge(v, {
|
||||
name = "prod-l7ilb-${local.region_shortnames[v.region]}"
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
module "prod-spoke-project" {
|
||||
source = "../../../modules/project"
|
||||
billing_account = var.billing_account.id
|
||||
|
|
@ -63,7 +50,6 @@ module "prod-spoke-vpc" {
|
|||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
delete_default_routes_on_create = true
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets_prod
|
||||
# Set explicit routes for googleapis; send everything else to NVAs
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -104,24 +104,6 @@ variable "folder_ids" {
|
|||
})
|
||||
}
|
||||
|
||||
variable "l7ilb_subnets" {
|
||||
description = "Subnets used for L7 ILBs."
|
||||
type = map(list(object({
|
||||
ip_cidr_range = string
|
||||
region = string
|
||||
})))
|
||||
default = {
|
||||
dev = [
|
||||
{ ip_cidr_range = "10.128.159.0/24", region = "primary" },
|
||||
{ ip_cidr_range = "10.128.191.0/24", region = "secondary" }
|
||||
]
|
||||
prod = [
|
||||
{ ip_cidr_range = "10.128.223.0/24", region = "primary" },
|
||||
{ ip_cidr_range = "10.128.255.0/24", region = "secondary" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
variable "onprem_cidr" {
|
||||
description = "Onprem addresses in name => range format."
|
||||
type = map(string)
|
||||
|
|
|
|||
|
|
@ -291,19 +291,18 @@ Regions are defined via the `regions` variable which sets up a mapping between t
|
|||
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | <code title="object({ id = string is_org_level = optional(bool, true) })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L92) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code title="object({ networking = string networking-dev = string networking-prod = string })">object({…})</code> | ✓ | | <code>1-resman</code> |
|
||||
| [organization](variables.tf#L118) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L134) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [organization](variables.tf#L102) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [custom_adv](variables.tf#L38) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev = "10.128.32.0/19" gcp_prod = "10.128.64.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||
| [custom_roles](variables.tf#L54) | Custom roles defined at the org level, in key => id format. | <code title="object({ service_project_network_admin = string })">object({…})</code> | | <code>null</code> | <code>0-bootstrap</code> |
|
||||
| [dns](variables.tf#L63) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ prod = ["10.0.1.1"] dev = ["10.0.2.1"] }">{…}</code> | |
|
||||
| [factories_config](variables.tf#L72) | Configuration for network resource factories. | <code title="object({ data_dir = optional(string, "data") firewall_policy_name = optional(string, "factory") })">object({…})</code> | | <code title="{ data_dir = "data" }">{…}</code> | |
|
||||
| [l7ilb_subnets](variables.tf#L102) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, ] }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L128) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [psa_ranges](variables.tf#L145) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L182) | Region definitions. | <code title="object({ primary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" }">{…}</code> | |
|
||||
| [router_onprem_configs](variables.tf#L192) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ prod-primary = { asn = "65533" adv = null } dev-primary = { asn = "65534" adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L215) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L227) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ dev-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_dev" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65544 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65544 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } prod-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_prod" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65543 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65543 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||
| [regions](variables.tf#L166) | Region definitions. | <code title="object({ primary = string })">object({…})</code> | | <code title="{ primary = "europe-west1" }">{…}</code> | |
|
||||
| [router_onprem_configs](variables.tf#L176) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ prod-primary = { asn = "65533" adv = null } dev-primary = { asn = "65534" adv = null } }">{…}</code> | |
|
||||
| [service_accounts](variables.tf#L199) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>1-resman</code> |
|
||||
| [vpn_onprem_configs](variables.tf#L211) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(string) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ dev-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_dev" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65544 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65544 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } prod-primary = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_prod" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = ["8.8.8.8"] } tunnels = [ { peer_asn = 65543 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65543 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
region: europe-west1
|
||||
description: Default subnet for dev Data Platform
|
||||
ip_cidr_range: 10.128.48.0/24
|
||||
secondary_ip_range:
|
||||
secondary_ip_ranges:
|
||||
pods: 100.128.48.0/20
|
||||
services: 100.255.48.0/24
|
||||
|
|
|
|||
|
|
@ -18,19 +18,6 @@
|
|||
|
||||
locals {
|
||||
custom_roles = coalesce(var.custom_roles, {})
|
||||
_l7ilb_subnets = {
|
||||
for k, v in var.l7ilb_subnets : k => [
|
||||
for s in v : merge(s, {
|
||||
active = true
|
||||
region = lookup(var.regions, s.region, s.region)
|
||||
})]
|
||||
}
|
||||
l7ilb_subnets = {
|
||||
for k, v in local._l7ilb_subnets : k => [
|
||||
for s in v : merge(s, {
|
||||
name = "${k}-l7ilb-${local.region_shortnames[s.region]}"
|
||||
})]
|
||||
}
|
||||
# combine all regions from variables and subnets
|
||||
regions = distinct(concat(
|
||||
values(var.regions),
|
||||
|
|
|
|||
|
|
@ -43,13 +43,12 @@ module "dev-spoke-project" {
|
|||
}
|
||||
|
||||
module "dev-spoke-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.dev-spoke-project.project_id
|
||||
name = "dev-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets.dev
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.dev-spoke-project.project_id
|
||||
name = "dev-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/dev"
|
||||
psa_config = try(var.psa_ranges.dev, null)
|
||||
# set explicit routes for googleapis in case the default route is deleted
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -43,13 +43,12 @@ module "prod-spoke-project" {
|
|||
}
|
||||
|
||||
module "prod-spoke-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.prod-spoke-project.project_id
|
||||
name = "prod-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
subnets_proxy_only = local.l7ilb_subnets.prod
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.prod-spoke-project.project_id
|
||||
name = "prod-spoke-0"
|
||||
mtu = 1500
|
||||
data_folder = "${var.factories_config.data_dir}/subnets/prod"
|
||||
psa_config = try(var.psa_ranges.prod, null)
|
||||
# set explicit routes for googleapis in case the default route is deleted
|
||||
routes = {
|
||||
private-googleapis = {
|
||||
|
|
|
|||
|
|
@ -99,22 +99,6 @@ variable "folder_ids" {
|
|||
})
|
||||
}
|
||||
|
||||
variable "l7ilb_subnets" {
|
||||
description = "Subnets used for L7 ILBs."
|
||||
type = map(list(object({
|
||||
ip_cidr_range = string
|
||||
region = string
|
||||
})))
|
||||
default = {
|
||||
prod = [
|
||||
{ ip_cidr_range = "10.128.92.0/24", region = "europe-west1" },
|
||||
]
|
||||
dev = [
|
||||
{ ip_cidr_range = "10.128.60.0/24", region = "europe-west1" },
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
variable "organization" {
|
||||
# tfdoc:variable:source 0-bootstrap
|
||||
description = "Organization details."
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ module "vpc" {
|
|||
```
|
||||
|
||||
### Subnet Options
|
||||
|
||||
```hcl
|
||||
module "vpc" {
|
||||
source = "./fabric/modules/net-vpc"
|
||||
|
|
@ -305,7 +306,7 @@ module "vpc" {
|
|||
|
||||
### Subnet Factory
|
||||
|
||||
The `net-vpc` module includes a subnet factory (see [Resource Factories](../../blueprints/factories/)) for the massive creation of subnets leveraging one configuration file per subnet.
|
||||
The `net-vpc` module includes a subnet factory (see [Resource Factories](../../blueprints/factories/)) for the massive creation of subnets leveraging one configuration file per subnet. The factory also supports proxy-only and PSC subnets via the `purpose` attribute.
|
||||
|
||||
```hcl
|
||||
module "vpc" {
|
||||
|
|
@ -314,7 +315,7 @@ module "vpc" {
|
|||
name = "my-network"
|
||||
data_folder = "config/subnets"
|
||||
}
|
||||
# tftest modules=1 resources=4 files=subnet-simple,subnet-detailed inventory=factory.yaml
|
||||
# tftest modules=1 resources=6 files=subnet-simple,subnet-detailed,subnet-proxy,subnet-psc inventory=factory.yaml
|
||||
```
|
||||
|
||||
```yaml
|
||||
|
|
@ -342,6 +343,20 @@ flow_logs: # enable, set to empty map to use defaults
|
|||
filter_expression: null
|
||||
```
|
||||
|
||||
```yaml
|
||||
# tftest-file id=subnet-proxy path=config/subnets/subnet-proxy.yaml
|
||||
region: europe-west4
|
||||
ip_cidr_range: 10.1.0.0/24
|
||||
purpose: REGIONAL_MANAGED_PROXY
|
||||
```
|
||||
|
||||
```yaml
|
||||
# tftest-file id=subnet-psc path=config/subnets/subnet-psc.yaml
|
||||
region: europe-west4
|
||||
ip_cidr_range: 10.2.0.0/24
|
||||
purpose: PRIVATE_SERVICE_CONNECT
|
||||
```
|
||||
|
||||
### Custom Routes
|
||||
|
||||
VPC routes can be configured through the `routes` variable.
|
||||
|
|
@ -380,7 +395,6 @@ module "vpc" {
|
|||
# tftest modules=5 resources=15 inventory=routes.yaml
|
||||
```
|
||||
|
||||
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ locals {
|
|||
iam_groups = try(v.iam_groups, [])
|
||||
iam_users = try(v.iam_users, [])
|
||||
iam_service_accounts = try(v.iam_service_accounts, [])
|
||||
purpose = try(v.purpose, null)
|
||||
active = try(v.active, null)
|
||||
}
|
||||
}
|
||||
_factory_subnets_iam = [
|
||||
|
|
@ -45,7 +47,7 @@ locals {
|
|||
formatlist("user:%s", lookup(v, "iam_users", [])),
|
||||
formatlist("serviceAccount:%s", lookup(v, "iam_service_accounts", []))
|
||||
)
|
||||
}
|
||||
} if v.purpose == null
|
||||
]
|
||||
_subnet_iam_members = flatten([
|
||||
for subnet, roles in(var.subnet_iam == null ? {} : var.subnet_iam) : [
|
||||
|
|
@ -61,17 +63,17 @@ locals {
|
|||
local._subnet_iam_members
|
||||
)
|
||||
subnets = merge(
|
||||
{ for subnet in var.subnets : "${subnet.region}/${subnet.name}" => subnet },
|
||||
local._factory_subnets
|
||||
{ for s in var.subnets : "${s.region}/${s.name}" => s },
|
||||
{ for k, v in local._factory_subnets : k => v if v.purpose == null }
|
||||
)
|
||||
subnets_proxy_only = merge(
|
||||
{ for s in var.subnets_proxy_only : "${s.region}/${s.name}" => s },
|
||||
{ for k, v in local._factory_subnets : k => v if v.purpose == "REGIONAL_MANAGED_PROXY" }
|
||||
)
|
||||
subnets_psc = merge(
|
||||
{ for s in var.subnets_psc : "${s.region}/${s.name}" => s },
|
||||
{ for k, v in local._factory_subnets : k => v if v.purpose == "PRIVATE_SERVICE_CONNECT" }
|
||||
)
|
||||
subnets_proxy_only = {
|
||||
for subnet in var.subnets_proxy_only :
|
||||
"${subnet.region}/${subnet.name}" => subnet
|
||||
}
|
||||
subnets_psc = {
|
||||
for subnet in var.subnets_psc :
|
||||
"${subnet.region}/${subnet.name}" => subnet
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_subnetwork" "subnetwork" {
|
||||
|
|
@ -120,9 +122,7 @@ resource "google_compute_subnetwork" "proxy_only" {
|
|||
: each.value.description
|
||||
)
|
||||
purpose = "REGIONAL_MANAGED_PROXY"
|
||||
role = (
|
||||
each.value.active || each.value.active == null ? "ACTIVE" : "BACKUP"
|
||||
)
|
||||
role = each.value.active != false ? "ACTIVE" : "BACKUP"
|
||||
}
|
||||
|
||||
resource "google_compute_subnetwork" "psc" {
|
||||
|
|
|
|||
|
|
@ -54,8 +54,16 @@ values:
|
|||
region: europe-west1
|
||||
role: roles/compute.networkUser
|
||||
subnetwork: subnet-detailed
|
||||
module.vpc.google_compute_subnetwork.proxy_only["europe-west4/subnet-proxy"]:
|
||||
region: europe-west4
|
||||
ip_cidr_range: 10.1.0.0/24
|
||||
purpose: REGIONAL_MANAGED_PROXY
|
||||
module.vpc.google_compute_subnetwork.psc["europe-west4/subnet-psc"]:
|
||||
region: europe-west4
|
||||
ip_cidr_range: 10.2.0.0/24
|
||||
purpose: PRIVATE_SERVICE_CONNECT
|
||||
|
||||
counts:
|
||||
google_compute_network: 1
|
||||
google_compute_subnetwork: 2
|
||||
google_compute_subnetwork: 4
|
||||
google_compute_subnetwork_iam_binding: 1
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
# limitations under the License.
|
||||
|
||||
import click
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import yaml
|
||||
|
|
|
|||
Loading…
Reference in New Issue