zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into elia-glb-mig

This commit is contained in:
Ludovico Magnocavallo 2023-01-29 12:32:10 +01:00 committed by GitHub
parent f742d66b72 7b96ed429c
commit b4b2555910
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 853 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/linting.yml vendored
View File

@ -69,4 +69,7 @@ jobs:
- name: Check python formatting
id: yapf
run: |
yapf --style="{based_on_style: google, indent_width: 2, SPLIT_BEFORE_NAMED_ASSIGNS: false}" -p -d tools/*.py
yapf --style="{based_on_style: google, indent_width: 2, SPLIT_BEFORE_NAMED_ASSIGNS: false}" -p -d \
tools/*.py \
blueprints/cloud-operations/network-dashboard/src/*py \
blueprints/cloud-operations/network-dashboard/src/plugins/*py

12
blueprints/apigee/bigquery-analytics/functions/export/package-lock.json generated
View File

@ -826,9 +826,9 @@
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
},
"node_modules/cookiejar": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.3.tgz",
"integrity": "sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ=="
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
"integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw=="
},
"node_modules/debug": {
"version": "2.6.9",
@ -3783,9 +3783,9 @@
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
},
"cookiejar": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.3.tgz",
"integrity": "sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ=="
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
"integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw=="
},
"debug": {
"version": "2.6.9",

20
blueprints/apigee/hybrid-gke/README.md
View File

@ -25,20 +25,24 @@ The diagram below depicts the architecture.
terraform apply
```
Create an A record in your DNS registrar to point the environment group hostname to the public IP address returned after the terraform configuration was applied. You might need to wait some time until the certificate is provisioned.
5. Install Apigee hybrid using de ansible playbook that is in the ansible folder by running this command
ansible-playbook playbook.yaml -vvvß
## Testing the blueprint
2. Deploy an api proxy
```
./deploy-apiproxy.sh
./deploy-apiproxy.sh apis-test
```
3. In the console check the IP address that has been allocated to the Apigee ingress gateway and send some traffic to the deployed API proxy.
3. Send a request
```
curl -k -v -H "Host:HOSTNAME" \
--resolve HOSTNAME:443:IP_ADDRESS \
https://HOSTNAME/httpbin/headers
curl -v https://HOSTNAME/httpbin/headers
```
<!-- BEGIN TFDOC -->
@ -56,4 +60,10 @@ The diagram below depicts the architecture.
| [region](variables.tf#L84) | Region. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [zone](variables.tf#L90) | Zone. | <code>string</code> | | <code>&#34;europe-west1-c&#34;</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| [ip_address](outputs.tf#L17) | GLB IP address. | |
<!-- END TFDOC -->

13
blueprints/apigee/hybrid-gke/ansible.tf
View File

@ -18,12 +18,13 @@
resource "local_file" "vars_file" {
content = yamlencode({
cluster = module.cluster.name
region = var.region
project_id = module.project.project_id
envgroup = local.envgroup
env = local.environment
hostname = var.hostname
cluster = module.cluster.name
region = var.region
project_id = module.project.project_id
envgroups = local.envgroups
environments = local.environments
service_accounts = local.google_sas
ingress_ip_name = local.ingress_ip_name
})
filename = "${path.module}/ansible/vars/vars.yaml"
file_permission = "0666"

28
blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/k8s_service_accounts.yaml Normal file
View File

@ -0,0 +1,28 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Create and annotate k8s service account
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: ServiceAccount
metadata:
name: "{{ k8s_service_account }}"
namespace: apigee
annotations:
iam.gke.io/gcp-service-account: "{{ google_service_account }}@{{ project_id }}.iam.gserviceaccount.com"
with_items: "{{ k8s_service_accounts }}"
loop_control:
loop_var: k8s_service_account

284
blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml
View File

@ -1,11 +1,11 @@
# Copyright 2023 Google LLC
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -19,18 +19,27 @@
--project {{ project_id }} \
--internal-ip
- name: Install cert-manager
shell: >
kubectl apply \
--validate=false \
-f https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml
- name: Download cert-manager
uri:
url: https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml
dest: ~/cert-manager.yaml
- name: Wait until pods are ready in cert-manager namespace
shell: >
kubectl wait --for=condition=ready pods \
-l app.kubernetes.io/instance=cert-manager \
-n cert-manager \
--timeout=90s
- name: Apply metrics-server manifest to the cluster.
kubernetes.core.k8s:
state: present
src: ~/cert-manager.yaml
- name:
kubernetes.core.k8s_info:
kind: Pod
wait: yes
label_selectors:
- "app.kubernetes.io/instance=cert-manager"
namespace: cert-manager
wait_timeout: 90
wait_condition:
type: Ready
status: True
- name: Fetch apigeectl version
uri:
@ -48,7 +57,7 @@
unarchive:
src: "~/apigeectl.tar.gz"
dest: "~"
remote_src: yes
remote_src: yes
- name: Move apigeectl folder
shell: >
@ -66,25 +75,69 @@
file:
src: ~/apigeectl/{{ item }}
dest: "~/hybrid-files/{{ item }}"
state: link
state: link
with_items:
- tools
- config
- templates
- plugins
- plugins
- name: Create service accounts
shell: >
~/hybrid-files/tools/create-service-account -i {{ project_id }} -e non-prod -d ~/hybrid-files/service-accounts
- name: Create apigee namespace
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Namespace
metadata:
name: apigee
- name: Create certificates
- name: Create k8s service accounts
include_tasks: k8s_service_accounts.yaml
vars:
google_service_account: "{{ item.key }}"
k8s_service_accounts: "{{ item.value }}"
with_dict: "{{ service_accounts }}"
- name: Set hostnames
set_fact:
hostnames: "{{ hostnames | default([]) + item.value }}"
with_dict: "{{ envgroups }}"
- name: Create certificate and private key
shell: >
openssl req \
-nodes \
-new \
-x509 \
-keyout ~/hybrid-files/certs/{{ envgroup }}.key \
-out ~/hybrid-files/certs/{{ envgroup }}.cert -subj '/CN='{{ hostname }}'' -days 3650
-keyout ~/hybrid-files/certs/server.key \
-out ~/hybrid-files/certs/server.crt \
-subj "/CN=apigee.com' \
-addext "subjectAltName={{ hostnames | map('regex_replace', '^', 'DNS:') | join(',') }}""
-days 3650
- name: Read certificate
slurp:
src: ~/hybrid-files/certs/server.crt
register: certificate_output
- name: Read private ket
slurp:
src: ~/hybrid-files/certs/server.key
register: privatekey_output
- name: Create secret
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Secret
metadata:
name: tls-hybrid-ingress
namespace: apigee
type: kubernetes.io/tls
data:
tls.crt: "{{ certificate_output.content }}"
tls.key: "{{ privatekey_output.content }}"
- name: Create overrides.yaml
template:
@ -96,48 +149,185 @@
curl -X POST -H "Authorization: Bearer $(gcloud auth print-access-token)" \
-H "Content-Type:application/json" \
"https://apigee.googleapis.com/v1/organizations/{{ project_id }}:setSyncAuthorization" \
-d '{"identities":["'"serviceAccount:apigee-non-prod@{{ project_id }}.iam.gserviceaccount.com"'"]}'
-d '{"identities":["'"serviceAccount:apigee-synchronizer@{{ project_id }}.iam.gserviceaccount.com"'"]}'
- name: Dry-run (init)
shell: >
~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client
~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client
args:
chdir: ~/hybrid-files
- name: Install the Apigee deployment services Apigee Deployment Controller and Apigee Admission Webhook.
shell: >
~/apigeectl/apigeectl init -f overrides/overrides.yaml
~/apigeectl/apigeectl init -f overrides/overrides.yaml
args:
chdir: ~/hybrid-files
chdir: ~/hybrid-files
- name: Wait until pods are ready in apigee-system namespace
shell: >
kubectl wait --for=condition=ready pods \
-l app=apigee-controller \
-n apigee-system \
--timeout=300s
- name: Wait for apigee-controller pod to be ready
kubernetes.core.k8s_info:
kind: Pod
wait: yes
label_selectors:
- "app=apigee-controller"
namespace: apigee-system
wait_timeout: 600
wait_condition:
type: Ready
status: True
- name: Wait until pods are ready in apigee namespace
shell: >
kubectl wait --for=condition=ready pods \
-l app=apigee-ingressgateway-manager \
-n apigee \
--timeout=300s
- name: Wait for apigee-selfsigned-issuer issuer to be ready
kubernetes.core.k8s_info:
kind: Issuer
wait: yes
name: apigee-selfsigned-issuer
namespace: apigee-system
wait_timeout: 600
wait_condition:
type: Ready
status: True
- name: Wait for apigee-serving-cert certificate to be ready
kubernetes.core.k8s_info:
kind: Certificate
wait: yes
name: apigee-serving-cert
namespace: apigee-system
wait_timeout: 600
wait_condition:
type: Ready
status: True
- name: Wait for apigee-resources-install job to be complete
kubernetes.core.k8s_info:
kind: Job
wait: yes
name: apigee-resources-install
namespace: apigee-system
wait_timeout: 360
wait_condition:
type: Complete
status: True
- name: Dry-run (apply)
shell: >
~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client
~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client
args:
chdir: ~/hybrid-files
- name: Install the Apigee runtime components
shell: >
~/apigeectl/apigeectl apply -f overrides/overrides.yaml
~/apigeectl/apigeectl apply -f overrides/overrides.yaml
args:
chdir: ~/hybrid-files
chdir: ~/hybrid-files
- name: Check status of the deployment
shell: >
while [ -n "$(kubectl get pods -n apigee | tail -n +2 | grep -v Running | grep -v Completed)" ]; do sleep 1; done
args:
chdir: ~/hybrid-files
- name: Wait for apigee-runtime pod to be ready
kubernetes.core.k8s_info:
kind: Pod
wait: yes
label_selectors:
- "app=apigee-runtime"
namespace: apigee
wait_timeout: 360
wait_condition:
type: Ready
status: True
- name:
kubernetes.core.k8s:
state: present
definition:
apiVersion: apigee.cloud.google.com/v1alpha1
kind: ApigeeRoute
metadata:
name: apigee-wildcard
namespace: apigee
spec:
hostnames:
- '*'
ports:
- number: 443
protocol: HTTPS
tls:
credentialName: tls-hybrid-ingress
mode: SIMPLE
selector:
app: apigee-ingressgateway
enableNonSniClient: true
- name: Create google-managed certificate
kubernetes.core.k8s:
state: present
definition:
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
name: "apigee-cert-hybrid"
namespace: apigee
spec:
domains: "{{ hostnames }}"
- name: Create backend config
kubernetes.core.k8s:
state: present
definition:
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
name: apigee-ingress-backendconfig
namespace: apigee
spec:
healthCheck:
requestPath: /healthz/ready
port: 15021
type: HTTP
logging:
enable: true
sampleRate: 0.5
- name: Create service
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Service
metadata:
name: apigee-ingressgateway-hybrid
namespace: apigee
annotations:
cloud.google.com/backend-config: '{"default": "apigee-ingress-backendconfig"}'
cloud.google.com/neg: '{"ingress": true}'
cloud.google.com/app-protocols: '{"https":"HTTPS", "status-port": "HTTP"}'
labels:
app: apigee-ingressgateway-hybrid
spec:
ports:
- name: status-port
port: 15021
targetPort: 15021
- name: https
port: 443
targetPort: 8443
selector:
app: apigee-ingressgateway
ingress_name: ingress
type: ClusterIP
- name: Create ingress
kubernetes.core.k8s:
state: present
definition:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
networking.gke.io/managed-certificates: "apigee-cert-hybrid"
kubernetes.io/ingress.global-static-ip-name: "{{ ingress_ip_name }}"
kubernetes.io/ingress.allow-http: "false"
name: xlb-apigee
namespace: apigee
spec:
defaultBackend:
service:
name: apigee-ingressgateway-hybrid
port:
number: 443

45
blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2
View File

@ -1,29 +1,26 @@
gcp:
region: {{ region }}
projectID: {{ project_id }}
workloadIdentityEnabled: true
k8sCluster:
name: {{ cluster }}
region: CLUSTER_LOCATION # Must be the closest Google Cloud region to your cluster.
region: {{ region }} # Must be the closest Google Cloud region to your cluster.
org: {{ project_id }}
instanceID: "instance-1"
instanceID: "{{ cluster }}-{{ region }}"
cassandra:
hostNetwork: false
# Set to false for single region installations and multi-region installations
# with connectivity between pods in different clusters, for example GKE installations.
# Set to true for multi-region installations with no communication between
# pods in different clusters, for example GKE On-prem, GKE on AWS, Anthos on bare metal,
# AKS, EKS, and OpenShift installations.
# See Multi-region deployment: Prerequisites
virtualhosts:
- name: {{ envgroup }}
{% for k in envgroups %}
- name: {{ k }}
sslSecret: tls-hybrid-ingress
additionalGateways: ["apigee-wildcard"]
selector:
app: apigee-ingressgateway
sslCertPath: ./certs/{{ envgroup }}.cert
sslKeyPath: ./certs/{{ envgroup }}.key
{% endfor %}
ao:
args:
@ -37,27 +34,9 @@ ingressGateways:
replicaCountMax: 10
envs:
- name: {{ env }}
serviceAccountPaths:
synchronizer: ./service-accounts/{{ project_id }}-apigee-non-prod.json
udca: ./service-accounts/{{ project_id }}-apigee-non-prod.json
runtime: ./service-accounts/{{ project_id }}-apigee-non-prod.json
mart:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
connectAgent:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
metrics:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
udca:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
watcher:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
{% for k in environments %}
- name: {{ k }}
{% endfor %}
logger:
enabled: true
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
enabled: false

67
blueprints/apigee/hybrid-gke/apigee.tf
View File

@ -15,8 +15,51 @@
*/
locals {
envgroup = "test"
environment = "apis-test"
envgroups = {
test = [var.hostname]
}
environments = {
apis-test = {
envgroups = ["test"]
}
}
org_short_name = (length(module.project.project_id) < 16 ?
module.project.project_id :
substr(module.project.project_id, 0, 15))
org_hash = format("%s-%s", local.org_short_name, substr(sha256(module.project.project_id), 0, 7))
org_env_hashes = {
for k, v in local.environments :
k => format("%s-%s-%s", local.org_short_name, length(k) < 16 ? k : substr(k, 0, 15), substr(sha256("${module.project.project_id}:${k}"), 0, 7))
}
google_sas = {
apigee-metrics = [
"apigee-metrics-sa"
]
apigee-cassandra = [
"apigee-cassandra-schema-setup-${local.org_hash}-sa",
"apigee-cassandra-user-setup-${local.org_hash}-sa"
]
apigee-mart = [
"apigee-mart-${local.org_hash}-sa",
"apigee-connect-agent-${local.org_hash}-sa"
]
apigee-watcher = [
"apigee-watcher-${local.org_hash}-sa"
]
apigee-udca = concat([
"apigee-udca-${local.org_hash}-sa"
],
[for k, v in local.org_env_hashes :
"apigee-udca-${local.org_env_hashes[k]}-sa"
])
apigee-synchronizer = [
for k, v in local.org_env_hashes :
"apigee-synchronizer-${local.org_env_hashes[k]}-sa"
]
apigee-runtime = [for k, v in local.org_env_hashes :
"apigee-runtime-${local.org_env_hashes[k]}-sa"
]
}
}
module "apigee" {
@ -26,20 +69,24 @@ module "apigee" {
analytics_region = var.region
runtime_type = "HYBRID"
}
envgroups = {
(local.envgroup) = [var.hostname]
}
environments = {
(local.environment) = {
envgroups = [local.envgroup]
}
envgroups = local.envgroups
environments = local.environments
}
module "sas" {
for_each = local.google_sas
source = "../../../modules/iam-service-account"
project_id = module.project.project_id
name = each.key
# authoritative roles granted *on* the service accounts to other identities
iam = {
"roles/iam.workloadIdentityUser" = [for v in each.value : "serviceAccount:${module.project.project_id}.svc.id.goog[apigee/${v}]"]
}
}
resource "local_file" "deploy_apiproxy_file" {
content = templatefile("${path.module}/templates/deploy-apiproxy.sh.tpl", {
org = module.project.project_id
env = local.environment
})
filename = "${path.module}/deploy-apiproxy.sh"
file_permission = "0777"

BIN
blueprints/apigee/hybrid-gke/diagram.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 35 KiB

After

Width:  |  Height:  |  Size: 35 KiB

25
blueprints/apigee/hybrid-gke/glb.tf Normal file
View File

@ -0,0 +1,25 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
ingress_ip_name = "apigee"
}
module "addresses" {
source = "../../../modules/net-address"
project_id = module.project.project_id
global_addresses = [local.ingress_ip_name]
}

9
blueprints/apigee/hybrid-gke/main.tf
View File

@ -40,5 +40,12 @@ module "project" {
"roles/resourcemanager.projectIamAdmin" = [module.mgmt_server.service_account_iam_email]
"roles/iam.serviceAccountAdmin" = [module.mgmt_server.service_account_iam_email]
"roles/iam.serviceAccountKeyAdmin" = [module.mgmt_server.service_account_iam_email]
"roles/monitoring.metricWriter" = [module.sas["apigee-metrics"].iam_email]
"roles/storage.objectAdmin" = [module.sas["apigee-cassandra"].iam_email]
"roles/apigeeconnect.Agent" = [module.sas["apigee-mart"].iam_email]
"roles/apigee.runtimeAgent" = [module.sas["apigee-watcher"].iam_email]
"roles/apigee.analyticsAgent" = [module.sas["apigee-udca"].iam_email]
"roles/apigee.synchronizerManager" = [module.sas["apigee-synchronizer"].iam_email]
"roles/cloudtrace.agent" = [module.sas["apigee-runtime"].iam_email]
}
}
}

10
blueprints/apigee/hybrid-gke/mgmt.tf
View File

@ -34,4 +34,12 @@ module "mgmt_server" {
type = var.mgmt_server_config.disk_type
size = var.mgmt_server_config.disk_size
}
}
metadata = {
startup-script = <<EOT
#!/bin/bash
apt update -y
apt install python3-pip -y
pip3 install kubernetes
EOT
}
}

20
blueprints/apigee/hybrid-gke/outputs.tf Normal file
View File

@ -0,0 +1,20 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "ip_address" {
description = "GLB IP address."
value = module.addresses.global_addresses["apigee"].address
}

7
blueprints/apigee/hybrid-gke/templates/deploy-apiproxy.sh.tpl
View File

@ -14,8 +14,13 @@
#!/bin/bash
if [ $# -lt 1 ]; then
echo "Usage: $0 ENVIRONMENT"
exit 1
fi
ORG_NAME=${org}
ENV_NAME=${env}
ENV_NAME=$1
wget https://github.com/apigee/api-platform-samples/raw/master/sample-proxies/apigee-quickstart/httpbin_rev1_2020_02_02.zip -O apiproxy.zip

271
blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json
View File

@ -1,5 +1,4 @@
{
"category": "CUSTOM",
"displayName": "quotas_utilization",
"mosaicLayout": {
"columns": 12,
@ -18,7 +17,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -40,9 +38,7 @@
}
}
},
"width": 6,
"xPos": 0,
"yPos": 0
"width": 6
},
{
"height": 4,
@ -58,7 +54,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -81,7 +76,6 @@
}
},
"width": 6,
"xPos": 0,
"yPos": 12
},
{
@ -98,7 +92,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -121,7 +114,6 @@
}
},
"width": 6,
"xPos": 0,
"yPos": 8
},
{
@ -138,7 +130,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -178,7 +169,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -201,7 +191,6 @@
}
},
"width": 6,
"xPos": 0,
"yPos": 4
},
{
@ -218,7 +207,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -241,8 +229,7 @@
}
},
"width": 6,
"xPos": 6,
"yPos": 0
"xPos": 6
},
{
"height": 4,
@ -258,7 +245,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -298,7 +284,6 @@
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
@ -330,17 +315,19 @@
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\""
"filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
@ -353,7 +340,6 @@
}
},
"width": 6,
"xPos": 0,
"yPos": 20
},
{
@ -366,21 +352,23 @@
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"alignmentPeriod": "3600s",
"crossSeriesReducer": "REDUCE_SUM",
"groupByFields": [
"metric.label.\"project\""
],
"perSeriesAligner": "ALIGN_MEAN"
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\""
"filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
@ -393,8 +381,7 @@
}
},
"width": 6,
"xPos": 0,
"yPos": 32
"yPos": 28
},
{
"height": 4,
@ -406,17 +393,19 @@
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\""
"filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
@ -430,7 +419,7 @@
},
"width": 6,
"xPos": 6,
"yPos": 28
"yPos": 24
},
{
"height": 4,
@ -442,17 +431,135 @@
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\""
"filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"yPos": 16
},
{
"height": 4,
"widget": {
"title": "Project static routes used",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
"crossSeriesReducer": "REDUCE_SUM",
"groupByFields": [
"metric.label.\"project\""
],
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/project/routes_static_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"xPos": 6,
"yPos": 20
},
{
"height": 4,
"widget": {
"title": "Peering group static routes used",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"yPos": 24
},
{
"height": 4,
"widget": {
"title": "Addresses used ratio per psa range [NEXT OLDER]",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/network/psa/addresses_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
@ -467,88 +574,6 @@
"width": 6,
"xPos": 6,
"yPos": 16
},
{
"height": 4,
"widget": {
"title": "Project static routes used",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"crossSeriesReducer": "REDUCE_SUM",
"groupByFields": [
"metric.label.\"project\""
],
"perSeriesAligner": "ALIGN_MEAN"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/project/routes_static_used_ratio\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_NONE"
}
}
}
}
],
"thresholds": [],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"xPos": 0,
"yPos": 24
},
{
"height": 4,
"widget": {
"title": "Peering group static routes used",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"apiSource": "DEFAULT_CLOUD",
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
},
"filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\""
}
}
}
],
"thresholds": [],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"xPos": 0,
"yPos": 28
}
]
}

2
blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md
View File

@ -74,7 +74,7 @@ dashboard_json_path = "../dashboards/quotas-utilization.json"
| [name](variables.tf#L75) | Name used to create Cloud Function related resources. | <code>string</code> | | <code>&#34;net-dash&#34;</code> |
| [project_create_config](variables.tf#L81) | Optional configuration if project creation is required. | <code title="object&#40;&#123;&#10; billing_account_id &#61; string&#10; parent_id &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [region](variables.tf#L95) | Compute region where the Cloud Function will be deployed. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | <code>string</code> | | <code>&#34;0&#47;30 &#42; &#42; &#42; &#42;&#34;</code> |
| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | <code>string</code> | | <code>&#34;&#42;&#47;30 &#42; &#42; &#42; &#42;&#34;</code> |
## Outputs

2
blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf
View File

@ -101,5 +101,5 @@ variable "region" {
variable "schedule_config" {
description = "Schedule timer configuration in crontab format."
type = string
default = "0/30 * * * *"
default = "*/30 * * * *"
}

4
blueprints/cloud-operations/network-dashboard/src/main.py
View File

@ -84,8 +84,8 @@ def do_discovery(resources):
{k: len(v) for k, v in resources.items() if not isinstance(v, str)}))
def do_init(resources, discovery_root, monitoring_project, folders=None, projects=None,
custom_quota=None):
def do_init(resources, discovery_root, monitoring_project, folders=None,
projects=None, custom_quota=None):
'''Calls init plugins to configure keys in the shared resource map.
Args:

58
blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py
View File

@ -39,7 +39,9 @@ TYPES = {
'subnetworks': 'compute.googleapis.com/Subnetwork',
'routers': 'compute.googleapis.com/Router',
'routes': 'compute.googleapis.com/Route',
'sql_instances': 'sqladmin.googleapis.com/Instance'
'sql_instances': 'sqladmin.googleapis.com/Instance',
'filestore_instances': 'file.googleapis.com/Instance',
'memorystore_instances': 'redis.googleapis.com/Instance',
}
NAMES = {v: k for k, v in TYPES.items()}
@ -62,8 +64,8 @@ def _handle_discovery(resources, response, data):
'Processes the asset API response and returns parsed resources or next URL.'
LOGGER.info('discovery handle request')
for result in parse_cai_results(data, 'cai-compute', method='list'):
resource = _handle_resource(
resources, result['assetType'], result['resource'])
resource = _handle_resource(resources, result['assetType'],
result['resource'])
if not resource:
continue
yield resource
@ -82,10 +84,16 @@ def _handle_resource(resources, asset_type, data):
# e.g. assetType = GlobalAddress but discoveryName = Address
resource_name = NAMES[asset_type]
resource = {
'id': attrs.get('id'),
'name': attrs['name'],
'self_link': _self_link(attrs['selfLink']),
'assetType': asset_type
'id':
attrs.get('id'),
'name':
attrs['name'],
# Some resources (ex: Filestore) don't have a self_link, using parent + name in that case
'self_link':
f'{data["parent"]}/{attrs["name"]}'
if not 'selfLink' in attrs else _self_link(attrs['selfLink']),
'assetType':
asset_type
}
# derive parent type and id and skip if parent is not within scope
parent_data = _get_parent(data['parent'], resources)
@ -212,8 +220,41 @@ def _handle_sql_instances(resource, data):
],
'region': data['region'],
'availabilityType': data['settings']['availabilityType'],
'network': data['settings']['ipConfiguration']['privateNetwork']
}
def _handle_filestore_instances(resource, data):
'Handles filestore instance type resource data.'
return {
# Getting only the instance name, removing the rest
'name': data['name'].split('/')[-1],
# Is a list but for now, only one network is supported for Filestore
'network': data['networks'][0]['network'],
'reservedIpRange': data['networks'][0]['reservedIpRange'],
'ipAddresses': data['networks'][0]['ipAddresses']
}
def _handle_memorystore_instances(resource, data):
'Handles Memorystore (Redis) instance type resource data.'
return {
# Getting only the instance name, removing the rest
'name':
data['name'].split('/')[-1],
'locationId':
data['locationId'],
'replicaCount':
0 if not 'replicaCount' in data else data['replicaCount'],
'network':
data['authorizedNetwork'],
'reservedIpRange':
'' if not 'reservedIpRange' in data else data['reservedIpRange'],
'host':
'' if not 'host' in data else data['host'],
}
def _handle_subnetworks(resource, data):
'Handles subnetwork type resource data.'
secondary_ranges = [{
@ -237,8 +278,7 @@ def _self_link(s):
def _url(resources):
'Returns discovery URL'
discovery_root = resources['config:discovery_root']
asset_types = '&'.join(
f'assetTypes={t}' for t in TYPES.values())
asset_types = '&'.join(f'assetTypes={t}' for t in TYPES.values())
return CAI_URL.format(root=discovery_root, asset_types=asset_types)

42
blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py
View File

@ -34,7 +34,28 @@ def _sql_addresses(sql_instances):
if not v['ipAddresses']:
continue
# 1 IP for the instance + 1 IP for the ILB + 1 IP if HA
yield v['ipAddresses'][0], 2 if v['availabilityType'] != 'REGIONAL' else 3
yield v['ipAddresses'][
0], 2 if v['availabilityType'] != 'REGIONAL' else 3, v['network']
def _filestore_addresses(filestore_instances):
'Returns counts of Filestore instances per PSA range.'
for v in filestore_instances.values():
if not v['ipAddresses'] or not v['reservedIpRange']:
continue
# Subnet size (reservedIpRange) can be /29, /26 or /24
yield v['ipAddresses'][0], ipaddress.ip_network(
v['reservedIpRange']).num_addresses, v['network']
def _memorystore_addresses(memorystore_instances):
'Returns counts of Memorystore (Redis) instances per PSA range.'
for v in memorystore_instances.values():
if not v['reservedIpRange'] or v['reservedIpRange'] == '':
continue
# Subnet size (reservedIpRange) can be minimum /28 or /29
yield v['host'], ipaddress.ip_network(
v['reservedIpRange']).num_addresses, v['network']
@register_timeseries
@ -46,25 +67,34 @@ def timeseries(resources):
('project', 'network', 'subnetwork'),
dtype.endswith('ratio'))
psa_nets = {
k: ipaddress.ip_network('{}/{}'.format(v['address'], v['prefixLength']))
for k, v in resources['global_addresses'].items() if v['prefixLength']
k: {
'network_link':
v['network'],
'network_prefix':
ipaddress.ip_network('{}/{}'.format(v['address'],
v['prefixLength']))
} for k, v in resources['global_addresses'].items() if v['prefixLength']
}
psa_counts = {}
for address, ip_count in _sql_addresses(resources.get('sql_instances', {})):
for address, ip_count, network in itertools.chain(
_sql_addresses(resources.get('sql_instances', {})),
_filestore_addresses(resources.get('filestore_instances', {})),
_memorystore_addresses(resources.get('memorystore_instances', {}))):
ip_address = ipaddress.ip_address(address)
for k, v in psa_nets.items():
if ip_address in v:
if network == v['network_link'] and ip_address in v['network_prefix']:
psa_counts[k] = psa_counts.get(k, 0) + ip_count
break
for k, v in psa_counts.items():
max_ips = psa_nets[k].num_addresses - 4
max_ips = psa_nets[k]['network_prefix'].num_addresses - 4
psa_range = resources['global_addresses'][k]
labels = {
'network': psa_range['network'],
'project': psa_range['project_id'],
'psa_range': psa_range['name']
}
yield TimeSeries('network/psa/addresses_available', max_ips, labels)
yield TimeSeries('network/psa/addresses_used', v, labels)
yield TimeSeries('network/psa/addresses_used_ratio',

2
blueprints/data-solutions/data-platform-foundations/README.md
View File

@ -219,7 +219,7 @@ module "data-platform" {
prefix = "myprefix"
}
# tftest modules=39 resources=286
# tftest modules=39 resources=287
```
## Customizations

19
blueprints/gke/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/install.yaml
View File

@ -23,25 +23,6 @@
set_fact:
context: "gke_{{ project_id }}_{{ region }}_{{ cluster }}"
- name: Install ASM in cluster
shell: >
gcloud container fleet mesh update \
--control-plane automatic \
--memberships {{ cluster }} \
--project {{ project_id }}
- name: Wait until MCP is provisioned
shell: >
for i in $(seq 12); do
result=$(gcloud container fleet mesh describe --project {{ project_id }} --format json \
| jq -r '.membershipStates | to_entries[] | select(.key | endswith("{{ cluster }}")) | .value.servicemesh.controlPlaneManagement.state')
if [ "$result" = "ACTIVE" ]; then
break
fi
echo "ASM control plane is not ready yet..."
sleep 60
done
- name: Get endpoint IP
shell: >
gcloud container clusters describe "{{ cluster }}" \

2
blueprints/gke/multitenant-fleet/README.md
View File

@ -224,7 +224,7 @@ module "gke" {
}
}
# tftest modules=8 resources=35
# tftest modules=8 resources=37
```
<!-- TFDOC OPTS files:1 -->

1
fast/stages/00-bootstrap/README.md
View File

@ -264,6 +264,7 @@ terraform init
terraform apply \
-var bootstrap_user=$(gcloud config list --format 'value(core.account)')
```
> If you see an error related to project name already exists, please make sure the project name is unique or the project was not deleted recently
Once the initial `apply` completes successfully, configure a remote backend using the new GCS bucket, and impersonation on the automation service account for this stage. To do this you can use the generated `providers.tf` file if you have configured output files as described above, or extract its contents from Terraform's output, then migrate state with `terraform init`:

7
fast/stages/00-bootstrap/organization.tf
View File

@ -23,9 +23,10 @@ locals {
"roles/browser" = [
"domain:${var.organization.domain}"
]
"roles/logging.admin" = [
module.automation-tf-bootstrap-sa.iam_email
]
"roles/logging.admin" = concat(
[module.automation-tf-bootstrap-sa.iam_email],
local._iam_bootstrap_user
)
"roles/owner" = local._iam_bootstrap_user
"roles/resourcemanager.folderAdmin" = [
module.automation-tf-resman-sa.iam_email

24
modules/apigee/README.md
View File

@ -25,14 +25,18 @@ module "apigee" {
}
environments = {
apis-test = {
display_name = "APIs test"
description = "APIs Test"
envgroups = ["test"]
display_name = "APIs test"
description = "APIs Test"
deployment_type = "ARCHIVE"
api_proxy_type = "PROGRAMMABLE"
envgroups = ["test"]
}
apis-prod = {
display_name = "APIs prod"
description = "APIs prod"
envgroups = ["prod"]
display_name = "APIs prod"
description = "APIs prod"
deployment_type = "PROXY"
api_proxy_type = "CONFIGURABLE"
envgroups = ["prod"]
iam = {
"roles/viewer" = ["group:devops@myorg.com"]
}
@ -169,12 +173,12 @@ module "apigee" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [project_id](variables.tf#L75) | Project ID. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L77) | Project ID. | <code>string</code> | ✓ | |
| [endpoint_attachments](variables.tf#L17) | Endpoint attachments. | <code title="map&#40;object&#40;&#123;&#10; region &#61; string&#10; service_attachment &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> |
| [envgroups](variables.tf#L26) | Environment groups (NAME => [HOSTNAMES]). | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>null</code> |
| [environments](variables.tf#L32) | Environments. | <code title="map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string&#41;&#10; description &#61; optional&#40;string, &#34;Terraform-managed&#34;&#41;&#10; node_config &#61; optional&#40;object&#40;&#123;&#10; min_node_count &#61; optional&#40;number&#41;&#10; max_node_count &#61; optional&#40;number&#41;&#10; &#125;&#41;&#41;&#10; iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10; envgroups &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> |
| [instances](variables.tf#L47) | Instances. | <code title="map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string&#41;&#10; description &#61; optional&#40;string, &#34;Terraform-managed&#34;&#41;&#10; region &#61; string&#10; environments &#61; list&#40;string&#41;&#10; psa_ip_cidr_range &#61; string&#10; disk_encryption_key &#61; optional&#40;string&#41;&#10; consumer_accept_list &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> |
| [organization](variables.tf#L61) | Apigee organization. If set to null the organization must already exist. | <code title="object&#40;&#123;&#10; display_name &#61; optional&#40;string&#41;&#10; description &#61; optional&#40;string, &#34;Terraform-managed&#34;&#41;&#10; authorized_network &#61; optional&#40;string&#41;&#10; runtime_type &#61; optional&#40;string, &#34;CLOUD&#34;&#41;&#10; billing_type &#61; optional&#40;string&#41;&#10; database_encryption_key &#61; optional&#40;string&#41;&#10; analytics_region &#61; optional&#40;string, &#34;europe-west1&#34;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [environments](variables.tf#L32) | Environments. | <code title="map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string&#41;&#10; description &#61; optional&#40;string, &#34;Terraform-managed&#34;&#41;&#10; deployment_type &#61; optional&#40;string&#41;&#10; api_proxy_type &#61; optional&#40;string&#41;&#10; node_config &#61; optional&#40;object&#40;&#123;&#10; min_node_count &#61; optional&#40;number&#41;&#10; max_node_count &#61; optional&#40;number&#41;&#10; &#125;&#41;&#41;&#10; iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10; envgroups &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> |
| [instances](variables.tf#L49) | Instances. | <code title="map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string&#41;&#10; description &#61; optional&#40;string, &#34;Terraform-managed&#34;&#41;&#10; region &#61; string&#10; environments &#61; list&#40;string&#41;&#10; psa_ip_cidr_range &#61; string&#10; disk_encryption_key &#61; optional&#40;string&#41;&#10; consumer_accept_list &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> |
| [organization](variables.tf#L63) | Apigee organization. If set to null the organization must already exist. | <code title="object&#40;&#123;&#10; display_name &#61; optional&#40;string&#41;&#10; description &#61; optional&#40;string, &#34;Terraform-managed&#34;&#41;&#10; authorized_network &#61; optional&#40;string&#41;&#10; runtime_type &#61; optional&#40;string, &#34;CLOUD&#34;&#41;&#10; billing_type &#61; optional&#40;string&#41;&#10; database_encryption_key &#61; optional&#40;string&#41;&#10; analytics_region &#61; optional&#40;string, &#34;europe-west1&#34;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
## Outputs

10
modules/apigee/main.tf
View File

@ -40,10 +40,12 @@ resource "google_apigee_envgroup" "envgroups" {
}
resource "google_apigee_environment" "environments" {
for_each = local.environments
name = each.key
display_name = each.value.display_name
description = each.value.description
for_each = local.environments
name = each.key
display_name = each.value.display_name
description = each.value.description
deployment_type = each.value.deployment_type
api_proxy_type = each.value.api_proxy_type
dynamic "node_config" {
for_each = try(each.value.node_config, null) != null ? [""] : []
content {

6
modules/apigee/variables.tf
View File

@ -32,8 +32,10 @@ variable "envgroups" {
variable "environments" {
description = "Environments."
type = map(object({
display_name = optional(string)
description = optional(string, "Terraform-managed")
display_name = optional(string)
description = optional(string, "Terraform-managed")
deployment_type = optional(string)
api_proxy_type = optional(string)
node_config = optional(object({
min_node_count = optional(number)
max_node_count = optional(number)

2
modules/gke-hub/README.md
View File

@ -295,7 +295,7 @@ module "hub" {
]
}
# tftest modules=8 resources=28
# tftest modules=8 resources=30
```
<!-- BEGIN TFDOC -->

14
modules/gke-hub/main.tf
View File

@ -70,6 +70,20 @@ resource "google_gke_hub_feature" "default" {
}
}
resource "google_gke_hub_feature_membership" "servicemesh" {
provider = google-beta
for_each = var.features.servicemesh ? var.clusters : {}
project = var.project_id
location = "global"
feature = google_gke_hub_feature.default["servicemesh"].name
membership = google_gke_hub_membership.default[each.key].membership_id
mesh {
management = "MANAGEMENT_AUTOMATIC"
control_plane = "AUTOMATIC"
}
}
resource "google_gke_hub_feature_membership" "default" {
provider = google-beta
for_each = local.cluster_cm_config

40
modules/net-glb/README.md
View File

@ -509,6 +509,46 @@ module "glb-0" {
# tftest modules=1 resources=5
```
Serverless NEGs don't use the port name but it should be set to `http`. An HTTPS frontend requires the protocol to be set to `HTTPS`, and the port name field will infer this value if omitted so you need to set it explicitly:
```hcl
module "glb-0" {
source = "./fabric/modules/net-glb"
project_id = "myprj"
name = "glb-test-0"
backend_service_configs = {
default = {
backends = [
{ backend = "neg-0" }
]
health_checks = []
port_name = "http"
}
}
# with a single serverless NEG the implied default health check is not needed
health_check_configs = {}
neg_configs = {
neg-0 = {
cloudrun = {
region = "europe-west8"
target_service = {
name = "hello"
}
}
}
}
protocol = "HTTPS"
ssl_certificates = {
managed_configs = {
default = {
domains = ["glb-test-0.example.org"]
}
}
}
}
# tftest modules=1 resources=6 inventory=https-sneg.yaml
```
### URL Map
The module exposes the full URL map resource configuration, with some minor changes to the interface to decrease verbosity, and support for aliasing backend services via keys.

3
modules/project/service-accounts.tf
View File

@ -75,7 +75,8 @@ locals {
"gkehub.googleapis.com",
"pubsub.googleapis.com",
"secretmanager.googleapis.com",
"sqladmin.googleapis.com"
"sqladmin.googleapis.com",
"cloudbuild.googleapis.com",
]
service_accounts_cmek_service_keys = distinct(flatten([
for s in keys(var.service_encryption_key_ids) : [

2
tests/blueprints/apigee/bigquery-analytics/basic.yaml
View File

@ -14,4 +14,4 @@
counts:
modules: 9
resources: 60
resources: 61

4
tests/blueprints/apigee/hybrid-gke/basic.yaml
View File

@ -13,5 +13,5 @@
# limitations under the License.
counts:
modules: 9
resources: 37
modules: 17
resources: 59

2
tests/blueprints/cloud_operations/asset_inventory_feed_remediation/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 6
assert len(resources) == 18
assert len(resources) == 19

2
tests/blueprints/cloud_operations/scheduled_asset_inventory_export_bq/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 7
assert len(resources) == 29
assert len(resources) == 30

2
tests/blueprints/cloud_operations/unmanaged_instances_healthcheck/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 10
assert len(resources) == 31
assert len(resources) == 32

2
tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
View File

@ -22,4 +22,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner(FIXTURES_DIR)
assert len(modules) == 38
assert len(resources) == 285
assert len(resources) == 286

2
tests/blueprints/gke/binauthz/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 13
assert len(resources) == 43
assert len(resources) == 44

2
tests/blueprints/gke/multi_cluster_mesh_gke_fleet_api/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 12
assert len(resources) == 53
assert len(resources) == 55

2
tests/blueprints/networking/private_cloud_function_from_onprem/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 10
assert len(resources) == 38
assert len(resources) == 39

2
tests/blueprints/serverless/api_gateway/test_plan.py
View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 7
assert len(resources) == 31
assert len(resources) == 32

2
tests/fast/stages/s00_bootstrap/simple.yaml
View File

@ -24,7 +24,7 @@ counts:
google_project_iam_binding: 9
google_project_iam_member: 1
google_project_service: 29
google_project_service_identity: 2
google_project_service_identity: 3
google_service_account: 3
google_service_account_iam_binding: 3
google_storage_bucket: 4

13
tests/modules/apigee/fixture/test.env_only_with_api_proxy_type.tfvars Normal file
View File

@ -0,0 +1,13 @@
project_id = "my-project"
environments = {
apis-test = {
display_name = "APIs test"
description = "APIs Test"
api_proxy_type = "PROGRAMMABLE"
envgroups = ["test"]
node_config = {
min_node_count = 2
max_node_count = 5
}
}
}

13
tests/modules/apigee/fixture/test.env_only_with_deployment_type.tfvars Normal file
View File

@ -0,0 +1,13 @@
project_id = "my-project"
environments = {
apis-test = {
display_name = "APIs test"
description = "APIs Test"
deployment_type = "ARCHIVE"
envgroups = ["test"]
node_config = {
min_node_count = 2
max_node_count = 5
}
}
}

6
tests/modules/apigee/fixture/variables.tf
View File

@ -32,8 +32,10 @@ variable "envgroups" {
variable "environments" {
description = "Environments."
type = map(object({
display_name = optional(string)
description = optional(string, "Terraform-managed")
display_name = optional(string)
description = optional(string, "Terraform-managed")
deployment_type = optional(string)
api_proxy_type = optional(string)
node_config = optional(object({
min_node_count = optional(number)
max_node_count = optional(number)

12
tests/modules/apigee/test_plan.py
View File

@ -54,6 +54,18 @@ def test_env_only(plan_runner):
'google_apigee_envgroup_attachment.envgroup_attachments': 1,
}
def test_env_only_with_deployment_type(plan_runner):
"Test that creates an environment in an existing environment group, with deployment_type set."
_, resources = plan_runner(tf_var_file='test.env_only_with_deployment_type.tfvars')
assert [r['values'].get('deployment_type') for r in resources
] == [None, 'ARCHIVE']
def test_env_only_with_api_proxy_type(plan_runner):
"Test that creates an environment in an existing environment group, with api_proxy_type set."
_, resources = plan_runner(tf_var_file='test.env_only_with_api_proxy_type.tfvars')
assert [r['values'].get('api_proxy_type') for r in resources
] == [None, 'PROGRAMMABLE']
def test_instance_only(plan_runner):
"Test that creates only an instance."
_, resources = plan_runner(tf_var_file='test.instance_only.tfvars')

2
tests/modules/gke_hub/fixture/variables.tf
View File

@ -31,7 +31,7 @@ variable "features" {
configmanagement = true
identityservice = false
multiclusteringress = null
servicemesh = false
servicemesh = true
multiclusterservicediscovery = false
}
}

5
tests/modules/gke_hub/test_plan.py
View File

@ -23,11 +23,14 @@ def resources(plan_runner):
def test_resource_count(resources):
"Test number of resources created."
assert len(resources) == 5
assert len(resources) == 8
assert sorted(r['address'] for r in resources) == [
'module.hub.google_gke_hub_feature.default["configmanagement"]',
'module.hub.google_gke_hub_feature.default["servicemesh"]',
'module.hub.google_gke_hub_feature_membership.default["cluster-1"]',
'module.hub.google_gke_hub_feature_membership.default["cluster-2"]',
'module.hub.google_gke_hub_feature_membership.servicemesh["cluster-1"]',
'module.hub.google_gke_hub_feature_membership.servicemesh["cluster-2"]',
'module.hub.google_gke_hub_membership.default["cluster-1"]',
'module.hub.google_gke_hub_membership.default["cluster-2"]'
]

35
tests/modules/net_glb/examples/https-sneg.yaml Normal file
View File

@ -0,0 +1,35 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
values:
module.glb-0.google_compute_backend_service.default["default"]:
port_name: http
protocol: HTTPS
module.glb-0.google_compute_global_forwarding_rule.default:
load_balancing_scheme: EXTERNAL
port_range: '443'
module.glb-0.google_compute_region_network_endpoint_group.serverless["neg-0"]:
cloud_run:
- service: hello
tag: null
url_mask: null
counts:
google_compute_backend_service: 1
google_compute_global_forwarding_rule: 1
google_compute_managed_ssl_certificate: 1
google_compute_region_network_endpoint_group: 1
google_compute_target_https_proxy: 1
google_compute_url_map: 1