zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improvements in apigee hybrid-gke: now using workload identity and GLB

This commit is contained in:
Miren Esnaola 2023-01-13 12:45:18 +01:00
parent 8c826c8c0a
commit 8945165bc3
13 changed files with 422 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
blueprints/apigee/hybrid-gke/README.md
View File

@ -25,20 +25,20 @@ The diagram below depicts the architecture.
terraform apply
```
Create an A record in your DNS registrar to point the environment group hostname to the public IP address returned after the terraform configuration was applied. You might need to wait some time until the certificate is provisioned.
## Testing the blueprint
2. Deploy an api proxy
```
./deploy-apiproxy.sh
./deploy-apiproxy.sh apis-test
```
3. In the console check the IP address that has been allocated to the Apigee ingress gateway and send some traffic to the deployed API proxy.
3. Send a request
```
curl -k -v -H "Host:HOSTNAME" \
--resolve HOSTNAME:443:IP_ADDRESS \
https://HOSTNAME/httpbin/headers
curl -v https://HOSTNAME/httpbin/headers
```
<!-- BEGIN TFDOC -->
@ -56,4 +56,10 @@ The diagram below depicts the architecture.
| [region](variables.tf#L84) | Region. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [zone](variables.tf#L90) | Zone. | <code>string</code> | | <code>&#34;europe-west1-c&#34;</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| [ip_address](outputs.tf#L17) | GLB IP address. | |
<!-- END TFDOC -->

13
blueprints/apigee/hybrid-gke/ansible.tf
View File

@ -18,12 +18,13 @@
resource "local_file" "vars_file" {
content = yamlencode({
cluster = module.cluster.name
region = var.region
project_id = module.project.project_id
envgroup = local.envgroup
env = local.environment
hostname = var.hostname
cluster = module.cluster.name
region = var.region
project_id = module.project.project_id
envgroups = local.envgroups
environments = local.environments
service_accounts = local.google_sas
ingress_ip_name = local.ingress_ip_name
})
filename = "${path.module}/ansible/vars/vars.yaml"
file_permission = "0666"

28
blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/k8s_service_accounts.yaml Normal file
View File

@ -0,0 +1,28 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Create and annotate k8s service account
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: ServiceAccount
metadata:
name: "{{ k8s_service_account }}"
namespace: apigee
annotations:
iam.gke.io/gcp-service-account: "{{ google_service_account }}@{{ project_id }}.iam.gserviceaccount.com"
with_items: "{{ k8s_service_accounts }}"
loop_control:
loop_var: k8s_service_account

284
blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml
View File

@ -1,11 +1,11 @@
# Copyright 2023 Google LLC
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -19,18 +19,27 @@
--project {{ project_id }} \
--internal-ip
- name: Install cert-manager
shell: >
kubectl apply \
--validate=false \
-f https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml
- name: Download cert-manager
uri:
url: https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml
dest: ~/cert-manager.yaml
- name: Wait until pods are ready in cert-manager namespace
shell: >
kubectl wait --for=condition=ready pods \
-l app.kubernetes.io/instance=cert-manager \
-n cert-manager \
--timeout=90s
- name: Apply metrics-server manifest to the cluster.
kubernetes.core.k8s:
state: present
src: ~/cert-manager.yaml
- name:
kubernetes.core.k8s_info:
kind: Pod
wait: yes
label_selectors:
- "app.kubernetes.io/instance=cert-manager"
namespace: cert-manager
wait_timeout: 90
wait_condition:
type: Ready
status: True
- name: Fetch apigeectl version
uri:
@ -48,7 +57,7 @@
unarchive:
src: "~/apigeectl.tar.gz"
dest: "~"
remote_src: yes
remote_src: yes
- name: Move apigeectl folder
shell: >
@ -66,25 +75,69 @@
file:
src: ~/apigeectl/{{ item }}
dest: "~/hybrid-files/{{ item }}"
state: link
state: link
with_items:
- tools
- config
- templates
- plugins
- plugins
- name: Create service accounts
shell: >
~/hybrid-files/tools/create-service-account -i {{ project_id }} -e non-prod -d ~/hybrid-files/service-accounts
- name: Create apigee namespace
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Namespace
metadata:
name: apigee
- name: Create certificates
- name: Create k8s service accounts
include_tasks: k8s_service_accounts.yaml
vars:
google_service_account: "{{ item.key }}"
k8s_service_accounts: "{{ item.value }}"
with_dict: "{{ service_accounts }}"
- name: Set hostnames
set_fact:
hostnames: "{{ hostnames | default([]) + item.value }}"
with_dict: "{{ envgroups }}"
- name: Create certificate and private key
shell: >
openssl req \
-nodes \
-new \
-x509 \
-keyout ~/hybrid-files/certs/{{ envgroup }}.key \
-out ~/hybrid-files/certs/{{ envgroup }}.cert -subj '/CN='{{ hostname }}'' -days 3650
-keyout ~/hybrid-files/certs/server.key \
-out ~/hybrid-files/certs/server.crt \
-subj "/CN=apigee.com' \
-addext "subjectAltName={{ hostnames | map('regex_replace', '^', 'DNS:') | join(',') }}""
-days 3650
- name: Read certificate
slurp:
src: ~/hybrid-files/certs/server.crt
register: certificate_output
- name: Read private ket
slurp:
src: ~/hybrid-files/certs/server.key
register: privatekey_output
- name: Create secret
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Secret
metadata:
name: tls-hybrid-ingress
namespace: apigee
type: kubernetes.io/tls
data:
tls.crt: "{{ certificate_output.content }}"
tls.key: "{{ privatekey_output.content }}"
- name: Create overrides.yaml
template:
@ -96,48 +149,185 @@
curl -X POST -H "Authorization: Bearer $(gcloud auth print-access-token)" \
-H "Content-Type:application/json" \
"https://apigee.googleapis.com/v1/organizations/{{ project_id }}:setSyncAuthorization" \
-d '{"identities":["'"serviceAccount:apigee-non-prod@{{ project_id }}.iam.gserviceaccount.com"'"]}'
-d '{"identities":["'"serviceAccount:apigee-synchronizer@{{ project_id }}.iam.gserviceaccount.com"'"]}'
- name: Dry-run (init)
shell: >
~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client
~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client
args:
chdir: ~/hybrid-files
- name: Install the Apigee deployment services Apigee Deployment Controller and Apigee Admission Webhook.
shell: >
~/apigeectl/apigeectl init -f overrides/overrides.yaml
~/apigeectl/apigeectl init -f overrides/overrides.yaml
args:
chdir: ~/hybrid-files
chdir: ~/hybrid-files
- name: Wait until pods are ready in apigee-system namespace
shell: >
kubectl wait --for=condition=ready pods \
-l app=apigee-controller \
-n apigee-system \
--timeout=300s
- name: Wait for apigee-controller pod to be ready
kubernetes.core.k8s_info:
kind: Pod
wait: yes
label_selectors:
- "app=apigee-controller"
namespace: apigee-system
wait_timeout: 600
wait_condition:
type: Ready
status: True
- name: Wait until pods are ready in apigee namespace
shell: >
kubectl wait --for=condition=ready pods \
-l app=apigee-ingressgateway-manager \
-n apigee \
--timeout=300s
- name: Wait for apigee-selfsigned-issuer issuer to be ready
kubernetes.core.k8s_info:
kind: Issuer
wait: yes
name: apigee-selfsigned-issuer
namespace: apigee-system
wait_timeout: 600
wait_condition:
type: Ready
status: True
- name: Wait for apigee-serving-cert certificate to be ready
kubernetes.core.k8s_info:
kind: Certificate
wait: yes
name: apigee-serving-cert
namespace: apigee-system
wait_timeout: 600
wait_condition:
type: Ready
status: True
- name: Wait for apigee-resources-install job to be complete
kubernetes.core.k8s_info:
kind: Job
wait: yes
name: apigee-resources-install
namespace: apigee-system
wait_timeout: 360
wait_condition:
type: Complete
status: True
- name: Dry-run (apply)
shell: >
~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client
~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client
args:
chdir: ~/hybrid-files
- name: Install the Apigee runtime components
shell: >
~/apigeectl/apigeectl apply -f overrides/overrides.yaml
~/apigeectl/apigeectl apply -f overrides/overrides.yaml
args:
chdir: ~/hybrid-files
chdir: ~/hybrid-files
- name: Check status of the deployment
shell: >
while [ -n "$(kubectl get pods -n apigee | tail -n +2 | grep -v Running | grep -v Completed)" ]; do sleep 1; done
args:
chdir: ~/hybrid-files
- name: Wait for apigee-runtime pod to be ready
kubernetes.core.k8s_info:
kind: Pod
wait: yes
label_selectors:
- "app=apigee-runtime"
namespace: apigee
wait_timeout: 360
wait_condition:
type: Ready
status: True
- name:
kubernetes.core.k8s:
state: present
definition:
apiVersion: apigee.cloud.google.com/v1alpha1
kind: ApigeeRoute
metadata:
name: apigee-wildcard
namespace: apigee
spec:
hostnames:
- '*'
ports:
- number: 443
protocol: HTTPS
tls:
credentialName: tls-hybrid-ingress
mode: SIMPLE
selector:
app: apigee-ingressgateway
enableNonSniClient: true
- name: Create google-managed certificate
kubernetes.core.k8s:
state: present
definition:
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
name: "apigee-cert-hybrid"
namespace: apigee
spec:
domains: "{{ hostnames }}"
- name: Create backend config
kubernetes.core.k8s:
state: present
definition:
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
name: apigee-ingress-backendconfig
namespace: apigee
spec:
healthCheck:
requestPath: /healthz/ready
port: 15021
type: HTTP
logging:
enable: true
sampleRate: 0.5
- name: Create service
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Service
metadata:
name: apigee-ingressgateway-hybrid
namespace: apigee
annotations:
cloud.google.com/backend-config: '{"default": "apigee-ingress-backendconfig"}'
cloud.google.com/neg: '{"ingress": true}'
cloud.google.com/app-protocols: '{"https":"HTTPS", "status-port": "HTTP"}'
labels:
app: apigee-ingressgateway-hybrid
spec:
ports:
- name: status-port
port: 15021
targetPort: 15021
- name: https
port: 443
targetPort: 8443
selector:
app: apigee-ingressgateway
ingress_name: ingress
type: ClusterIP
- name: Create ingress
kubernetes.core.k8s:
state: present
definition:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
networking.gke.io/managed-certificates: "apigee-cert-hybrid"
kubernetes.io/ingress.global-static-ip-name: "{{ ingress_ip_name }}"
kubernetes.io/ingress.allow-http: "false"
name: xlb-apigee
namespace: apigee
spec:
defaultBackend:
service:
name: apigee-ingressgateway-hybrid
port:
number: 443

45
blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2
View File

@ -1,29 +1,26 @@
gcp:
region: {{ region }}
projectID: {{ project_id }}
workloadIdentityEnabled: true
k8sCluster:
name: {{ cluster }}
region: CLUSTER_LOCATION # Must be the closest Google Cloud region to your cluster.
region: {{ region }} # Must be the closest Google Cloud region to your cluster.
org: {{ project_id }}
instanceID: "instance-1"
instanceID: "{{ cluster }}-{{ region }}"
cassandra:
hostNetwork: false
# Set to false for single region installations and multi-region installations
# with connectivity between pods in different clusters, for example GKE installations.
# Set to true for multi-region installations with no communication between
# pods in different clusters, for example GKE On-prem, GKE on AWS, Anthos on bare metal,
# AKS, EKS, and OpenShift installations.
# See Multi-region deployment: Prerequisites
virtualhosts:
- name: {{ envgroup }}
{% for k in envgroups %}
- name: {{ k }}
sslSecret: tls-hybrid-ingress
additionalGateways: ["apigee-wildcard"]
selector:
app: apigee-ingressgateway
sslCertPath: ./certs/{{ envgroup }}.cert
sslKeyPath: ./certs/{{ envgroup }}.key
{% endfor %}
ao:
args:
@ -37,27 +34,9 @@ ingressGateways:
replicaCountMax: 10
envs:
- name: {{ env }}
serviceAccountPaths:
synchronizer: ./service-accounts/{{ project_id }}-apigee-non-prod.json
udca: ./service-accounts/{{ project_id }}-apigee-non-prod.json
runtime: ./service-accounts/{{ project_id }}-apigee-non-prod.json
mart:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
connectAgent:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
metrics:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
udca:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
watcher:
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
{% for k in environments %}
- name: {{ k }}
{% endfor %}
logger:
enabled: true
serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
enabled: false

67
blueprints/apigee/hybrid-gke/apigee.tf
View File

@ -15,8 +15,51 @@
*/
locals {
envgroup = "test"
environment = "apis-test"
envgroups = {
test = [var.hostname]
}
environments = {
apis-test = {
envgroups = ["test"]
}
}
org_short_name = (length(module.project.project_id) < 16 ?
module.project.project_id :
substr(module.project.project_id, 0, 15))
org_hash = format("%s-%s", local.org_short_name, substr(sha256(module.project.project_id), 0, 7))
org_env_hashes = {
for k, v in local.environments :
k => format("%s-%s-%s", local.org_short_name, length(k) < 16 ? k : substr(k, 0, 15), substr(sha256("${module.project.project_id}:${k}"), 0, 7))
}
google_sas = {
apigee-metrics = [
"apigee-metrics-sa"
]
apigee-cassandra = [
"apigee-cassandra-schema-setup-${local.org_hash}-sa",
"apigee-cassandra-user-setup-${local.org_hash}-sa"
]
apigee-mart = [
"apigee-mart-${local.org_hash}-sa",
"apigee-connect-agent-${local.org_hash}-sa"
]
apigee-watcher = [
"apigee-watcher-${local.org_hash}-sa"
]
apigee-udca = concat([
"apigee-udca-${local.org_hash}-sa"
],
[for k, v in local.org_env_hashes :
"apigee-udca-${local.org_env_hashes[k]}-sa"
])
apigee-synchronizer = [
for k, v in local.org_env_hashes :
"apigee-synchronizer-${local.org_env_hashes[k]}-sa"
]
apigee-runtime = [for k, v in local.org_env_hashes :
"apigee-runtime-${local.org_env_hashes[k]}-sa"
]
}
}
module "apigee" {
@ -26,20 +69,24 @@ module "apigee" {
analytics_region = var.region
runtime_type = "HYBRID"
}
envgroups = {
(local.envgroup) = [var.hostname]
}
environments = {
(local.environment) = {
envgroups = [local.envgroup]
}
envgroups = local.envgroups
environments = local.environments
}
module "sas" {
for_each = local.google_sas
source = "../../../modules/iam-service-account"
project_id = module.project.project_id
name = each.key
# authoritative roles granted *on* the service accounts to other identities
iam = {
"roles/iam.workloadIdentityUser" = [for v in each.value : "serviceAccount:${module.project.project_id}.svc.id.goog[apigee/${v}]"]
}
}
resource "local_file" "deploy_apiproxy_file" {
content = templatefile("${path.module}/templates/deploy-apiproxy.sh.tpl", {
org = module.project.project_id
env = local.environment
})
filename = "${path.module}/deploy-apiproxy.sh"
file_permission = "0777"

BIN
blueprints/apigee/hybrid-gke/diagram.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 35 KiB

After

Width:  |  Height:  |  Size: 35 KiB

25
blueprints/apigee/hybrid-gke/glb.tf Normal file
View File

@ -0,0 +1,25 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
ingress_ip_name = "apigee"
}
module "addresses" {
source = "../../../modules/net-address"
project_id = module.project.project_id
global_addresses = [local.ingress_ip_name]
}

9
blueprints/apigee/hybrid-gke/main.tf
View File

@ -40,5 +40,12 @@ module "project" {
"roles/resourcemanager.projectIamAdmin" = [module.mgmt_server.service_account_iam_email]
"roles/iam.serviceAccountAdmin" = [module.mgmt_server.service_account_iam_email]
"roles/iam.serviceAccountKeyAdmin" = [module.mgmt_server.service_account_iam_email]
"roles/monitoring.metricWriter" = [module.sas["apigee-metrics"].iam_email]
"roles/storage.objectAdmin" = [module.sas["apigee-cassandra"].iam_email]
"roles/apigeeconnect.Agent" = [module.sas["apigee-mart"].iam_email]
"roles/apigee.runtimeAgent" = [module.sas["apigee-watcher"].iam_email]
"roles/apigee.analyticsAgent" = [module.sas["apigee-udca"].iam_email]
"roles/apigee.synchronizerManager" = [module.sas["apigee-synchronizer"].iam_email]
"roles/cloudtrace.agent" = [module.sas["apigee-runtime"].iam_email]
}
}
}

10
blueprints/apigee/hybrid-gke/mgmt.tf
View File

@ -34,4 +34,12 @@ module "mgmt_server" {
type = var.mgmt_server_config.disk_type
size = var.mgmt_server_config.disk_size
}
}
metadata = {
startup-script = <<EOT
#!/bin/bash
apt update -y
apt install python3-pip -y
pip3 install kubernetes
EOT
}
}

20
blueprints/apigee/hybrid-gke/outputs.tf Normal file
View File

@ -0,0 +1,20 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "ip_address" {
description = "GLB IP address."
value = module.addresses.global_addresses["apigee"].address
}

7
blueprints/apigee/hybrid-gke/templates/deploy-apiproxy.sh.tpl
View File

@ -14,8 +14,13 @@
#!/bin/bash
if [ $# -lt 1 ]; then
echo "Usage: $0 ENVIRONMENT"
exit 1
fi
ORG_NAME=${org}
ENV_NAME=${env}
ENV_NAME=$1
wget https://github.com/apigee/api-platform-samples/raw/master/sample-proxies/apigee-quickstart/httpbin_rev1_2020_02_02.zip -O apiproxy.zip

4
tests/blueprints/apigee/hybrid-gke/basic.yaml
View File

@ -13,5 +13,5 @@
# limitations under the License.
counts:
modules: 9
resources: 37
modules: 17
resources: 59