Improvements in apigee hybrid-gke: now using workload identity and GLB

blueprints/apigee/hybrid-gke/README.md

@@ -25,20 +25,20 @@ The diagram below depicts the architecture.
     terraform apply
     ```
 
+Create an A record in your DNS registrar to point the environment group hostname to the public IP address returned after the terraform configuration was applied. You might need to wait some time until the certificate is provisioned.    
+
 ## Testing the blueprint
 
 2. Deploy an api proxy
 
     ```
-    ./deploy-apiproxy.sh
+    ./deploy-apiproxy.sh apis-test
     ```
 
-3. In the console check the IP address that has been allocated to the Apigee ingress gateway and send some traffic to the deployed API proxy.
+3. Send a request
 
     ```
-    curl -k -v -H "Host:HOSTNAME" \
+    curl -v https://HOSTNAME/httpbin/headers
-    --resolve HOSTNAME:443:IP_ADDRESS \
-    https://HOSTNAME/httpbin/headers
     ```
 <!-- BEGIN TFDOC -->
 
@@ -56,4 +56,10 @@ The diagram below depicts the architecture.
 | [region](variables.tf#L84) | Region. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
 | [zone](variables.tf#L90) | Zone. | <code>string</code> |  | <code>&#34;europe-west1-c&#34;</code> |
 
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [ip_address](outputs.tf#L17) | GLB IP address. |  |
+
 <!-- END TFDOC -->

blueprints/apigee/hybrid-gke/ansible.tf

@@ -18,12 +18,13 @@
 
 resource "local_file" "vars_file" {
   content = yamlencode({
-    cluster    = module.cluster.name
+    cluster          = module.cluster.name
-    region     = var.region
+    region           = var.region
-    project_id = module.project.project_id
+    project_id       = module.project.project_id
-    envgroup   = local.envgroup
+    envgroups        = local.envgroups
-    env        = local.environment
+    environments     = local.environments
-    hostname   = var.hostname
+    service_accounts = local.google_sas
+    ingress_ip_name  = local.ingress_ip_name
   })
   filename        = "${path.module}/ansible/vars/vars.yaml"
   file_permission = "0666"

blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/k8s_service_accounts.yaml

@@ -0,0 +1,28 @@
+# Copyright 2023 Google LLC
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create and annotate k8s service account
+  kubernetes.core.k8s:
+    state: present
+    definition: 
+      apiVersion: v1
+      kind: ServiceAccount
+      metadata:
+        name: "{{ k8s_service_account }}"
+        namespace: apigee
+        annotations:
+          iam.gke.io/gcp-service-account: "{{ google_service_account }}@{{ project_id }}.iam.gserviceaccount.com"
+  with_items: "{{ k8s_service_accounts }}"  
+  loop_control:
+    loop_var: k8s_service_account

blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml

@@ -1,11 +1,11 @@
 # Copyright 2023 Google LLC
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,18 +19,27 @@
     --project {{ project_id }} \
     --internal-ip
 
-- name: Install cert-manager
+- name: Download cert-manager
-  shell: >
+  uri:
-    kubectl apply \
+    url: https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml
-    --validate=false \
+    dest: ~/cert-manager.yaml
-    -f https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml
 
-- name: Wait until pods are ready in cert-manager namespace
+- name: Apply metrics-server manifest to the cluster.
-  shell: >
+  kubernetes.core.k8s:
-    kubectl wait --for=condition=ready pods \
+    state: present
-    -l app.kubernetes.io/instance=cert-manager \
+    src: ~/cert-manager.yaml
-    -n cert-manager \ 
+
-    --timeout=90s
+- name:
+  kubernetes.core.k8s_info:
+    kind: Pod
+    wait: yes
+    label_selectors:
+      - "app.kubernetes.io/instance=cert-manager"
+    namespace: cert-manager
+    wait_timeout: 90
+    wait_condition:
+      type: Ready
+      status: True
 
 - name: Fetch apigeectl version
   uri:
@@ -48,7 +57,7 @@
   unarchive:
     src: "~/apigeectl.tar.gz"
     dest: "~"
-    remote_src: yes    
+    remote_src: yes
 
 - name: Move apigeectl folder
   shell: >
@@ -66,25 +75,69 @@
   file:
     src: ~/apigeectl/{{ item }}
     dest: "~/hybrid-files/{{ item }}"
-    state: link    
+    state: link
   with_items:
     - tools
     - config
     - templates
-    - plugins    
+    - plugins
 
-- name: Create service accounts
+- name: Create apigee namespace
-  shell: >
+  kubernetes.core.k8s:
-    ~/hybrid-files/tools/create-service-account -i {{ project_id }} -e non-prod -d ~/hybrid-files/service-accounts
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: apigee
 
-- name: Create certificates
+- name: Create k8s service accounts
+  include_tasks: k8s_service_accounts.yaml
+  vars:
+    google_service_account: "{{ item.key }}"
+    k8s_service_accounts: "{{ item.value }}"
+  with_dict: "{{ service_accounts }}"
+
+- name: Set hostnames
+  set_fact:
+    hostnames: "{{ hostnames | default([]) + item.value }}"
+  with_dict: "{{ envgroups }}"
+
+- name: Create certificate and private key
   shell: >
     openssl req  \ 
     -nodes \
     -new \
     -x509 \
-    -keyout ~/hybrid-files/certs/{{ envgroup }}.key \
+    -keyout ~/hybrid-files/certs/server.key \
-    -out ~/hybrid-files/certs/{{ envgroup }}.cert -subj '/CN='{{ hostname }}'' -days 3650
+    -out ~/hybrid-files/certs/server.crt \
+    -subj "/CN=apigee.com' \ 
+    -addext "subjectAltName={{ hostnames | map('regex_replace', '^', 'DNS:') | join(',') }}""
+    -days 3650
+
+- name: Read certificate 
+  slurp: 
+    src: ~/hybrid-files/certs/server.crt
+  register: certificate_output
+
+- name: Read private ket 
+  slurp: 
+    src: ~/hybrid-files/certs/server.key
+  register: privatekey_output
+
+- name: Create secret
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: tls-hybrid-ingress
+        namespace: apigee
+      type: kubernetes.io/tls
+      data:
+        tls.crt: "{{ certificate_output.content }}"
+        tls.key: "{{ privatekey_output.content }}"
 
 - name: Create overrides.yaml
   template:
@@ -96,48 +149,185 @@
     curl -X POST -H "Authorization: Bearer $(gcloud auth print-access-token)" \
     -H "Content-Type:application/json" \
     "https://apigee.googleapis.com/v1/organizations/{{ project_id }}:setSyncAuthorization" \
-    -d '{"identities":["'"serviceAccount:apigee-non-prod@{{ project_id }}.iam.gserviceaccount.com"'"]}'
+    -d '{"identities":["'"serviceAccount:apigee-synchronizer@{{ project_id }}.iam.gserviceaccount.com"'"]}'
 
 - name: Dry-run (init)
   shell: >
-    ~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client  
+    ~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client
   args:
     chdir: ~/hybrid-files
 
 - name: Install the Apigee deployment services Apigee Deployment Controller and Apigee Admission Webhook.
   shell: >
-    ~/apigeectl/apigeectl init -f overrides/overrides.yaml 
+    ~/apigeectl/apigeectl init -f overrides/overrides.yaml
   args:
-    chdir: ~/hybrid-files    
+    chdir: ~/hybrid-files
 
-- name: Wait until pods are ready in apigee-system namespace
+- name: Wait for apigee-controller pod to be ready
-  shell: >
+  kubernetes.core.k8s_info:
-    kubectl wait --for=condition=ready pods \
+    kind: Pod
-    -l app=apigee-controller \
+    wait: yes
-    -n apigee-system \ 
+    label_selectors:
-    --timeout=300s    
+      - "app=apigee-controller"
+    namespace: apigee-system
+    wait_timeout: 600
+    wait_condition:
+      type: Ready
+      status: True
 
-- name: Wait until pods are ready in apigee namespace
+- name: Wait for apigee-selfsigned-issuer issuer to be ready
-  shell: >
+  kubernetes.core.k8s_info:
-    kubectl wait --for=condition=ready pods \
+    kind: Issuer
-    -l app=apigee-ingressgateway-manager \
+    wait: yes
-    -n apigee \ 
+    name: apigee-selfsigned-issuer
-    --timeout=300s    
+    namespace: apigee-system
+    wait_timeout: 600
+    wait_condition:
+      type: Ready
+      status: True
+
+- name: Wait for apigee-serving-cert certificate to be ready
+  kubernetes.core.k8s_info:
+    kind: Certificate
+    wait: yes
+    name: apigee-serving-cert
+    namespace: apigee-system
+    wait_timeout: 600
+    wait_condition:
+      type: Ready
+      status: True
+
+- name: Wait for apigee-resources-install job to be complete
+  kubernetes.core.k8s_info:
+    kind: Job
+    wait: yes
+    name: apigee-resources-install
+    namespace: apigee-system
+    wait_timeout: 360
+    wait_condition:
+      type: Complete
+      status: True
 
 - name: Dry-run (apply)
   shell: >
-    ~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client  
+    ~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client
   args:
     chdir: ~/hybrid-files
 
 - name: Install the Apigee runtime components
   shell: >
-    ~/apigeectl/apigeectl apply -f overrides/overrides.yaml 
+    ~/apigeectl/apigeectl apply -f overrides/overrides.yaml
   args:
-    chdir: ~/hybrid-files        
+    chdir: ~/hybrid-files
 
-- name: Check status of the deployment
+- name: Wait for apigee-runtime pod to be ready
-  shell: >
+  kubernetes.core.k8s_info:
-    while [ -n "$(kubectl get pods -n apigee | tail -n +2 | grep -v Running | grep -v Completed)" ]; do sleep 1; done
+    kind: Pod
-  args:
+    wait: yes
-    chdir: ~/hybrid-files
+    label_selectors:
+      - "app=apigee-runtime"
+    namespace: apigee
+    wait_timeout: 360
+    wait_condition:
+      type: Ready
+      status: True
+
+- name: 
+  kubernetes.core.k8s:
+    state: present   
+    definition: 
+      apiVersion: apigee.cloud.google.com/v1alpha1
+      kind: ApigeeRoute
+      metadata:
+        name: apigee-wildcard
+        namespace: apigee
+      spec:
+        hostnames:
+        - '*'
+        ports:
+        - number: 443
+          protocol: HTTPS
+          tls:
+            credentialName: tls-hybrid-ingress
+            mode: SIMPLE
+        selector:
+          app: apigee-ingressgateway
+        enableNonSniClient: true 
+
+- name: Create google-managed certificate
+  kubernetes.core.k8s:
+    state: present   
+    definition: 
+      apiVersion: networking.gke.io/v1
+      kind: ManagedCertificate
+      metadata:
+        name: "apigee-cert-hybrid"
+        namespace: apigee
+      spec:
+        domains: "{{ hostnames }}"
+      
+- name: Create backend config
+  kubernetes.core.k8s:
+    state: present  
+    definition:  
+      apiVersion: cloud.google.com/v1
+      kind: BackendConfig
+      metadata:
+        name: apigee-ingress-backendconfig
+        namespace: apigee
+      spec:
+        healthCheck:
+          requestPath: /healthz/ready
+          port: 15021
+          type: HTTP
+        logging:
+          enable: true
+          sampleRate: 0.5
+
+- name: Create service
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: apigee-ingressgateway-hybrid
+        namespace: apigee
+        annotations:
+          cloud.google.com/backend-config: '{"default": "apigee-ingress-backendconfig"}'
+          cloud.google.com/neg: '{"ingress": true}'
+          cloud.google.com/app-protocols: '{"https":"HTTPS", "status-port": "HTTP"}'
+        labels:
+          app: apigee-ingressgateway-hybrid
+      spec:
+        ports:
+        - name: status-port
+          port: 15021
+          targetPort: 15021
+        - name: https
+          port: 443
+          targetPort: 8443
+        selector:
+          app: apigee-ingressgateway
+          ingress_name: ingress
+        type: ClusterIP
+
+- name: Create ingress
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: networking.k8s.io/v1
+      kind: Ingress
+      metadata:
+        annotations:
+          networking.gke.io/managed-certificates: "apigee-cert-hybrid"
+          kubernetes.io/ingress.global-static-ip-name: "{{ ingress_ip_name }}"
+          kubernetes.io/ingress.allow-http: "false"
+        name: xlb-apigee
+        namespace: apigee
+      spec:
+        defaultBackend:
+          service:
+            name: apigee-ingressgateway-hybrid
+            port:
+              number: 443    

blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2

@@ -1,29 +1,26 @@
 gcp:
   region: {{ region }}
   projectID: {{ project_id }}
+  workloadIdentityEnabled: true
 
 k8sCluster:
   name: {{ cluster }}
-  region: CLUSTER_LOCATION # Must be the closest Google Cloud region to your cluster.
+  region: {{ region }} # Must be the closest Google Cloud region to your cluster.
 org: {{ project_id }}
 
-instanceID: "instance-1"
+instanceID: "{{ cluster }}-{{ region }}"
 
 cassandra:
   hostNetwork: false
-    # Set to false for single region installations and multi-region installations
-    # with connectivity between pods in different clusters, for example GKE installations.
-    # Set to true  for multi-region installations with no communication between
-    # pods in different clusters, for example GKE On-prem, GKE on AWS, Anthos on bare metal,
-    # AKS, EKS, and OpenShift installations.
-    # See Multi-region deployment: Prerequisites
 
 virtualhosts:
-  - name: {{ envgroup }}
+{% for k in envgroups %}
+  - name: {{ k }}
+    sslSecret: tls-hybrid-ingress
+    additionalGateways: ["apigee-wildcard"]
     selector:
       app: apigee-ingressgateway
-    sslCertPath: ./certs/{{ envgroup }}.cert
+{% endfor %}
-    sslKeyPath: ./certs/{{ envgroup }}.key
 
 ao:
   args:
@@ -37,27 +34,9 @@ ingressGateways:
   replicaCountMax: 10
 
 envs:
-  - name: {{ env }}
+{% for k in environments %}
-    serviceAccountPaths:
+   - name: {{ k }}
-      synchronizer: ./service-accounts/{{ project_id }}-apigee-non-prod.json
+{% endfor %}
-      udca: ./service-accounts/{{ project_id }}-apigee-non-prod.json
-      runtime: ./service-accounts/{{ project_id }}-apigee-non-prod.json
-
-mart:
-  serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
-
-connectAgent:
-  serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
-
-metrics:
-  serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
-
-udca:
-  serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
-
-watcher:
-  serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json
 
 logger:
-  enabled: true
+  enabled: false
-  serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json

blueprints/apigee/hybrid-gke/apigee.tf

@@ -15,8 +15,51 @@
  */
 
 locals {
-  envgroup    = "test"
+  envgroups = {
-  environment = "apis-test"
+    test = [var.hostname]
+  }
+  environments = {
+    apis-test = {
+      envgroups = ["test"]
+    }
+  }
+  org_short_name = (length(module.project.project_id) < 16 ?
+    module.project.project_id :
+  substr(module.project.project_id, 0, 15))
+  org_hash = format("%s-%s", local.org_short_name, substr(sha256(module.project.project_id), 0, 7))
+  org_env_hashes = {
+    for k, v in local.environments :
+    k => format("%s-%s-%s", local.org_short_name, length(k) < 16 ? k : substr(k, 0, 15), substr(sha256("${module.project.project_id}:${k}"), 0, 7))
+  }
+  google_sas = {
+    apigee-metrics = [
+      "apigee-metrics-sa"
+    ]
+    apigee-cassandra = [
+      "apigee-cassandra-schema-setup-${local.org_hash}-sa",
+      "apigee-cassandra-user-setup-${local.org_hash}-sa"
+    ]
+    apigee-mart = [
+      "apigee-mart-${local.org_hash}-sa",
+      "apigee-connect-agent-${local.org_hash}-sa"
+    ]
+    apigee-watcher = [
+      "apigee-watcher-${local.org_hash}-sa"
+    ]
+    apigee-udca = concat([
+      "apigee-udca-${local.org_hash}-sa"
+      ],
+      [for k, v in local.org_env_hashes :
+        "apigee-udca-${local.org_env_hashes[k]}-sa"
+    ])
+    apigee-synchronizer = [
+      for k, v in local.org_env_hashes :
+      "apigee-synchronizer-${local.org_env_hashes[k]}-sa"
+    ]
+    apigee-runtime = [for k, v in local.org_env_hashes :
+      "apigee-runtime-${local.org_env_hashes[k]}-sa"
+    ]
+  }
 }
 
 module "apigee" {
@@ -26,20 +69,24 @@ module "apigee" {
     analytics_region = var.region
     runtime_type     = "HYBRID"
   }
-  envgroups = {
+  envgroups    = local.envgroups
-    (local.envgroup) = [var.hostname]
+  environments = local.environments
-  }
+}
-  environments = {
+
-    (local.environment) = {
+module "sas" {
-      envgroups = [local.envgroup]
+  for_each   = local.google_sas
-    }
+  source     = "../../../modules/iam-service-account"
+  project_id = module.project.project_id
+  name       = each.key
+  # authoritative roles granted *on* the service accounts to other identities
+  iam = {
+    "roles/iam.workloadIdentityUser" = [for v in each.value : "serviceAccount:${module.project.project_id}.svc.id.goog[apigee/${v}]"]
   }
 }
 
 resource "local_file" "deploy_apiproxy_file" {
   content = templatefile("${path.module}/templates/deploy-apiproxy.sh.tpl", {
     org = module.project.project_id
-    env = local.environment
   })
   filename        = "${path.module}/deploy-apiproxy.sh"
   file_permission = "0777"

blueprints/apigee/hybrid-gke/glb.tf

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  ingress_ip_name = "apigee"
+}
+
+module "addresses" {
+  source           = "../../../modules/net-address"
+  project_id       = module.project.project_id
+  global_addresses = [local.ingress_ip_name]
+}

blueprints/apigee/hybrid-gke/main.tf

@@ -40,5 +40,12 @@ module "project" {
     "roles/resourcemanager.projectIamAdmin" = [module.mgmt_server.service_account_iam_email]
     "roles/iam.serviceAccountAdmin"         = [module.mgmt_server.service_account_iam_email]
     "roles/iam.serviceAccountKeyAdmin"      = [module.mgmt_server.service_account_iam_email]
+    "roles/monitoring.metricWriter"         = [module.sas["apigee-metrics"].iam_email]
+    "roles/storage.objectAdmin"             = [module.sas["apigee-cassandra"].iam_email]
+    "roles/apigeeconnect.Agent"             = [module.sas["apigee-mart"].iam_email]
+    "roles/apigee.runtimeAgent"             = [module.sas["apigee-watcher"].iam_email]
+    "roles/apigee.analyticsAgent"           = [module.sas["apigee-udca"].iam_email]
+    "roles/apigee.synchronizerManager"      = [module.sas["apigee-synchronizer"].iam_email]
+    "roles/cloudtrace.agent"                = [module.sas["apigee-runtime"].iam_email]
   }
-}
+}

blueprints/apigee/hybrid-gke/mgmt.tf

@@ -34,4 +34,12 @@ module "mgmt_server" {
     type  = var.mgmt_server_config.disk_type
     size  = var.mgmt_server_config.disk_size
   }
-}
+  metadata = {
+    startup-script = <<EOT
+#!/bin/bash
+apt update -y
+apt install python3-pip -y
+pip3 install kubernetes
+EOT
+  }
+}

blueprints/apigee/hybrid-gke/outputs.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "ip_address" {
+  description = "GLB IP address."
+  value       = module.addresses.global_addresses["apigee"].address
+}

blueprints/apigee/hybrid-gke/templates/deploy-apiproxy.sh.tpl

@@ -14,8 +14,13 @@
  
 #!/bin/bash
 
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 ENVIRONMENT"
+    exit 1
+fi  
+
 ORG_NAME=${org}
-ENV_NAME=${env}
+ENV_NAME=$1
 
 wget https://github.com/apigee/api-platform-samples/raw/master/sample-proxies/apigee-quickstart/httpbin_rev1_2020_02_02.zip -O apiproxy.zip 
 

tests/blueprints/apigee/hybrid-gke/basic.yaml

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 counts:
-  modules: 9
+  modules: 17
-  resources: 37
+  resources: 59