Envoy with Traffic Director cloud-config (#70)

modules/cloud-config-container/cos-generic-metadata/README.md

@@ -0,0 +1,87 @@
+# Generic cloud-init generator for Container Optimized OS
+
+This helper module manages a `cloud-config` configuration that can start a container on [Container Optimized OS](https://cloud.google.com/container-optimized-os/docs) (COS). Either a complete `cloud-config` template can be provided via the `cloud_config` variable with optional template variables via the `config_variables`, or a generic `cloud-config` can be generated based on typical parameters needed to start a container.
+
+Logging can be enabled via the [Google Cloud Logging docker driver](https://docs.docker.com/config/containers/logging/gcplogs/) using the `gcp_logging` variable. This is enabled by default, but requires that the service account running the COS instance have the `roles/logging.logWriter` IAM role or equivalent permissions on the project. If it doesn't, the container will fail to start unless this is disabled.
+
+The module renders the generated cloud config in the `cloud_config` output, which can be directly used in instances or instance templates via the `user-data` metadata attribute.
+
+## Examples
+
+### Default configuration
+
+This example will create a `cloud-config` that starts [Envoy Proxy](https://www.envoyproxy.io) and expose it on port 80. For a complete example, look at the sibling [`envoy-traffic-director`](../envoy-traffic-director/README.md) module that uses this module to start Envoy Proxy and connect it to [Traffic Director](https://cloud.google.com/traffic-director).
+
+```hcl
+module "cos-envoy" {
+  source = "./modules/cos-generic-metadata"
+
+  container_image = "envoyproxy/envoy:v1.14.1"
+  container_name  = "envoy"
+  container_args  = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields"
+
+  container_volumes = [
+    { host      = "/etc/envoy/envoy.yaml",
+      container = "/etc/envoy/envoy.yaml"
+    }
+  ]
+
+  docker_args = "--network host --pid host"
+
+  files = {
+    "/var/run/envoy/customize.sh" = {
+      content     = file("customize.sh")
+      owner       = "root"
+      permissions = "0744"
+    }
+    "/etc/envoy/envoy.yaml" = {
+      content     = file("envoy.yaml")
+      owner       = "root"
+      permissions = "0644"
+    }
+  }
+
+  run_commands = [
+    "iptables -t nat -N ENVOY_IN_REDIRECT",
+    "iptables -t nat -A ENVOY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001",
+    "iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j ENVOY_IN_REDIRECT",
+    "iptables -t filter -A INPUT -p tcp -m tcp --dport 15001 -m state --state NEW,ESTABLISHED -j ACCEPT",
+    "/var/run/envoy/customize.sh",
+    "systemctl daemon-reload",
+    "systemctl start envoy",
+  ]
+
+  users = [
+    {
+      username = "envoy",
+      uid      = 1337
+    }
+  ]
+}
+```
+
+<!-- BEGIN TFDOC -->
+## Variables
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| container\_image | Container image. | `string` | n/a | yes |
+| boot\_commands | List of cloud-init `bootcmd`s | `list(string)` | `[]` | no |
+| cloud\_config | Cloud config template path. If provided, takes precedence over all other arguments. | `string` | `null` | no |
+| config\_variables | Additional variables used to render the template passed via `cloud_config` | `map(any)` | `{}` | no |
+| container\_args | Arguments for container | `string` | `""` | no |
+| container\_name | Name of the container to be run | `string` | `"container"` | no |
+| container\_volumes | List of volumes | <pre>list(object({<br>    host      = string,<br>    container = string<br>  }))</pre> | `[]` | no |
+| docker\_args | Extra arguments to be passed for docker | `string` | `null` | no |
+| file\_defaults | Default owner and permissions for files. | <pre>object({<br>    owner       = string<br>    permissions = string<br>  })</pre> | <pre>{<br>  "owner": "root",<br>  "permissions": "0644"<br>}</pre> | no |
+| files | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | <pre>map(object({<br>    content     = string<br>    owner       = string<br>    permissions = string<br>  }))</pre> | `{}` | no |
+| gcp\_logging | Should container logs be sent to Google Cloud Logging | `bool` | `true` | no |
+| run\_commands | List of cloud-init `runcmd`s | `list(string)` | `[]` | no |
+| users | List of usernames to be created. If provided, first user will be used to run the container. | <pre>list(object({<br>    username = string,<br>    uid      = number,<br>  }))</pre> | `[]` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloud\_config | Rendered cloud-config file to be passed as user-data instance metadata. |
+<!-- END TFDOC -->

modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml

@@ -0,0 +1,82 @@
+#cloud-config
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%{ if length(users) > 0 ~}
+users:
+%{ for user in users ~}
+  - name: ${user.username}
+    uid: ${user.uid}
+%{ endfor ~}
+%{ endif ~}
+
+write_files:
+  - path: /var/lib/docker/daemon.json
+    permissions: 0644
+    owner: root
+    content: |
+      {
+        "live-restore": true,
+        "storage-driver": "overlay2",
+        "log-opts": {
+          "max-size": "1024m"
+        }
+      }
+  # ${container_name} container service
+  - path: /etc/systemd/system/${container_name}.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Start ${container_name} container
+      After=gcr-online.target docker.socket
+      Wants=gcr-online.target docker.socket docker-events-collector.service
+      [Service]
+      ExecStart=/usr/bin/docker run --rm --name=${container_name}  \
+%{ if length(users) > 0 ~}
+        --user=${users[0].uid} \
+%{ endif ~}
+%{ if gcp_logging == true ~}
+        --log-driver=gcplogs \
+%{ endif ~}
+%{ if docker_args != null ~}
+        ${docker_args} \
+%{ endif ~}
+%{ for volume in container_volumes ~}
+        -v ${volume.host}:${volume.container} \
+%{ endfor ~}
+        ${container_image} ${container_args}
+      ExecStop=/usr/bin/docker stop ${container_name}
+%{ for path, data in files ~}
+  - path: ${path}
+    owner: ${lookup(data, "owner", "root")}
+    permissions: ${lookup(data, "permissions", "0644")}
+    content: |
+      ${indent(6, data.content)}
+%{ endfor ~}
+
+%{ if length(boot_commands) > 0 ~}
+bootcmd:
+%{ for command in boot_commands ~}
+  - ${command}
+%{ endfor ~}
+%{ endif ~}
+
+%{ if length(run_commands) > 0 ~}
+runcmd:
+%{ for command in run_commands ~}
+  - ${command}
+%{ endfor ~}
+%{ endif ~}

modules/cloud-config-container/cos-generic-metadata/main.tf

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cloud_config = templatefile(local.template, merge(var.config_variables, {
+    boot_commands     = var.boot_commands
+    container_args    = var.container_args
+    container_image   = var.container_image
+    container_name    = var.container_name
+    container_volumes = var.container_volumes
+    docker_args       = var.docker_args
+    files             = local.files
+    gcp_logging       = var.gcp_logging
+    run_commands      = var.run_commands
+    users             = var.users
+  }))
+  files = {
+    for path, attrs in var.files : path => {
+      content = attrs.content,
+      owner   = attrs.owner == null ? var.file_defaults.owner : attrs.owner,
+      permissions = (
+        attrs.permissions == null
+        ? var.file_defaults.permissions
+        : attrs.permissions
+      )
+    }
+  }
+  template = (
+    var.cloud_config == null
+    ? "${path.module}/cloud-config.yaml"
+    : var.cloud_config
+  )
+}

modules/cloud-config-container/cos-generic-metadata/outputs.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "cloud_config" {
+  description = "Rendered cloud-config file to be passed as user-data instance metadata."
+  value       = local.cloud_config
+}

modules/cloud-config-container/cos-generic-metadata/variables.tf

@@ -0,0 +1,110 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "boot_commands" {
+  description = "List of cloud-init `bootcmd`s"
+  type        = list(string)
+  default     = []
+}
+
+variable "cloud_config" {
+  description = "Cloud config template path. If provided, takes precedence over all other arguments."
+  type        = string
+  default     = null
+}
+
+variable "config_variables" {
+  description = "Additional variables used to render the template passed via `cloud_config`"
+  type        = map(any)
+  default     = {}
+}
+
+variable "container_args" {
+  description = "Arguments for container"
+  type        = string
+  default     = ""
+}
+
+
+variable "container_image" {
+  description = "Container image."
+  type        = string
+}
+
+variable "container_name" {
+  description = "Name of the container to be run"
+  type        = string
+  default     = "container"
+}
+
+variable "container_volumes" {
+  description = "List of volumes"
+  type = list(object({
+    host      = string,
+    container = string
+  }))
+  default = []
+}
+
+variable "docker_args" {
+  description = "Extra arguments to be passed for docker"
+  type        = string
+  default     = null
+}
+
+variable "file_defaults" {
+  description = "Default owner and permissions for files."
+  type = object({
+    owner       = string
+    permissions = string
+  })
+  default = {
+    owner       = "root"
+    permissions = "0644"
+  }
+}
+
+variable "files" {
+  description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null."
+  type = map(object({
+    content     = string
+    owner       = string
+    permissions = string
+  }))
+  default = {}
+}
+
+variable "gcp_logging" {
+  description = "Should container logs be sent to Google Cloud Logging"
+  type        = bool
+  default     = true
+}
+
+variable "run_commands" {
+  description = "List of cloud-init `runcmd`s"
+  type        = list(string)
+  default     = []
+}
+
+variable "users" {
+  description = "List of usernames to be created. If provided, first user will be used to run the container."
+  type = list(object({
+    username = string,
+    uid      = number,
+  }))
+  default = [
+  ]
+}

modules/cloud-config-container/envoy-traffic-director/README.md

@@ -0,0 +1,59 @@
+# Containerized Envoy Proxy with Traffic Director on Container Optimized OS
+
+This module manages a `cloud-config` configuration that starts a containerized Envoy Proxy on Container Optimized OS connected to Traffic Director. The default configuration creates a reverse proxy exposed on the node's port 80. Traffic routing policies and management should be managed by other means via Traffic Director.
+
+## Examples
+
+### Default configuration
+
+```hcl
+# Envoy TD config
+module "cos-envoy-td" {
+  source = "./modules/cloud-config-container/envoy-traffic-director"
+}
+
+# COS VM
+module "vm-cos" {
+  source     = "./modules/compute-vm"
+  project_id = local.project_id
+  region     = local.region
+  zone       = local.zone
+  name       = "cos-envoy-td"
+  network_interfaces = [{
+    network    = local.vpc.self_link,
+    subnetwork = local.vpc.subnet_self_link,
+    nat        = false,
+    addresses  = null
+  }]
+  instance_count = 1
+  tags           = ["ssh", "http"]
+
+  metadata = {
+    user-data = module.cos-envoy-td.cloud_config
+  }
+
+  boot_disk = {
+    image = "projects/cos-cloud/global/images/family/cos-stable"
+    type  = "pd-ssd"
+    size  = 10
+  }
+
+  service_account_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+}
+```
+
+
+<!-- BEGIN TFDOC -->
+## Variables
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| envoy\_image | Envoy Proxy container image to use. | `string` | `"envoyproxy/envoy:v1.14.1"` | no |
+| gcp\_logging | Should container logs be sent to Google Cloud Logging | `bool` | `true` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloud\_config | Rendered cloud-config file to be passed as user-data instance metadata. |
+<!-- END TFDOC -->

modules/cloud-config-container/envoy-traffic-director/files/customize.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+ENVOY_NODE_ID=$(uuidgen)~$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip)
+ENVOY_ZONE=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/zone | cut -f 4 -d '/')
+CONFIG_PROJECT_NUMBER=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | cut -f 2 -d '/')
+VPC_NETWORK_NAME=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | cut -f 4 -d '/')
+sed -i "s/_ENVOY_NODE_ID_/${ENVOY_NODE_ID}/" /etc/envoy/envoy.yaml
+sed -i "s/_ENVOY_ZONE_/${ENVOY_ZONE}/" /etc/envoy/envoy.yaml
+sed -i "s/_CONFIG_PROJECT_NUMBER_/${CONFIG_PROJECT_NUMBER}/" /etc/envoy/envoy.yaml
+sed -i "s/_VPC_NETWORK_NAME_/${VPC_NETWORK_NAME}/" /etc/envoy/envoy.yaml

modules/cloud-config-container/envoy-traffic-director/files/envoy.yaml

@@ -0,0 +1,140 @@
+node:
+  id: "_ENVOY_NODE_ID_"
+  cluster: cluster  # unused
+  locality:
+    zone: "_ENVOY_ZONE_"
+  metadata:
+    TRAFFICDIRECTOR_INTERCEPTION_PORT: "15001"
+    TRAFFICDIRECTOR_NETWORK_NAME: "_VPC_NETWORK_NAME_"
+    TRAFFICDIRECTOR_GCP_PROJECT_NUMBER: "_CONFIG_PROJECT_NUMBER_"
+    TRAFFICDIRECTOR_ENABLE_TRACING: "false"
+    TRAFFICDIRECTOR_ACCESS_LOG_PATH: ""
+    TRAFFICDIRECTOR_INBOUND_BACKEND_PORTS: ""
+
+dynamic_resources:
+  lds_config: {ads: {}}
+  cds_config: {ads: {}}
+  ads_config:
+    api_type: GRPC
+    grpc_services:
+    - google_grpc:
+        target_uri: trafficdirector.googleapis.com:443
+        stat_prefix: trafficdirector
+        channel_credentials:
+          ssl_credentials:
+            root_certs:
+              filename: /etc/ssl/certs/ca-certificates.crt
+        call_credentials:
+          google_compute_engine: {}
+
+cluster_manager:
+  load_stats_config:
+    api_type: GRPC
+    grpc_services:
+    - google_grpc:
+        target_uri: trafficdirector.googleapis.com:443
+        stat_prefix: trafficdirector
+        channel_credentials:
+          ssl_credentials:
+            root_certs:
+              filename: /etc/ssl/certs/ca-certificates.crt
+        call_credentials:
+          google_compute_engine: {}
+
+admin:
+  access_log_path: /dev/stdout
+  address:
+    socket_address:
+      address: 127.0.0.1  # Admin page is only accessible locally.
+      port_value: 15000
+
+tracing:
+  http:
+    name: envoy.tracers.opencensus
+    typed_config:
+      "@type": type.googleapis.com/envoy.config.trace.v2.OpenCensusConfig
+      stackdriver_exporter_enabled: "false"
+      stackdriver_project_id: ""
+
+layered_runtime:
+  layers:
+  - name: rtds_layer
+    rtds_layer:
+      name: traffic_director_runtime
+      rtds_config: {ads: {}}
+  - name: static_layer
+    static_layer:
+      envoy:
+        deprecated_features:
+          cluster:
+            proto:ORIGINAL_DST_LB: "true"
+            proto:extension_protocol_options: "true"
+            proto:tls_context: "true"
+          health_check:
+            proto:use_http2: "true"
+          http_connection_manager:
+            proto:operation_name: "true"
+          listener:
+            proto:tls_context: "true"
+          listener_components:
+            proto:config: "true"
+          route_components:
+            proto:allow_origin: "true"
+            proto:method: "true"
+            proto:pattern: "true"
+            proto:regex: "true"
+            proto:regex_match: "true"
+            proto:value: "true"
+          string:
+            proto:regex: "true"
+          trace:
+            proto:HTTP_JSON_V1: "true"
+        deprecated_features:envoy:
+          api:
+            v2:
+              Cluster:
+                LbPolicy:
+                  ORIGINAL_DST_LB: "true"
+                extension_protocol_options: "true"
+                tls_context: "true"
+              Listener:
+                tls_context: "true"
+              core:
+                HealthCheck:
+                  HttpHealthCheck:
+                    use_http2: "true"
+              listener:
+                Filter:
+                  config: "true"
+                ListenerFilter:
+                  config: "true"
+              route:
+                CorsPolicy:
+                  allow_origin: "true"
+                HeaderMatcher:
+                  regex_match: "true"
+                QueryParameterMatcher:
+                  regex: "true"
+                  value: "true"
+                RouteMatch:
+                  regex: "true"
+                VirtualCluster:
+                  method: "true"
+                  pattern: "true"
+          config:
+            filter:
+              network:
+                http_connection_manager:
+                  v2:
+                    HttpConnectionManager:
+                      Tracing:
+                        operation_name: "true"
+            trace:
+              v2:
+                ZipkinConfig:
+                  CollectorEndpointVersion:
+                    HTTP_JSON_V1: "true"
+          type:
+            matcher:
+              StringMatcher:
+                regex: "true"

modules/cloud-config-container/envoy-traffic-director/main.tf

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "cos-envoy-td" {
+  source = "./modules/cos-generic-metadata"
+
+  boot_commands = [
+    "systemctl start node-problem-detector",
+  ]
+
+  container_image = var.envoy_image
+  container_name  = "envoy"
+  container_args  = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields"
+
+  container_volumes = [
+    { host      = "/etc/envoy/envoy.yaml",
+      container = "/etc/envoy/envoy.yaml"
+    }
+  ]
+
+  docker_args = "--network host --pid host"
+
+  files = {
+    "/var/run/envoy/customize.sh" = {
+      content     = file("${path.module}/files/customize.sh")
+      owner       = "root"
+      permissions = "0744"
+    }
+    "/etc/envoy/envoy.yaml" = {
+      content     = file("${path.module}/files/envoy.yaml")
+      owner       = "root"
+      permissions = "0644"
+    }
+  }
+
+  gcp_logging = var.gcp_logging
+
+  run_commands = [
+    "iptables -t nat -N ENVOY_IN_REDIRECT",
+    "iptables -t nat -A ENVOY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001",
+    "iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j ENVOY_IN_REDIRECT",
+    "iptables -t filter -A INPUT -p tcp -m tcp --dport 15001 -m state --state NEW,ESTABLISHED -j ACCEPT",
+    "/var/run/envoy/customize.sh",
+    "systemctl daemon-reload",
+    "systemctl start envoy",
+  ]
+
+  users = [
+    {
+      username = "envoy",
+      uid      = 1337
+    }
+  ]
+}

modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata

@@ -0,0 +1 @@
+../../cos-generic-metadata

modules/cloud-config-container/envoy-traffic-director/outputs.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "cloud_config" {
+  description = "Rendered cloud-config file to be passed as user-data instance metadata."
+  value       = module.cos-envoy-td.cloud_config
+}

modules/cloud-config-container/envoy-traffic-director/variables.tf

@@ -0,0 +1,11 @@
+variable "envoy_image" {
+  description = "Envoy Proxy container image to use."
+  type        = string
+  default     = "envoyproxy/envoy:v1.14.1"
+}
+
+variable "gcp_logging" {
+  description = "Should container logs be sent to Google Cloud Logging"
+  type        = bool
+  default     = true
+}