diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/Dockerfile b/blueprints/networking/nginx-reverse-proxy-cluster/Dockerfile new file mode 100644 index 00000000..748a64a2 --- /dev/null +++ b/blueprints/networking/nginx-reverse-proxy-cluster/Dockerfile @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM marketplace.gcr.io/google/debian11 + +RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y curl gnupg2 +RUN curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +RUN bash add-google-cloud-ops-agent-repo.sh --also-install +RUN rm -f add-google-cloud-ops-agent-repo.sh + +RUN echo '#!/bin/bash' > /entrypoint.sh +RUN echo 'cd /tmp' >> /entrypoint.sh +RUN echo '/opt/google-cloud-ops-agent/libexec/google_cloud_ops_agent_engine -service=otel -in /etc/google-cloud-ops-agent/config.yaml' >> /entrypoint.sh +RUN echo '/opt/google-cloud-ops-agent/subagents/opentelemetry-collector/otelopscol --config=/tmp/otel.yaml --feature-gates=exporter.googlecloud.OTLPDirect' >> /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT /entrypoint.sh +CMD [] \ No newline at end of file diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/README.md b/blueprints/networking/nginx-reverse-proxy-cluster/README.md new file mode 100644 index 00000000..9cacb4b0 --- /dev/null +++ b/blueprints/networking/nginx-reverse-proxy-cluster/README.md @@ -0,0 +1,45 @@ +# Nginx-based reverse proxy cluster + +This blueprint shows how to deploy an autoscaling reverse proxy cluster using Nginx, based on regional +Managed Instance Groups. + +![High-level diagram](reverse-proxy.png "High-level diagram") + +The autoscaling is driven by Nginx current connections metric, sent by Cloud Ops Agent. + +The example is for Nginx, but it could be easily adapted to any other reverse proxy software (eg. +Squid, Varnish, etc). + +## Ops Agent image + +There is a simple [`Dockerfile`](Dockerfile) available for building Ops Agent to be run +inside the ContainerOS instance. Build the container, push it to your Container/Artifact +Repository and set the `ops_agent_image` to point to the image you built. + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [autoscaling_metric](variables.tf#L31) | | object({…} | ✓ | | +| [project_name](variables.tf#L106) | Name of an existing project or of the new project | string | ✓ | | +| [autoscaling](variables.tf#L17) | Autoscaling configuration for the instance group. | object({…}) | | {…} | +| [backends](variables.tf#L49) | Nginx locations configurations to proxy traffic to. | string | | "<<-EOT…EOT" | +| [cidrs](variables.tf#L59) | Subnet IP CIDR ranges. | map(string) | | {…} | +| [network](variables.tf#L67) | Network name. | string | | "reverse-proxy-vpc" | +| [network_create](variables.tf#L73) | Create network or use existing one. | bool | | true | +| [nginx_image](variables.tf#L79) | Nginx container image to use. | string | | "gcr.io/cloud-marketplace/google/nginx1:latest" | +| [ops_agent_image](variables.tf#L85) | Google Cloud Ops Agent container image to use. | string | | "gcr.io/sfans-hub-project-d647/ops-agent:latest" | +| [prefix](variables.tf#L91) | Prefix used for resources that need unique names. | string | | "" | +| [project_create](variables.tf#L97) | Parameters for the creation of the new project | object({…}) | | null | +| [region](variables.tf#L111) | Default region for resources. | string | | "europe-west4" | +| [subnetwork](variables.tf#L117) | Subnetwork name. | string | | "gce" | +| [tls](variables.tf#L123) | Also offer reverse proxying with TLS (self-signed certificate). | bool | | false | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [load_balancer_url](outputs.tf#L17) | Load balancer for the reverse proxy instance group. | | + + diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/main.tf b/blueprints/networking/nginx-reverse-proxy-cluster/main.tf new file mode 100644 index 00000000..db5b6247 --- /dev/null +++ b/blueprints/networking/nginx-reverse-proxy-cluster/main.tf @@ -0,0 +1,403 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + monitoring_agent_unit = <<-EOT + [Unit] + Description=Start monitoring agent container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + + [Service] + Environment="HOME=/home/opsagent" + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker + ExecStart=/usr/bin/docker run --rm --name=monitoring-agent \ + --log-driver=gcplogs \ + --network host \ + -v /etc/google-cloud-ops-agent/config.yaml:/etc/google-cloud-ops-agent/config.yaml \ + ${var.ops_agent_image} + ExecStop=/usr/bin/docker stop monitoring-agent + EOT + monitoring_agent_config = <<-EOT + logging: + service: + pipelines: + default_pipeline: + receivers: [] + metrics: + receivers: + hostmetrics: + type: hostmetrics + nginx: + type: nginx + collection_interval: 10s + stub_status_url: http://localhost/healthz + service: + pipelines: + default_pipeline: + receivers: + - hostmetrics + - nginx + EOT + nginx_config = <<-EOT + server { + listen 80; + server_name HOSTNAME localhost; + %{if var.tls} + listen 443 ssl; + ssl_certificate /etc/ssl/self-signed.crt; + ssl_certificate_key /etc/ssl/self-signed.key; + %{endif} + + keepalive_timeout 650s; + keepalive_requests 10000; + + proxy_connect_timeout 60s; + proxy_read_timeout 5m; + proxy_send_timeout 5m; + + error_log stderr; + access_log /dev/stdout combined; + + set_real_ip_from ${module.xlb.ip_address}/32; + set_real_ip_from 35.191.0.0/16; + set_real_ip_from 130.211.0.0/22; + real_ip_header X-Forwarded-For; + real_ip_recursive off; + + location /healthz { + stub_status on; + access_log off; + allow 127.0.0.1; + allow 35.191.0.0/16; + allow 130.211.0.0/22; + deny all; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + ${var.backends} + } + EOT + nginx_files = { + "/etc/systemd/system/monitoring-agent.service" = { + content = local.monitoring_agent_unit + owner = "root" + permissions = "0644" + } + "/etc/nginx/conf.d/default.conf" = { + content = local.nginx_config + owner = "root" + permissions = "0644" + } + "/etc/google-cloud-ops-agent/config.yaml" = { + content = local.monitoring_agent_config + owner = "root" + permissions = "0644" + } + } + users = [ + { + username = "opsagent" + uid = 2001 + } + ] +} + +module "project" { + source = "../../../modules/project" + billing_account = (var.project_create != null + ? var.project_create.billing_account_id + : null + ) + name = var.project_name + parent = (var.project_create != null + ? var.project_create.parent + : null + ) + + services = [ + "cloudresourcemanager.googleapis.com", + "compute.googleapis.com", + "iam.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + ] + + project_create = var.project_create != null +} + +module "vpc" { + source = "../../../modules/net-vpc" + project_id = module.project.project_id + name = var.network + subnets = [ + { + name = var.subnetwork + ip_cidr_range = var.cidrs[var.subnetwork] + region = var.region + secondary_ip_range = null + }, + ] + + vpc_create = var.network_create +} + +module "firewall" { + source = "../../../modules/net-vpc-firewall" + project_id = module.project.project_id + network = module.vpc.name + custom_rules = { + format("%sallow-http-to-proxy-cluster", var.prefix) = { + description = "Allow Nginx HTTP(S) ingress traffic" + direction = "INGRESS" + action = "allow" + sources = [] + ranges = [var.cidrs[var.subnetwork], "35.191.0.0/16", "130.211.0.0/22"] + targets = [module.service-account-proxy.email] + use_service_accounts = true + rules = [{ protocol = "tcp", ports = [80, 443] }] + extra_attributes = {} + } + format("%sallow-iap-ssh", var.prefix) = { + description = "Allow Nginx SSH traffic from IAP" + direction = "INGRESS" + action = "allow" + sources = [] + ranges = ["35.235.240.0/20"] + targets = [module.service-account-proxy.email] + use_service_accounts = true + rules = [{ protocol = "tcp", ports = [22] }] + extra_attributes = {} + } + } +} + +module "nat" { + source = "../../../modules/net-cloudnat" + project_id = module.project.project_id + region = var.region + name = format("%snat", var.prefix) + router_network = module.vpc.name + config_source_subnets = "LIST_OF_SUBNETWORKS" + + logging_filter = "ALL" + + config_min_ports_per_vm = 4000 + subnetworks = [ + { + self_link = module.vpc.subnet_self_links[format("%s/%s", var.region, var.subnetwork)] + config_source_ranges = ["ALL_IP_RANGES"] + secondary_ranges = null + } + ] +} + +############################################################################### +# Proxy resources # +############################################################################### + +module "service-account-proxy" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = format("%sreverse-proxy", var.prefix) + iam_project_roles = { + (module.project.project_id) = [ + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + "roles/storage.objectViewer", // For pulling the Ops Agent image + ] + } +} + +module "cos-nginx" { + count = !var.tls ? 1 : 0 + source = "../../../modules/cloud-config-container/nginx" + + image = var.nginx_image + files = local.nginx_files + users = local.users + + runcmd_pre = ["sed -i \"s/HOSTNAME/$${HOSTNAME}/\" /etc/nginx/conf.d/default.conf"] + runcmd_post = ["systemctl start monitoring-agent"] +} + +module "cos-nginx-tls" { + count = var.tls ? 1 : 0 + source = "../../../modules/cloud-config-container/nginx-tls" + + nginx_image = var.nginx_image + files = local.nginx_files + users = local.users + + runcmd_post = ["systemctl start monitoring-agent"] +} + +module "mig-proxy" { + source = "../../../modules/compute-mig" + project_id = module.project.project_id + + location = var.region + regional = true + + name = format("%sproxy-cluster", var.prefix) + + named_ports = { + http = "80" + https = "443" + } + + autoscaler_config = var.autoscaling == null ? null : { + min_replicas = var.autoscaling.min_replicas + max_replicas = var.autoscaling.max_replicas + cooldown_period = var.autoscaling.cooldown_period + cpu_utilization_target = null + load_balancing_utilization_target = null + metric = var.autoscaling_metric + } + + update_policy = { + type = "PROACTIVE" + minimal_action = "REPLACE" + min_ready_sec = 60 + max_surge_type = "fixed" + max_surge = 3 + max_unavailable_type = null + max_unavailable = null + } + + default_version = { + instance_template = module.proxy-vm.template.self_link + name = "proxy-vm" + } + + health_check_config = { + type = "http" + check = { + port = 80 + request_path = "/healthz" + } + config = { + check_interval_sec = 10 + healthy_threshold = 1 + unhealthy_threshold = 1 + timeout_sec = 10 + } + logging = true + } + auto_healing_policies = { + health_check = module.mig-proxy.health_check.self_link + initial_delay_sec = 60 + } +} + +module "proxy-vm" { + source = "../../../modules/compute-vm" + + project_id = module.project.project_id + + zone = format("%s-c", var.region) + name = "nginx-test-vm" + + instance_type = "e2-standard-2" + + tags = ["proxy-cluster"] + network_interfaces = [{ + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links[format("%s/%s", var.region, var.subnetwork)] + nat = false + addresses = null + }] + + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + + create_template = true + metadata = { + user-data = !var.tls ? module.cos-nginx.0.cloud_config : module.cos-nginx-tls.0.cloud_config + } + + service_account = module.service-account-proxy.email + service_account_create = false +} + +module "xlb" { + source = "../../../modules/net-glb" + name = format("%sreverse-proxy-xlb", var.prefix) + project_id = module.project.project_id + + reserve_ip_address = true + + health_checks_config = { + format("%sreverse-proxy-hc", var.prefix) = { + type = "http" + logging = false + options = { + check_interval_sec = 10 + healthy_threshold = 1 + unhealthy_threshold = 1 + timeout_sec = 10 + } + check = { + port_specification = "USE_NAMED_PORT" + port_name = "http" + request_path = "/healthz" + } + } + } + + backend_services_config = { + format("%sreverse-proxy-backend", var.prefix) = { + bucket_config = null + enable_cdn = false + cdn_config = null + + group_config = { + backends = [ + { + group = module.mig-proxy.group_manager.instance_group + options = null + } + ] + + health_checks = [format("%sreverse-proxy-hc", var.prefix)] + log_config = null + options = { + affinity_cookie_ttl_sec = null + custom_request_headers = null + custom_response_headers = null + connection_draining_timeout_sec = null + load_balancing_scheme = null + locality_lb_policy = null + port_name = !var.tls ? "http" : "https" + protocol = !var.tls ? "HTTP" : "HTTPS" + security_policy = null + session_affinity = null + timeout_sec = null + circuits_breakers = null + consistent_hash = null + iap = null + } + } + } + } +} diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/outputs.tf b/blueprints/networking/nginx-reverse-proxy-cluster/outputs.tf new file mode 100644 index 00000000..ee529a3d --- /dev/null +++ b/blueprints/networking/nginx-reverse-proxy-cluster/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "load_balancer_url" { + description = "Load balancer for the reverse proxy instance group." + value = !var.tls ? format("http://%s/", module.xlb.ip_address) : format("https://%s/", module.xlb.ip_address) +} diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/reverse-proxy.png b/blueprints/networking/nginx-reverse-proxy-cluster/reverse-proxy.png new file mode 100644 index 00000000..dd0c8e62 Binary files /dev/null and b/blueprints/networking/nginx-reverse-proxy-cluster/reverse-proxy.png differ diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/variables.tf b/blueprints/networking/nginx-reverse-proxy-cluster/variables.tf new file mode 100644 index 00000000..e4409424 --- /dev/null +++ b/blueprints/networking/nginx-reverse-proxy-cluster/variables.tf @@ -0,0 +1,130 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "autoscaling" { + description = "Autoscaling configuration for the instance group." + type = object({ + min_replicas = number + max_replicas = number + cooldown_period = number + }) + default = { + min_replicas = 1 + max_replicas = 10 + cooldown_period = 30 + } +} + +variable "autoscaling_metric" { + type = object({ + name = string + single_instance_assignment = number + target = number + type = string # GAUGE, DELTA_PER_SECOND, DELTA_PER_MINUTE + filter = string + }) + + default = { + name = "workload.googleapis.com/nginx.connections_current" + single_instance_assignment = null + target = 10 # Target 10 connections per instance, just for demonstration purposes + type = "GAUGE" + filter = null + } +} + +variable "backends" { + description = "Nginx locations configurations to proxy traffic to." + type = string + default = <<-EOT + location / { + proxy_pass http://10.0.16.58:80; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + EOT +} + +variable "cidrs" { + description = "Subnet IP CIDR ranges." + type = map(string) + default = { + gce = "10.0.16.0/24" + } +} + +variable "network" { + description = "Network name." + type = string + default = "reverse-proxy-vpc" +} + +variable "network_create" { + description = "Create network or use existing one." + type = bool + default = true +} + +variable "nginx_image" { + description = "Nginx container image to use." + type = string + default = "gcr.io/cloud-marketplace/google/nginx1:latest" +} + +variable "ops_agent_image" { + description = "Google Cloud Ops Agent container image to use." + type = string + default = "gcr.io/sfans-hub-project-d647/ops-agent:latest" +} + +variable "prefix" { + description = "Prefix used for resources that need unique names." + type = string + default = "" +} + +variable "project_create" { + description = "Parameters for the creation of the new project" + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "project_name" { + description = "Name of an existing project or of the new project" + type = string +} + +variable "region" { + description = "Default region for resources." + type = string + default = "europe-west4" +} + +variable "subnetwork" { + description = "Subnetwork name." + type = string + default = "gce" +} + +variable "tls" { + description = "Also offer reverse proxying with TLS (self-signed certificate)." + type = bool + default = false +} + diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/versions.tf b/blueprints/networking/nginx-reverse-proxy-cluster/versions.tf new file mode 100644 index 00000000..8abac788 --- /dev/null +++ b/blueprints/networking/nginx-reverse-proxy-cluster/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.32.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.32.0" # tftest + } + } +} + + diff --git a/modules/cloud-config-container/cos-generic-metadata/README.md b/modules/cloud-config-container/cos-generic-metadata/README.md index 9cbaad20..69e16235 100644 --- a/modules/cloud-config-container/cos-generic-metadata/README.md +++ b/modules/cloud-config-container/cos-generic-metadata/README.md @@ -64,7 +64,7 @@ module "cos-envoy" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [container_image](variables.tf#L42) | Container image. | string | ✓ | | -| [authenticate_gcr](variables.tf#L118) | Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined. | bool | | false | +| [authenticate_gcr](variables.tf#L124) | Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined. | bool | | false | | [boot_commands](variables.tf#L17) | List of cloud-init `bootcmd`s. | list(string) | | [] | | [cloud_config](variables.tf#L23) | Cloud config template path. If provided, takes precedence over all other arguments. | string | | null | | [config_variables](variables.tf#L29) | Additional variables used to render the template passed via `cloud_config`. | map(any) | | {} | @@ -76,6 +76,7 @@ module "cos-envoy" { | [file_defaults](variables.tf#L74) | Default owner and permissions for files. | object({…}) | | {…} | | [files](variables.tf#L86) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | | [gcp_logging](variables.tf#L96) | Should container logs be sent to Google Cloud Logging. | bool | | true | +| [run_as_first_user](variables.tf#L118) | Run as the first user if users are specified. | bool | | true | | [run_commands](variables.tf#L102) | List of cloud-init `runcmd`s. | list(string) | | [] | | [users](variables.tf#L108) | List of usernames to be created. If provided, first user will be used to run the container. | list(object({…})) | | […] | diff --git a/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml b/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml index cb3f76ff..9f15f84f 100644 --- a/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml +++ b/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml @@ -49,7 +49,7 @@ write_files: ExecStartPre=/usr/bin/docker-credential-gcr configure-docker %{~ endif ~} ExecStart=/usr/bin/docker run --rm --name=${container_name} \ - %{~ if length(users) > 0 ~} + %{~ if length(users) > 0 && run_as_first_user ~} --user=${users[0].uid} \ %{~ endif ~} %{~ if docker_logging ~} diff --git a/modules/cloud-config-container/cos-generic-metadata/main.tf b/modules/cloud-config-container/cos-generic-metadata/main.tf index 835183f3..ff02f325 100644 --- a/modules/cloud-config-container/cos-generic-metadata/main.tf +++ b/modules/cloud-config-container/cos-generic-metadata/main.tf @@ -28,6 +28,7 @@ locals { run_commands = var.run_commands users = var.users authenticate_gcr = var.authenticate_gcr + run_as_first_user = var.run_as_first_user })) files = { for path, attrs in var.files : path => { diff --git a/modules/cloud-config-container/cos-generic-metadata/variables.tf b/modules/cloud-config-container/cos-generic-metadata/variables.tf index b84842f5..934c0520 100644 --- a/modules/cloud-config-container/cos-generic-metadata/variables.tf +++ b/modules/cloud-config-container/cos-generic-metadata/variables.tf @@ -115,6 +115,12 @@ variable "users" { ] } +variable "run_as_first_user" { + description = "Run as the first user if users are specified." + type = bool + default = true +} + variable "authenticate_gcr" { description = "Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined." type = bool diff --git a/modules/cloud-config-container/nginx-tls/README.md b/modules/cloud-config-container/nginx-tls/README.md index 44807d5d..45cf1196 100644 --- a/modules/cloud-config-container/nginx-tls/README.md +++ b/modules/cloud-config-container/nginx-tls/README.md @@ -50,7 +50,11 @@ module "vm-nginx-tls" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [docker_logging](variables.tf#L23) | Log via the Docker gcplogs driver. Disable if you use the legacy Logging Agent instead. | bool | | true | +| [files](variables.tf#L41) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | null | | [nginx_image](variables.tf#L17) | Nginx container image to use. | string | | "nginx:1.23.1" | +| [runcmd_post](variables.tf#L35) | Extra commands to run after starting nginx. | list(string) | | [] | +| [runcmd_pre](variables.tf#L29) | Extra commands to run before starting nginx. | list(string) | | [] | +| [users](variables.tf#L51) | Additional list of usernames to be created. | list(object({…})) | | […] | ## Outputs diff --git a/modules/cloud-config-container/nginx-tls/files/customize.sh b/modules/cloud-config-container/nginx-tls/files/customize.sh index 0d773771..afbf56db 100644 --- a/modules/cloud-config-container/nginx-tls/files/customize.sh +++ b/modules/cloud-config-container/nginx-tls/files/customize.sh @@ -16,4 +16,5 @@ FQDN=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/hostname) HOSTNAME=$(echo $FQDN | cut -d"." -f1) openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj /CN=$HOSTNAME/ -addext "subjectAltName = DNS:$FQDN" -keyout /etc/ssl/self-signed.key -out /etc/ssl/self-signed.crt +chgrp nginx /etc/ssl/self-signed.key -out /etc/ssl/self-signed.crt sed -i "s/HOSTNAME/${HOSTNAME}/" /etc/nginx/conf.d/default.conf \ No newline at end of file diff --git a/modules/cloud-config-container/nginx-tls/main.tf b/modules/cloud-config-container/nginx-tls/main.tf index ae668cc7..6a4e4ea6 100644 --- a/modules/cloud-config-container/nginx-tls/main.tf +++ b/modules/cloud-config-container/nginx-tls/main.tf @@ -14,9 +14,34 @@ * limitations under the License. */ +locals { + default_files = { + "/var/run/nginx/customize.sh" = { + content = file("${path.module}/files/customize.sh") + owner = "root" + permissions = "0744" + } + "/etc/nginx/conf.d/default.conf" = { + content = file("${path.module}/files/default.conf") + owner = "root" + permissions = "0644" + } + } + files = var.files != null ? merge(local.default_files, var.files) : local.default_files +} + module "cos-envoy-td" { source = "../cos-generic-metadata" + authenticate_gcr = true + users = concat([ + { + username = "nginx" + uid = 2000 + } + ], var.users) + run_as_first_user = false + boot_commands = [ "systemctl start node-problem-detector", ] @@ -32,27 +57,16 @@ module "cos-envoy-td" { docker_args = "--network host --pid host" - files = { - "/var/run/nginx/customize.sh" = { - content = file("${path.module}/files/customize.sh") - owner = "root" - permissions = "0744" - } - "/etc/nginx/conf.d/default.conf" = { - content = file("${path.module}/files/default.conf") - owner = "root" - permissions = "0644" - } - } + files = local.files gcp_logging = var.docker_logging - run_commands = [ + run_commands = concat(var.runcmd_pre, [ "iptables -I INPUT 1 -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT", "iptables -I INPUT 1 -p tcp -m tcp --dport 443 -m state --state NEW,ESTABLISHED -j ACCEPT", "/var/run/nginx/customize.sh", "systemctl daemon-reload", "systemctl start nginx", - ] + ], var.runcmd_post) } diff --git a/modules/cloud-config-container/nginx-tls/variables.tf b/modules/cloud-config-container/nginx-tls/variables.tf index 246e6d07..dc2295f8 100644 --- a/modules/cloud-config-container/nginx-tls/variables.tf +++ b/modules/cloud-config-container/nginx-tls/variables.tf @@ -25,3 +25,37 @@ variable "docker_logging" { type = bool default = true } + +variable "runcmd_pre" { + description = "Extra commands to run before starting nginx." + type = list(string) + default = [] +} + +variable "runcmd_post" { + description = "Extra commands to run after starting nginx." + type = list(string) + default = [] +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = null +} + +variable "users" { + description = "Additional list of usernames to be created." + type = list(object({ + username = string, + uid = number, + })) + default = [ + ] +} + + diff --git a/modules/cloud-config-container/nginx/README.md b/modules/cloud-config-container/nginx/README.md index 6ae4f63c..104255c4 100644 --- a/modules/cloud-config-container/nginx/README.md +++ b/modules/cloud-config-container/nginx/README.md @@ -64,8 +64,11 @@ module "cos-nginx" { | [files](variables.tf#L59) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | | [image](variables.tf#L35) | Nginx container image. | string | | "nginxdemos/hello:plain-text" | | [nginx_config](variables.tf#L41) | Nginx configuration path, if null container default will be used. | string | | null | +| [runcmd_post](variables.tf#L75) | Extra commands to run after starting nginx. | list(string) | | [] | +| [runcmd_pre](variables.tf#L69) | Extra commands to run before starting nginx. | list(string) | | [] | | [test_instance](variables-instance.tf#L17) | Test/development instance attributes, leave null to skip creation. | object({…}) | | null | | [test_instance_defaults](variables-instance.tf#L30) | Test/development instance defaults used for optional configuration. If image is null, COS stable will be used. | object({…}) | | {…} | +| [users](variables.tf#L81) | List of additional usernames to be created. | list(object({…})) | | […] | ## Outputs diff --git a/modules/cloud-config-container/nginx/cloud-config.yaml b/modules/cloud-config-container/nginx/cloud-config.yaml index f7be84df..af3116a3 100644 --- a/modules/cloud-config-container/nginx/cloud-config.yaml +++ b/modules/cloud-config-container/nginx/cloud-config.yaml @@ -20,6 +20,10 @@ users: - name: nginx uid: 2000 + %{ for user in users } + - name: ${user.username} + uid: ${user.uid} + %{ endfor } write_files: - path: /var/lib/docker/daemon.json @@ -52,6 +56,8 @@ write_files: After=gcr-online.target docker.socket Wants=gcr-online.target docker.socket docker-events-collector.service [Service] + Environment="HOME=/home/nginx" + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker ExecStart=/usr/bin/docker run --rm --name=nginx \ %{~ if docker_logging ~} --log-driver=gcplogs \ @@ -68,13 +74,19 @@ write_files: owner: ${lookup(data, "owner", "root")} permissions: ${lookup(data, "permissions", "0644")} content: | - ${indent(4, data.content)} + ${indent(6, data.content)} %{ endfor } bootcmd: - systemctl start node-problem-detector runcmd: +%{ for cmd in runcmd_pre ~} + - ${cmd} +%{ endfor ~} - iptables -I INPUT 1 -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT - systemctl daemon-reload - systemctl start nginx +%{ for cmd in runcmd_post ~} + - ${cmd} +%{ endfor ~} diff --git a/modules/cloud-config-container/nginx/main.tf b/modules/cloud-config-container/nginx/main.tf index 688545d7..608e7fa2 100644 --- a/modules/cloud-config-container/nginx/main.tf +++ b/modules/cloud-config-container/nginx/main.tf @@ -21,13 +21,16 @@ locals { var.nginx_config != null || length([ for name in keys(var.files) : name if substr(name, 0, 18) == "/etc/nginx/conf.d/" - ]) > 1 + ]) > 0 ) files = local.files + users = var.users image = var.image nginx_config = (var.nginx_config == null ? null : templatefile( var.nginx_config, var.config_variables )) + runcmd_pre = var.runcmd_pre + runcmd_post = var.runcmd_post })) files = { for path, attrs in var.files : path => { diff --git a/modules/cloud-config-container/nginx/variables.tf b/modules/cloud-config-container/nginx/variables.tf index c0ad3f6e..ab77d774 100644 --- a/modules/cloud-config-container/nginx/variables.tf +++ b/modules/cloud-config-container/nginx/variables.tf @@ -65,3 +65,25 @@ variable "files" { })) default = {} } + +variable "runcmd_pre" { + description = "Extra commands to run before starting nginx." + type = list(string) + default = [] +} + +variable "runcmd_post" { + description = "Extra commands to run after starting nginx." + type = list(string) + default = [] +} + +variable "users" { + description = "List of additional usernames to be created." + type = list(object({ + username = string, + uid = number, + })) + default = [ + ] +}