Refactor the onprem module (#55)

* move onprem to cos-container

* compute-vm: fix external addresses output

* folders-unit: update README

* update onprem module, add new fields to cos-container test instance

* coredns: process corefile as a template

* onprem: fixes

* modules/cos-container: rename to cloud-config-container infra/onprem: remove test output

* Update README.md

* update CHANGELOG for v1.1.0

* fix cloud config modules tests

* Update main.tf

* add container nginx module
This commit is contained in:
Ludovico Magnocavallo 2020-04-06 16:27:13 +02:00 committed by GitHub
parent c486bfc66f
commit 409407ae7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
67 changed files with 932 additions and 556 deletions

View File

@ -4,10 +4,18 @@ All notable changes to this project will be documented in this file.
## [Unreleased]
## [1.1.0] - 2020-03-27
- rename the `cos-container` suite of modules to `cloud-config-container`
- refactor the `onprem-in-a-box` module to only manage the `cloud-config` configuration, and make it part of the `cloud-config-container` suite of modules
- update the `onprem-google-access-dns` example to use the refactored `onprem` module
- fix the `external_addresses` output in the `compute-vm` module
- small tweaks and fixes to the `cloud-config-container` modules
## [1.0.0] - 2020-03-27
- merge development branch with suite of new modules and end-to-end examples
[Unreleased]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.0.0...HEAD
[1.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.0.0...v1.1.0
[1.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v0.1...v1.0

View File

@ -34,8 +34,8 @@ Currently available modules:
- **foundational** - [folders](./modules/folders), [log sinks](./modules/logging-sinks), [project](./modules/project), [service accounts](./modules/iam-service-accounts)
- **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), VPN ([static](./modules/net-vpn-static), [dynamic](./modules/net-vpn-dynamic), [HA](./modules/net-vpn-ha)), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns)
- **compute** - [VM/VM group](./modules/compute-vm), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/compute-vm-cos-coredns)
- **compute** - [VM/VM group](./modules/compute-vm), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cos-container)
- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery)
- **other** - [KMS](./modules/kms), [on-premises in Docker](./modules/on-prem-in-a-box)
- **security** - [KMS](./modules/kms)
For more information and usage examples see each module's README file.

View File

@ -96,12 +96,13 @@ sudo docker exec -it onprem_bird_1 ip route |grep bird
10.0.0.0/24 via 169.254.1.1 dev vti0 proto bird src 10.0.16.2
35.199.192.0/19 via 169.254.1.1 dev vti0 proto bird src 10.0.16.2
199.36.153.4/30 via 169.254.1.1 dev vti0 proto bird src 10.0.16.2
199.36.153.8/30 via 169.254.1.1 dev vti0 proto bird src 10.0.16.2
# get a shell on the toolbox container
sudo docker exec -it onprem_toolbox_1 sh
# test forwarding from CoreDNS via the Cloud DNS inbound policy
dig test-1.gcp.example.com +short
dig test-1.gcp.example.org +short
10.0.0.3
# test that Private Access is configured correctly
@ -124,8 +125,8 @@ gcloud compute instances list
gcloud compute ssh test-1
# test forwarding from Cloud DNS to onprem CoreDNS (address may differ)
dig gw.onprem.example.com +short
10.0.16.2
dig gw.onprem.example.org +short
10.0.16.1
# test a request to the onprem web server
curl www.onprem.example.com -s |grep h1

View File

@ -1,11 +1,11 @@
onprem.example.com {
onprem.example.org {
root /etc/coredns
hosts onprem.hosts
log
errors
}
gcp.example.com googleapis.com {
forward . ${forwarder_address}
gcp.example.org googleapis.com {
forward . ${dns_forwarder_address}
log
errors
}

View File

@ -0,0 +1,20 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
terraform {
backend "gcs" {
bucket = ""
}
}

View File

@ -18,11 +18,13 @@ locals {
bgp_interface_gcp = "${cidrhost(var.bgp_interface_ranges.gcp, 1)}"
bgp_interface_onprem = "${cidrhost(var.bgp_interface_ranges.gcp, 2)}"
netblocks = {
dns = data.google_netblock_ip_ranges.dns-forwarders.cidr_blocks_ipv4.0
api = data.google_netblock_ip_ranges.private-googleapis.cidr_blocks_ipv4.0
dns = data.google_netblock_ip_ranges.dns-forwarders.cidr_blocks_ipv4.0
private = data.google_netblock_ip_ranges.private-googleapis.cidr_blocks_ipv4.0
restricted = data.google_netblock_ip_ranges.restricted-googleapis.cidr_blocks_ipv4.0
}
vips = {
api = [for i in range(4) : cidrhost(local.netblocks.api, i)]
private = [for i in range(4) : cidrhost(local.netblocks.private, i)]
restricted = [for i in range(4) : cidrhost(local.netblocks.restricted, i)]
}
vm-startup-script = join("\n", [
"#! /bin/bash",
@ -30,12 +32,16 @@ locals {
])
}
data "google_netblock_ip_ranges" "dns-forwarders" {
range_type = "dns-forwarders"
}
data "google_netblock_ip_ranges" "private-googleapis" {
range_type = "private-googleapis"
}
data "google_netblock_ip_ranges" "dns-forwarders" {
range_type = "dns-forwarders"
data "google_netblock_ip_ranges" "restricted-googleapis" {
range_type = "restricted-googleapis"
}
################################################################################
@ -80,15 +86,16 @@ module "vpn" {
bgp_peer_options = {
advertise_groups = ["ALL_SUBNETS"]
advertise_ip_ranges = {
(local.netblocks.api) = "private-googleapis"
(local.netblocks.dns) = "dns-forwarders"
(local.netblocks.dns) = "DNS resolvers"
(local.netblocks.private) = "private.gooogleapis.com"
(local.netblocks.restricted) = "restricted.gooogleapis.com"
}
advertise_mode = "CUSTOM"
route_priority = 1000
}
bgp_session_range = "${local.bgp_interface_gcp}/30"
ike_version = 2
peer_ip = module.on-prem.external_address
peer_ip = module.vm-onprem.external_ips.0
shared_secret = ""
}
}
@ -112,7 +119,7 @@ module "dns-gcp" {
project_id = var.project_id
type = "private"
name = "gcp-example"
domain = "gcp.example.com."
domain = "gcp.example.org."
client_networks = [module.vpc.self_link]
recordsets = concat(
[{ name = "localhost", type = "A", ttl = 300, records = ["127.0.0.1"] }],
@ -131,12 +138,9 @@ module "dns-api" {
domain = "googleapis.com."
client_networks = [module.vpc.self_link]
recordsets = [
{
name = "*", type = "CNAME", ttl = 300, records = ["private.googleapis.com."]
},
{
name = "private", type = "A", ttl = 300, records = local.vips.api
},
{ name = "*", type = "CNAME", ttl = 300, records = ["private.googleapis.com."] },
{ name = "private", type = "A", ttl = 300, records = local.vips.private },
{ name = "restricted", type = "A", ttl = 300, records = local.vips.restricted },
]
}
@ -145,7 +149,7 @@ module "dns-onprem" {
project_id = var.project_id
type = "forwarding"
name = "onprem-example"
domain = "onprem.example.com."
domain = "onprem.example.org."
client_networks = [module.vpc.self_link]
forwarders = [cidrhost(var.ip_ranges.onprem, 3)]
}
@ -198,10 +202,21 @@ module "vm-test" {
# On prem #
################################################################################
data "template_file" "corefile" {
template = file("assets/Corefile")
vars = {
forwarder_address = var.forwarder_address
module "config-onprem" {
source = "../../modules/cloud-config-container/onprem"
config_variables = { dns_forwarder_address = var.dns_forwarder_address }
coredns_config = "assets/Corefile"
local_ip_cidr_range = var.ip_ranges.onprem
vpn_config = {
peer_ip = module.vpn.address
shared_secret = module.vpn.random_secret
type = "dynamic"
}
vpn_dynamic_config = {
local_bgp_asn = var.bgp_asn.onprem
local_bgp_address = local.bgp_interface_onprem
peer_bgp_asn = var.bgp_asn.gcp
peer_bgp_address = local.bgp_interface_gcp
}
}
@ -218,27 +233,28 @@ module "service-account-onprem" {
}
}
module "on-prem" {
source = "../../modules/on-prem-in-a-box/"
project_id = var.project_id
zone = "${var.region}-b"
network = module.vpc.name
subnet_self_link = module.vpc.subnet_self_links.default
local_ip_cidr_range = var.ip_ranges.onprem
coredns_config = data.template_file.corefile.rendered
vpn_config = {
peer_ip = module.vpn.address
shared_secret = module.vpn.random_secret
type = "dynamic"
module "vm-onprem" {
source = "../../modules/compute-vm"
project_id = var.project_id
region = var.region
zone = "${var.region}-b"
instance_type = "f1-micro"
name = "onprem"
boot_disk = {
image = "ubuntu-os-cloud/ubuntu-1804-lts"
type = "pd-ssd"
size = 10
}
vpn_dynamic_config = {
local_bgp_asn = var.bgp_asn.onprem
local_bgp_address = local.bgp_interface_onprem
peer_bgp_asn = var.bgp_asn.gcp
peer_bgp_address = local.bgp_interface_gcp
}
service_account = {
email = module.service-account-onprem.email
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
metadata = {
user-data = module.config-onprem.cloud_config
}
network_interfaces = [{
network = module.vpc.name
subnetwork = module.vpc.subnet_self_links.default
nat = true,
addresses = null
}]
service_account = module.service-account-onprem.email
service_account_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
tags = ["ssh"]
}

View File

@ -16,11 +16,11 @@
output "onprem-instance" {
description = "Onprem instance details."
value = join(" ", [
module.on-prem.instance_name,
module.on-prem.internal_address,
module.on-prem.external_address
])
value = {
external_ip = module.vm-onprem.external_ips.0
internal_ip = module.vm-onprem.internal_ips.0
name = module.vm-onprem.names.0
}
}
output "test-instance" {
@ -30,10 +30,3 @@ output "test-instance" {
module.vm-test.internal_ips[0]
])
}
output "foo" {
value = {
dns = data.google_netblock_ip_ranges.dns-forwarders.cidr_blocks_ipv4
apis = data.google_netblock_ip_ranges.private-googleapis.cidr_blocks_ipv4
}
}

View File

@ -31,6 +31,12 @@ variable "bgp_interface_ranges" {
}
}
variable "dns_forwarder_address" {
description = "Address of the DNS server used to forward queries from on-premises."
type = string
default = "10.0.0.2"
}
variable "ip_ranges" {
description = "IP CIDR ranges."
type = map(string)

View File

@ -31,7 +31,7 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google
## Compute/Container
- [COS container](./modules/compute-vm-cos-coredns)
- [COS container](./modules/cos-container) (coredns, mysql, onprem)
- [GKE cluster](./modules/gke-cluster)
- [GKE nodepool](./modules/gke-nodepool)
- [VM/VM group](./modules/compute-vm)
@ -41,7 +41,6 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google
- [BigQuery dataset](./modules/bigquery)
- [GCS](./modules/gcs)
## Other
## Security
- [Cloud KMS](./modules/kms)
- [on-premises in Docker](./modules/on-prem-in-a-box)

View File

@ -1,19 +1,20 @@
# Container Optimized OS modules
# Instance Configuration via `cloud-config`
This set of modules creates specialized [cloud-config](https://cloud.google.com/container-optimized-os/docs/how-to/run-container-instance#starting_a_docker_container_via_cloud-config) configurations for [Container Optimized OS](https://cloud.google.com/container-optimized-os/docs), that are used to quickly spin up containerized services for DNS, HTTP, or databases.
This set of modules creates specialized [cloud-config](https://cloud.google.com/container-optimized-os/docs/how-to/run-container-instance#starting_a_docker_container_via_cloud-config) configurations, which are designed for use with [Container Optimized OS](https://cloud.google.com/container-optimized-os/docs) (the [onprem module](./onprem/) is the only exception) but can also be used as a basis for other image types or cloud providers.
It's meant to fullfill different use cases:
These modules are designed for several use cases:
- when designing, to quickly prototype specialized services (eg MySQL access or HTTP serving)
- when planning migrations, to emulate production services for core infrastructure or perfomance testing
- in production, to easily add glue components for services like DNS (eg to work around inbound/outbound forwarding limitations)
- as a basis to implement cloud-native production deployments that leverage cloud-init for configuration management
- to quickly prototype specialized services (eg MySQL access or HTTP serving) for prototyping infrastructure
- to emulate production services for perfomance testing
- to easily add glue components for services like DNS (eg to work around inbound/outbound forwarding limitations)
- to implement cloud-native production deployments that leverage cloud-init for configuration management, without the need of a separate tool
## Available modules
- [CoreDNS](./coredns)
- [MySQL](./mysql)
- [ ] Nginx
- [Nginx](./nginx)
- [On-prem in Docker](./onprem)
- [ ] Squid forward proxy
## Using the modules
@ -23,3 +24,7 @@ All modules are designed to be as lightweight as possible, so that specialized m
To use the modules with instances or instance templates, simply set use their `cloud_config` output for the `user-data` metadata. When updating the metadata after a variable change remember to manually restart the instances that use a module's output, or the changes won't effect the running system.
For convenience when developing or prototyping infrastructure, an optional test instance is included in all modules. If it's not needed, the linked `*instance.tf` files can be removed from the modules without harm.
## TODO
- [ ] convert all `xxx_config` variables to use file content instead of path

View File

@ -63,11 +63,8 @@ module "cos-coredns" {
zone = "europe-west1-b"
name = "cos-coredns"
type = "f1-micro"
tags = ["ssh"]
metadata = {}
network = "default"
subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/my-subnet"
disks = []
}
}
```
@ -82,7 +79,8 @@ module "cos-coredns" {
| *coredns_config* | CoreDNS configuration path, if null default will be used. | <code title="">string</code> | | <code title="">null</code> |
| *file_defaults* | Default owner and permissions for files. | <code title="object&#40;&#123;&#10;owner &#61; string&#10;permissions &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;owner &#61; &#34;root&#34;&#10;permissions &#61; &#34;0644&#34;&#10;&#125;">...</code> |
| *files* | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | <code title="map&#40;object&#40;&#123;&#10;content &#61; string&#10;owner &#61; string&#10;permissions &#61; string&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *test_instance* | Test/development instance attributes, leave null to skip creation. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;zone &#61; string&#10;name &#61; string&#10;type &#61; string&#10;tags &#61; list&#40;string&#41;&#10;metadata &#61; map&#40;string&#41;&#10;network &#61; string&#10;subnetwork &#61; string&#10;disks &#61; map&#40;object&#40;&#123;&#10;read_only &#61; bool&#10;size &#61; number&#10;&#125;&#41;&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *test_instance* | Test/development instance attributes, leave null to skip creation. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;zone &#61; string&#10;name &#61; string&#10;type &#61; string&#10;network &#61; string&#10;subnetwork &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *test_instance_defaults* | Test/development instance defaults used for optional configuration. | <code title="object&#40;&#123;&#10;disks &#61; map&#40;object&#40;&#123;&#10;read_only &#61; bool&#10;size &#61; number&#10;&#125;&#41;&#41;&#10;metadata &#61; map&#40;string&#41;&#10;service_account_roles &#61; list&#40;string&#41;&#10;tags &#61; list&#40;string&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;disks &#61; &#123;&#125;&#10;metadata &#61; &#123;&#125;&#10;service_account_roles &#61; &#91;&#10;&#34;roles&#47;logging.logWriter&#34;,&#10;&#34;roles&#47;monitoring.metricWriter&#34;&#10;&#93;&#10;tags &#61; &#91;&#34;ssh&#34;&#93;&#10;&#125;">...</code> |
## Outputs

View File

@ -16,10 +16,10 @@
locals {
cloud_config = templatefile(local.template, merge(var.config_variables, {
corefile = local.corefile
corefile = templatefile(local.corefile, var.config_variables)
files = local.files
}))
corefile = file(
corefile = (
var.coredns_config == null ? "${path.module}/Corefile" : var.coredns_config
)
files = {

View File

@ -21,7 +21,7 @@ variable "cloud_config" {
}
variable "config_variables" {
description = "Additional variables used to render the cloud-config template."
description = "Additional variables used to render the cloud-config and CoreDNS templates."
type = map(any)
default = {}
}

View File

@ -14,32 +14,35 @@
* limitations under the License.
*/
locals {
disks = var.test_instance == null ? {} : var.test_instance.disks
sa_roles = ["roles/logging.logWriter", "roles/monitoring.metricWriter"]
}
resource "google_service_account" "default" {
count = var.test_instance == null ? 0 : 1
project = var.test_instance.project_id
account_id = "cos-test-${var.test_instance.name}"
account_id = "fabric-container-${var.test_instance.name}"
display_name = "Managed by the cos Terraform module."
}
resource "google_project_iam_member" "default" {
for_each = var.test_instance == null ? toset([]) : toset(local.sa_roles)
project = var.test_instance.project_id
role = each.value
member = "serviceAccount:${google_service_account.default[0].email}"
for_each = (
var.test_instance == null
? toset([])
: toset(var.test_instance_defaults.service_account_roles)
)
project = var.test_instance.project_id
role = each.value
member = "serviceAccount:${google_service_account.default[0].email}"
}
resource "google_compute_disk" "disks" {
for_each = local.disks
project = var.test_instance.project_id
zone = var.test_instance.zone
name = each.key
type = "pd-ssd"
size = each.value.size
for_each = (
var.test_instance == null
? {}
: var.test_instance_defaults.disks
)
project = var.test_instance.project_id
zone = var.test_instance.zone
name = each.key
type = "pd-ssd"
size = each.value.size
}
resource "google_compute_instance" "default" {
@ -48,16 +51,16 @@ resource "google_compute_instance" "default" {
zone = var.test_instance.zone
name = var.test_instance.name
description = "Managed by the cos Terraform module."
tags = var.test_instance.tags
tags = var.test_instance_defaults.tags
machine_type = (
var.test_instance.type == null ? "f1-micro" : var.test_instance.type
)
metadata = merge(var.test_instance.metadata, {
metadata = merge(var.test_instance_defaults.metadata, {
user-data = local.cloud_config
})
dynamic attached_disk {
for_each = local.disks
for_each = var.test_instance_defaults.disks
iterator = disk
content {
device_name = disk.key
@ -68,15 +71,26 @@ resource "google_compute_instance" "default" {
boot_disk {
initialize_params {
type = "pd-ssd"
image = "projects/cos-cloud/global/images/family/cos-stable"
size = 10
type = "pd-ssd"
image = (
var.test_instance_defaults.image == null
? "projects/cos-cloud/global/images/family/cos-stable"
: var.test_instance_defaults.image
)
size = 10
}
}
network_interface {
network = var.test_instance.network
subnetwork = var.test_instance.subnetwork
dynamic access_config {
for_each = var.test_instance_defaults.nat ? [""] : []
iterator = config
content {
nat_ip = null
}
}
}
service_account {

View File

@ -68,11 +68,8 @@ module "cos-mysql" {
zone = "europe-west1-b"
name = "cos-mysql"
type = "n1-standard-1"
tags = ["ssh"]
metadata = {}
network = "default"
subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/my-subnet"
disks = []
}
}
```
@ -89,7 +86,8 @@ module "cos-mysql" {
| *kms_config* | Optional KMS configuration to decrypt passed-in password. Leave null if a plaintext password is used. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;keyring &#61; string&#10;location &#61; string&#10;key &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *mysql_config* | MySQL configuration file content, if null container default will be used. | <code title="">string</code> | | <code title="">null</code> |
| *mysql_data_disk* | MySQL data disk name in /dev/disk/by-id/ including the google- prefix. If null the boot disk will be used for data. | <code title="">string</code> | | <code title="">null</code> |
| *test_instance* | Test/development instance attributes, leave null to skip creation. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;zone &#61; string&#10;name &#61; string&#10;type &#61; string&#10;tags &#61; list&#40;string&#41;&#10;metadata &#61; map&#40;string&#41;&#10;network &#61; string&#10;subnetwork &#61; string&#10;disks &#61; map&#40;object&#40;&#123;&#10;read_only &#61; bool&#10;size &#61; number&#10;&#125;&#41;&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *test_instance* | Test/development instance attributes, leave null to skip creation. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;zone &#61; string&#10;name &#61; string&#10;type &#61; string&#10;network &#61; string&#10;subnetwork &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *test_instance_defaults* | Test/development instance defaults used for optional configuration. | <code title="object&#40;&#123;&#10;disks &#61; map&#40;object&#40;&#123;&#10;read_only &#61; bool&#10;size &#61; number&#10;&#125;&#41;&#41;&#10;metadata &#61; map&#40;string&#41;&#10;service_account_roles &#61; list&#40;string&#41;&#10;tags &#61; list&#40;string&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;disks &#61; &#123;&#125;&#10;metadata &#61; &#123;&#125;&#10;service_account_roles &#61; &#91;&#10;&#34;roles&#47;logging.logWriter&#34;,&#10;&#34;roles&#47;monitoring.metricWriter&#34;&#10;&#93;&#10;tags &#61; &#91;&#34;ssh&#34;&#93;&#10;&#125;">...</code> |
## Outputs

View File

@ -0,0 +1,75 @@
# Containerized Nginx on Container Optimized OS
This module manages a `cloud-config` configuration that starts a containerized [Nginx](https://nginx.org/en/) service on Container Optimized OS, using the [hello demo image](https://hub.docker.com/r/nginxdemos/hello/).
The resulting `cloud-config` can be customized in a number of ways:
- a custom Nginx configuration can be set in `/etc/nginx/conf.d` using the `nginx_config` variable
- additional files (eg for hosts or zone files) can be passed in via the `files` variable
- a completely custom `cloud-config` can be passed in via the `cloud_config` variable, and additional template variables can be passed in via `config_variables`
The default instance configuration inserts iptables rules to allow traffic on port 80.
Logging and monitoring are enabled via the [Google Cloud Logging driver](https://docs.docker.com/config/containers/logging/gcplogs/) configured for the CoreDNS container, and the [Node Problem Detector](https://cloud.google.com/container-optimized-os/docs/how-to/monitoring) service started by default on boot.
The module renders the generated cloud config in the `cloud_config` output, to be used in instances or instance templates via the `user-data` metadata.
For convenience during development or for simple use cases, the module can optionally manage a single instance via the `test_instance` variable. If the instance is not needed the `instance*tf` files can be safely removed. Refer to the [top-level README](../README.md) for more details on the included instance.
## Examples
### Default configuration
This example will create a `cloud-config` that uses the module's defaults, creating a simple hello web server showing host name and request id.
```hcl
module "cos-nginx" {
source = "./modules/cos-container/nginx"
}
# use it as metadata in a compute instance or template
resource "google_compute_instance" "default" {
metadata = {
user-data = module.cos-nginx.cloud_config
}
```
### Nginx instance
This example shows how to create the single instance optionally managed by the module, providing all required attributes in the `test_instance` variable. The instance is purposefully kept simple and should only be used in development, or when designing infrastructures.
```hcl
module "cos-nginx" {
source = "./modules/cos-container/nginx"
test_instance = {
project_id = "my-project"
zone = "europe-west1-b"
name = "cos-nginx"
type = "f1-micro"
network = "default"
subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/my-subnet"
}
}
```
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
| *cloud_config* | Cloud config template path. If null default will be used. | <code title="">string</code> | | <code title="">null</code> |
| *config_variables* | Additional variables used to render the cloud-config and Nginx templates. | <code title="map&#40;any&#41;">map(any)</code> | | <code title="">{}</code> |
| *file_defaults* | Default owner and permissions for files. | <code title="object&#40;&#123;&#10;owner &#61; string&#10;permissions &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;owner &#61; &#34;root&#34;&#10;permissions &#61; &#34;0644&#34;&#10;&#125;">...</code> |
| *files* | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | <code title="map&#40;object&#40;&#123;&#10;content &#61; string&#10;owner &#61; string&#10;permissions &#61; string&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *image* | Nginx container image. | <code title="">string</code> | | <code title="">nginxdemos/hello:plain-text</code> |
| *nginx_config* | Nginx configuration path, if null container default will be used. | <code title="">string</code> | | <code title="">null</code> |
| *test_instance* | Test/development instance attributes, leave null to skip creation. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;zone &#61; string&#10;name &#61; string&#10;type &#61; string&#10;network &#61; string&#10;subnetwork &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *test_instance_defaults* | Test/development instance defaults used for optional configuration. If image is null, COS stable will be used. | <code title="object&#40;&#123;&#10;disks &#61; map&#40;object&#40;&#123;&#10;read_only &#61; bool&#10;size &#61; number&#10;&#125;&#41;&#41;&#10;image &#61; string&#10;metadata &#61; map&#40;string&#41;&#10;nat &#61; bool&#10;service_account_roles &#61; list&#40;string&#41;&#10;tags &#61; list&#40;string&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;disks &#61; &#123;&#125;&#10;image &#61; null&#10;metadata &#61; &#123;&#125;&#10;nat &#61; false&#10;service_account_roles &#61; &#91;&#10;&#34;roles&#47;logging.logWriter&#34;,&#10;&#34;roles&#47;monitoring.metricWriter&#34;&#10;&#93;&#10;tags &#61; &#91;&#34;ssh&#34;&#93;&#10;&#125;">...</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| cloud_config | Rendered cloud-config file to be passed as user-data instance metadata. | |
| test_instance | Optional test instance name and address | |
<!-- END TFDOC -->

View File

@ -0,0 +1,78 @@
#cloud-config
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://hub.docker.com/r/nginx/nginx/
# https://nginx.io/manual/toc/#installation
users:
- name: nginx
uid: 2000
write_files:
- path: /var/lib/docker/daemon.json
permissions: 0644
owner: root
content: |
{
"live-restore": true,
"storage-driver": "overlay2",
"log-opts": {
"max-size": "1024m"
}
}
%{~ if nginx_config != null ~}
- path: /etc/nginx/conf.d/nginx.conf
permissions: 0644
owner: root
content: |
${indent(6, nginx_config)}
%{~ endif ~}
# nginx container service
- path: /etc/systemd/system/nginx.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Start nginx container
After=gcr-online.target docker.socket
Wants=gcr-online.target docker.socket docker-events-collector.service
[Service]
ExecStart=/usr/bin/docker run --rm --name=nginx \
--log-driver=gcplogs --network host \
%{~ if etc_mount ~}
-v /etc/nginx/conf.d:/etc/nginx/conf.d \
%{~ endif ~}
${image}
ExecStop=/usr/bin/docker stop nginx
%{ for path, data in files }
- path: ${path}
owner: ${lookup(data, "owner", "root")}
permissions: ${lookup(data, "permissions", "0644")}
content: |
${indent(4, data.content)}
%{ endfor }
bootcmd:
- systemctl start node-problem-detector
runcmd:
- iptables -I INPUT 1 -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
- systemctl daemon-reload
- systemctl restart systemd-resolved.service
- systemctl start nginx

View File

@ -0,0 +1,101 @@
/**
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
resource "google_service_account" "default" {
count = var.test_instance == null ? 0 : 1
project = var.test_instance.project_id
account_id = "fabric-container-${var.test_instance.name}"
display_name = "Managed by the cos Terraform module."
}
resource "google_project_iam_member" "default" {
for_each = (
var.test_instance == null
? toset([])
: toset(var.test_instance_defaults.service_account_roles)
)
project = var.test_instance.project_id
role = each.value
member = "serviceAccount:${google_service_account.default[0].email}"
}
resource "google_compute_disk" "disks" {
for_each = (
var.test_instance == null
? {}
: var.test_instance_defaults.disks
)
project = var.test_instance.project_id
zone = var.test_instance.zone
name = each.key
type = "pd-ssd"
size = each.value.size
}
resource "google_compute_instance" "default" {
count = var.test_instance == null ? 0 : 1
project = var.test_instance.project_id
zone = var.test_instance.zone
name = var.test_instance.name
description = "Managed by the cos Terraform module."
tags = var.test_instance_defaults.tags
machine_type = (
var.test_instance.type == null ? "f1-micro" : var.test_instance.type
)
metadata = merge(var.test_instance_defaults.metadata, {
user-data = local.cloud_config
})
dynamic attached_disk {
for_each = var.test_instance_defaults.disks
iterator = disk
content {
device_name = disk.key
mode = disk.value.read_only ? "READ_ONLY" : "READ_WRITE"
source = google_compute_disk.disks[disk.key].name
}
}
boot_disk {
initialize_params {
type = "pd-ssd"
image = (
var.test_instance_defaults.image == null
? "projects/cos-cloud/global/images/family/cos-stable"
: var.test_instance_defaults.image
)
size = 10
}
}
network_interface {
network = var.test_instance.network
subnetwork = var.test_instance.subnetwork
dynamic access_config {
for_each = var.test_instance_defaults.nat ? [""] : []
iterator = config
content {
nat_ip = null
}
}
}
service_account {
email = google_service_account.default[0].email
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
}
}

View File

@ -0,0 +1,47 @@
/**
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
cloud_config = templatefile(local.template, merge(var.config_variables, {
etc_mount = (
var.nginx_config != null || length([
for name in keys(var.files) :
name if substr(name, 0, 18) == "/etc/nginx/conf.d/"
]) > 1
)
files = local.files
image = var.image
nginx_config = (var.nginx_config == null ? null : templatefile(
var.nginx_config, var.config_variables
))
}))
files = {
for path, attrs in var.files : path => {
content = attrs.content,
owner = attrs.owner == null ? var.file_defaults.owner : attrs.owner,
permissions = (
attrs.permissions == null
? var.file_defaults.permissions
: attrs.permissions
)
}
}
template = (
var.cloud_config == null
? "${path.module}/cloud-config.yaml"
: var.cloud_config
)
}

View File

@ -17,8 +17,12 @@
output "test_instance" {
description = "Optional test instance name and address"
value = (var.test_instance == null ? {} : {
address = google_compute_instance.default[0].network_interface.0.network_ip
name = google_compute_instance.default[0].name
address = google_compute_instance.default[0].network_interface.0.network_ip
name = google_compute_instance.default[0].name
nat_address = try(
google_compute_instance.default[0].network_interface.0.access_config.0.nat_ip,
null
)
service_account = google_service_account.default[0].email
})
}

View File

@ -0,0 +1,20 @@
/**
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "cloud_config" {
description = "Rendered cloud-config file to be passed as user-data instance metadata."
value = local.cloud_config
}

View File

@ -21,14 +21,34 @@ variable "test_instance" {
zone = string
name = string
type = string
tags = list(string)
metadata = map(string)
network = string
subnetwork = string
})
default = null
}
variable "test_instance_defaults" {
description = "Test/development instance defaults used for optional configuration. If image is null, COS stable will be used."
type = object({
disks = map(object({
read_only = bool
size = number
}))
image = string
metadata = map(string)
nat = bool
service_account_roles = list(string)
tags = list(string)
})
default = null
default = {
disks = {}
image = null
metadata = {}
nat = false
service_account_roles = [
"roles/logging.logWriter",
"roles/monitoring.metricWriter"
]
tags = ["ssh"]
}
}

View File

@ -0,0 +1,61 @@
/**
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "cloud_config" {
description = "Cloud config template path. If null default will be used."
type = string
default = null
}
variable "config_variables" {
description = "Additional variables used to render the cloud-config and Nginx templates."
type = map(any)
default = {}
}
variable "image" {
description = "Nginx container image."
type = string
default = "nginxdemos/hello:plain-text"
}
variable "nginx_config" {
description = "Nginx configuration path, if null container default will be used."
type = string
default = null
}
variable "file_defaults" {
description = "Default owner and permissions for files."
type = object({
owner = string
permissions = string
})
default = {
owner = "root"
permissions = "0644"
}
}
variable "files" {
description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null."
type = map(object({
content = string
owner = string
permissions = string
}))
default = {}
}

View File

@ -0,0 +1,9 @@
. {
hosts /etc/coredns/onprem.hosts onprem.example.org {
127.0.0.1 localhost.example.org localhost
}
forward . /etc/resolv.conf
reload
log
errors
}

View File

@ -0,0 +1,81 @@
# Containerized on-premises infrastructure
This module manages a `cloud-config` configuration that starts an emulated on-premises infrastructure running in Docker Compose on a single instance, and connects it via static or dynamic VPN to a Google Cloud VPN gateway.
The emulated on-premises infrastructure is composed of:
- a Strongswan container managing the VPN tunnel to GCP
- an optional Bird container managing the BGP session
- a CoreDNS container servng local DNS and forwarding to GCP
- an Nginx container serving a simple static web page
- a generic Linux container used as a jump host inside the on-premises network
A [complete scenario using this module](../../../infrastructure/onprem-google-access-dns) is available in the infrastructure examples.
The module renders the generated cloud config in the `cloud_config` output, to be used in instances or instance templates via the `user-data` metadata.
For convenience during development or for simple use cases, the module can optionally manage a single instance via the `test_instance` variable. If the instance is not needed the `instance*tf` files can be safely removed. Refer to the [top-level README](../README.md) for more details on the included instance.
## Examples
### Static VPN
The test instance is optional, as described above.
```hcl
module "cloud-vpn" {
source = "./modules/net-vpn-static"
project_id = "my-project"
region = "europe-west1"
network = "my-vpc"
name = "to-on-prem"
remote_ranges = ["192.168.192.0/24"]
tunnels = {
remote-0 = {
ike_version = 2
peer_ip = module.on-prem.external_address
shared_secret = ""
traffic_selectors = { local = ["0.0.0.0/0"], remote = null }
}
}
}
module "on-prem" {
source = "./modules/cos-container/on-prem"
name = "onprem"
vpn_config = {
type = "static"
peer_ip = module.cloud-vpn.address
shared_secret = module.cloud-vpn.random_secret
}
test_instance = {
project_id = "my-project"
zone = "europe-west1-b"
name = "cos-coredns"
type = "f1-micro"
network = "default"
subnetwork = "https://www.googleapis.com/compute/v1/projects/my-project/regions/europe-west1/subnetworks/my-subnet"
}
}
```
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
| vpn_config | VPN configuration, type must be one of 'dynamic' or 'static'. | <code title="object&#40;&#123;&#10;peer_ip &#61; string&#10;shared_secret &#61; string&#10;type &#61; string&#10;&#125;&#41;">object({...})</code> | ✓ | |
| *coredns_config* | CoreDNS configuration path, if null default will be used. | <code title="">string</code> | | <code title="">null</code> |
| *local_ip_cidr_range* | IP CIDR range used for the Docker onprem network. | <code title="">string</code> | | <code title="">192.168.192.0/24</code> |
| *test_instance* | Test/development instance attributes, leave null to skip creation. | <code title="object&#40;&#123;&#10;project_id &#61; string&#10;zone &#61; string&#10;name &#61; string&#10;type &#61; string&#10;network &#61; string&#10;subnetwork &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="">null</code> |
| *test_instance_defaults* | Test/development instance defaults used for optional configuration. | <code title="object&#40;&#123;&#10;disks &#61; map&#40;object&#40;&#123;&#10;read_only &#61; bool&#10;size &#61; number&#10;&#125;&#41;&#41;&#10;metadata &#61; map&#40;string&#41;&#10;service_account_roles &#61; list&#40;string&#41;&#10;tags &#61; list&#40;string&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;disks &#61; &#123;&#125;&#10;metadata &#61; &#123;&#125;&#10;service_account_roles &#61; &#91;&#10;&#34;roles&#47;logging.logWriter&#34;,&#10;&#34;roles&#47;monitoring.metricWriter&#34;&#10;&#93;&#10;tags &#61; &#91;&#34;ssh&#34;&#93;&#10;&#125;">...</code> |
| *vpn_dynamic_config* | BGP configuration for dynamic VPN, ignored if VPN type is 'static'. | <code title="object&#40;&#123;&#10;local_bgp_asn &#61; number&#10;local_bgp_address &#61; string&#10;peer_bgp_asn &#61; number&#10;peer_bgp_address &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;local_bgp_asn &#61; 65002&#10;local_bgp_address &#61; &#34;169.254.0.2&#34;&#10;peer_bgp_asn &#61; 65001&#10;peer_bgp_address &#61; &#34;169.254.0.1&#34;&#10;&#125;">...</code> |
| *vpn_static_ranges* | Remote CIDR ranges for static VPN, ignored if VPN type is 'dynamic'. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">["10.0.0.0/8"]</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| cloud_config | Rendered cloud-config file to be passed as user-data instance metadata. | |
| test_instance | Optional test instance name and address | |
<!-- END TFDOC -->

View File

@ -65,11 +65,13 @@ write_files:
image: gcr.io/pso-cft-fabric/strongswan:latest
networks:
onprem:
ipv4_address: ${vpn_ip_address}
ipv4_address: ${local_addresses.vpn}
ports:
- "500:500/udp"
- "4500:4500/udp"
%{~ if vpn_config.type == "dynamic" ~}
- "179:179/tcp"
%{~ endif ~}
privileged: true
cap_add:
- NET_ADMIN
@ -78,9 +80,12 @@ write_files:
- "/etc/localtime:/etc/localtime:ro"
- "/var/lib/docker-compose/onprem/ipsec/ipsec.conf:/etc/ipsec.conf:ro"
- "/var/lib/docker-compose/onprem/ipsec/ipsec.secrets:/etc/ipsec.secrets:ro"
%{~ if vpn_config.type == "dynamic" ~}
- "/var/lib/docker-compose/onprem/ipsec/vti.conf:/etc/strongswan.d/vti.conf:ro"
%{~ endif ~}
environment:
- LAN_NETWORKS=${local_ip_cidr_range}
- LAN_NETWORKS=${ip_cidr_ranges.local}
%{~ if vpn_config.type == "dynamic" ~}
bird:
image: pierky/bird
network_mode: service:vpn
@ -91,15 +96,18 @@ write_files:
privileged: true
volumes:
- "/var/lib/docker-compose/onprem/bird/bird.conf:/etc/bird/bird.conf:ro"
%{~ endif ~}
dns:
image: coredns/coredns
command: "-conf /etc/coredns/Corefile"
depends_on:
- "vpn"
%{~ if vpn_config.type == "dynamic" ~}
- "bird"
%{~ endif ~}
networks:
onprem:
ipv4_address: ${dns_ip_address}
ipv4_address: ${local_addresses.dns}
volumes:
- "/var/lib/docker-compose/onprem/coredns:/etc/coredns:ro"
routing_sidecar_dns:
@ -108,19 +116,21 @@ write_files:
command: |
/bin/sh -c "\
ip route del default &&\
ip route add default via ${vpn_ip_address}"
ip route add default via ${local_addresses.vpn}"
privileged: true
web:
image: nginx:stable-alpine
depends_on:
- "vpn"
%{~ if vpn_config.type == "dynamic" ~}
- "bird"
%{~ endif ~}
- "dns"
dns:
- ${dns_ip_address}
- ${local_addresses.dns}
networks:
onprem:
ipv4_address: ${web_ip_address}
ipv4_address: ${local_addresses.www}
volumes:
- "/var/lib/docker-compose/onprem/nginx:/usr/share/nginx/html:ro"
routing_sidecar_web:
@ -129,40 +139,40 @@ write_files:
command: |
/bin/sh -c "\
ip route del default &&\
ip route add default via ${vpn_ip_address}"
ip route add default via ${local_addresses.vpn}"
privileged: true
toolbox:
image: gcr.io/pso-cft-fabric/toolbox:latest
networks:
onprem:
ipv4_address: ${toolbox_ip_address}
ipv4_address: ${local_addresses.shell}
depends_on:
- "vpn"
- "dns"
- "web"
dns:
- ${dns_ip_address}
- ${local_addresses.dns}
routing_sidecar_toolbox:
image: alpine
network_mode: service:toolbox
command: |
/bin/sh -c "\
ip route del default &&\
ip route add default via ${vpn_ip_address}"
ip route add default via ${local_addresses.vpn}"
privileged: true
networks:
onprem:
ipam:
driver: default
config:
- subnet: ${local_ip_cidr_range}
- subnet: ${ip_cidr_ranges.local}
# IPSEC tunnel secret
- path: /var/lib/docker-compose/onprem/ipsec/ipsec.secrets
owner: root:root
permissions: '0600'
content: |
: PSK "${shared_secret}"
: PSK "${vpn_config.shared_secret}"
# IPSEC tunnel configuration
- path: /var/lib/docker-compose/onprem/ipsec/ipsec.conf
@ -181,19 +191,30 @@ write_files:
authby=psk
conn gcp
leftupdown="/var/lib/strongswan/ipsec-vti.sh 0 ${peer_bgp_address}/30 ${local_bgp_address}/30"
%{~ if vpn_config.type == "dynamic" ~}
leftupdown="/var/lib/strongswan/ipsec-vti.sh 0 ${vpn_dynamic_config.peer_bgp_address}/30 ${vpn_dynamic_config.local_bgp_address}/30"
%{~ endif ~}
left=%any
leftid=%any
%{~ if vpn_config.type == "dynamic" ~}
leftsubnet=0.0.0.0/0
%{~ else ~}
leftsubnet=${ip_cidr_ranges.local}
%{~ endif ~}
leftauth=psk
right=${peer_ip_wildcard}
rightid=${peer_ip}
right=${vpn_config.peer_ip_wildcard}
rightid=${vpn_config.peer_ip}
%{~ if vpn_config.type == "dynamic" ~}
rightsubnet=0.0.0.0/0
%{~ else ~}
rightsubnet=${ip_cidr_ranges.remote}
%{~ endif ~}
rightauth=psk
type=tunnel
auto=start
dpdaction=restart
closeaction=restart
%{~ if vpn_config.type == "dynamic" ~}
mark=%unique
# Charon configuration
@ -210,70 +231,70 @@ write_files:
owner: root:root
permissions: '0644'
content: |
router id ${local_bgp_address};
router id ${vpn_dynamic_config.local_bgp_address};
# Watch interface up/down events
# watch interface up/down events
protocol device {
scan time 10;
scan time 10;
}
# Sync routes to kernel
# sync routes to kernel
protocol kernel {
learn;
merge paths on; # For ECMP
export filter {
krt_prefsrc = ${vpn_ip_address}; # Internal IP Address of the strongSwan VM.
accept; # Sync all routes to kernel
};
import all; # Required due to /32 on GCE VMs for the static route below
learn;
merge paths on; # For ECMP
export filter {
# internal IP of the strongswan VM
krt_prefsrc = ${local_addresses.vpn};
# sync all routes to kernel
accept;
};
import all; # Required due to /32 on GCE VMs for the static route below
}
# Configure a static route to make sure route exists
protocol static {
# Network connected to eth0
route ${local_ip_cidr_range} recursive ${local_gw_ip};
# Private google access
route 199.36.153.4/30 via ${peer_bgp_address};
# Cloud DNS forwarding zone
route 35.199.192.0/19 via ${peer_bgp_address};
# network connected to eth0
route ${ip_cidr_ranges.local} recursive ${local_addresses.gw};
%{~ for range in netblocks ~}
# route ${range} via ${vpn_dynamic_config.peer_bgp_address};
%{~ endfor ~}
}
# Prefix lists for routing security
# Allow any possible GCP Subnet
define GCP_VPC_A_PREFIXES = [ 10.0.0.0/8{9,29}, 172.16.0.0/12{12,29}, 192.168.0.0/16{16,29} ];
define LOCAL_PREFIXES = [ ${local_ip_cidr_range} ];
# prefix lists for routing security
# allow any possible GCP Subnet
define GCP_VPC_A_PREFIXES = [ 10.0.0.0/8{8,29}, 172.16.0.0/12{12,29}, 192.168.0.0/16{16,29} ];
define GCP_NETBLOCKS = [ ${join(", ", netblocks)} ];
define LOCAL_PREFIXES = [ ${ip_cidr_ranges.local} ];
# Filter received prefixes
filter gcp_vpc_a_in
{
if (net ~ GCP_VPC_A_PREFIXES) then accept;
else reject;
# filter received prefixes
filter gcp_vpc_a_in {
if (net ~ GCP_VPC_A_PREFIXES || net ~ GCP_NETBLOCKS) then accept;
else reject;
}
# Filter advertised prefixes
filter gcp_vpc_a_out
{
if (net ~ LOCAL_PREFIXES) then accept;
else reject;
# filter advertised prefixes
filter gcp_vpc_a_out {
if (net ~ LOCAL_PREFIXES) then accept;
else reject;
}
template bgp gcp_vpc_a {
keepalive time 20;
hold time 60;
graceful restart aware; # Cloud Router uses GR during maintenance
import filter gcp_vpc_a_in;
import limit 10 action warn; # restart | block | disable
export filter gcp_vpc_a_out;
export limit 10 action warn; # restart | block | disable
keepalive time 20;
hold time 60;
# Cloud Router uses GR during maintenance
graceful restart aware;
import filter gcp_vpc_a_in;
import limit 10 action warn; # restart | block | disable
export filter gcp_vpc_a_out;
export limit 10 action warn; # restart | block | disable
}
protocol bgp gcp_vpc_a_tun1 from gcp_vpc_a
{
local ${local_bgp_address} as ${local_bgp_asn};
neighbor ${peer_bgp_address} as ${peer_bgp_asn};
protocol bgp gcp_vpc_a_tun1 from gcp_vpc_a {
local ${vpn_dynamic_config.local_bgp_address} as ${vpn_dynamic_config.local_bgp_asn};
neighbor ${vpn_dynamic_config.peer_bgp_address} as ${vpn_dynamic_config.peer_bgp_asn};
}
%{~ endif ~}
# CoreDNS configuration
- path: /var/lib/docker-compose/onprem/coredns/Corefile
owner: root:root
@ -286,10 +307,9 @@ write_files:
owner: root:root
permissions: '0644'
content: |
${vpn_ip_address} gw.${dns_domain}
${dns_ip_address} ns.${dns_domain}
${web_ip_address} www.${dns_domain}
${toolbox_ip_address} toolbox.${dns_domain}
%{~ for name, address in local_addresses ~}
${address} ${name}.onprem.example.org
%{~ endfor ~}
# Minimal nginx index page
- path: /var/lib/docker-compose/onprem/nginx/index.html
@ -301,7 +321,7 @@ write_files:
<head><meta charset="utf-8"></head>
<body>
<h1>On Prem in a Box</h1>
<p>${instance_name}</p>
<p>onprem</p>
</body>
</html>

View File

@ -0,0 +1 @@
../instance.tf

View File

@ -0,0 +1,66 @@
/**
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
cloud_config = templatefile(
"${path.module}/cloud-config.yaml",
merge(local.cloud_config_vars, var.config_variables)
)
corefile = (
var.coredns_config == null ?
"${path.module}/Corefile"
: var.coredns_config
)
cloud_config_vars = {
coredns_config = indent(4, templatefile(local.corefile, var.config_variables))
ip_cidr_ranges = {
local = var.local_ip_cidr_range
remote = join(",", concat(
var.vpn_static_ranges, local.netblocks
))
}
local_addresses = {
gw = cidrhost(var.local_ip_cidr_range, 1)
vpn = cidrhost(var.local_ip_cidr_range, 2)
dns = cidrhost(var.local_ip_cidr_range, 3)
www = cidrhost(var.local_ip_cidr_range, 4)
shell = cidrhost(var.local_ip_cidr_range, 5)
}
netblocks = local.netblocks
vpn_config = local.vpn_config
vpn_dynamic_config = var.vpn_dynamic_config
}
netblocks = concat(
data.google_netblock_ip_ranges.dns-forwarders.cidr_blocks_ipv4,
data.google_netblock_ip_ranges.private-googleapis.cidr_blocks_ipv4,
data.google_netblock_ip_ranges.restricted-googleapis.cidr_blocks_ipv4
)
vpn_config = merge(var.vpn_config, {
peer_ip_wildcard = "%${var.vpn_config.peer_ip}"
})
}
data "google_netblock_ip_ranges" "dns-forwarders" {
range_type = "dns-forwarders"
}
data "google_netblock_ip_ranges" "private-googleapis" {
range_type = "private-googleapis"
}
data "google_netblock_ip_ranges" "restricted-googleapis" {
range_type = "restricted-googleapis"
}

View File

@ -0,0 +1 @@
../outputs-instance.tf

View File

@ -0,0 +1,20 @@
/**
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "cloud_config" {
description = "Rendered cloud-config file to be passed as user-data instance metadata."
value = local.cloud_config
}

View File

@ -0,0 +1 @@
../variables-instance.tf

View File

@ -14,16 +14,16 @@
* limitations under the License.
*/
variable "coredns_config" {
description = "CoreDNS configuration, set to null to use default."
type = string
default = null
variable "config_variables" {
description = "Additional variables used to render the cloud-config and CoreDNS templates."
type = map(any)
default = {}
}
variable "dns_domain" {
description = "DNS domain used for on-prem host records."
variable "coredns_config" {
description = "CoreDNS configuration path, if null default will be used."
type = string
default = "onprem.example.com"
default = null
}
variable "local_ip_cidr_range" {
@ -32,55 +32,6 @@ variable "local_ip_cidr_range" {
default = "192.168.192.0/24"
}
variable "machine_type" {
description = "Machine type."
type = string
default = "g1-small"
}
variable "name" {
description = "On-prem-in-a-box compute instance name."
type = string
default = "onprem"
}
variable "network" {
description = "VPC network name."
type = string
}
variable "network_tags" {
description = "Network tags."
type = list(string)
default = ["ssh"]
}
variable "project_id" {
description = "Project id."
type = string
}
variable "service_account" {
description = "Service account customization."
type = object({
email = string
scopes = list(string)
})
default = {
email = null
scopes = [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write"
]
}
}
variable "subnet_self_link" {
description = "VPC subnet self link."
type = string
}
variable "vpn_config" {
description = "VPN configuration, type must be one of 'dynamic' or 'static'."
type = object({
@ -109,10 +60,5 @@ variable "vpn_dynamic_config" {
variable "vpn_static_ranges" {
description = "Remote CIDR ranges for static VPN, ignored if VPN type is 'dynamic'."
type = list(string)
default = []
}
variable "zone" {
description = "Compute zone."
type = string
default = ["10.0.0.0/8"]
}

View File

@ -0,0 +1,28 @@
/**
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "test_instance" {
description = "Optional test instance name and address"
value = (var.test_instance == null ? {} : {
address = google_compute_instance.default[0].network_interface.0.network_ip
name = google_compute_instance.default[0].name
nat_address = try(
google_compute_instance.default[0].network_interface.0.access_config.0.nat_ip,
null
)
service_account = google_service_account.default[0].email
})
}

View File

@ -0,0 +1,54 @@
/**
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "test_instance" {
description = "Test/development instance attributes, leave null to skip creation."
type = object({
project_id = string
zone = string
name = string
type = string
network = string
subnetwork = string
})
default = null
}
variable "test_instance_defaults" {
description = "Test/development instance defaults used for optional configuration. If image is null, COS stable will be used."
type = object({
disks = map(object({
read_only = bool
size = number
}))
image = string
metadata = map(string)
nat = bool
service_account_roles = list(string)
tags = list(string)
})
default = {
disks = {}
image = null
metadata = {}
nat = false
service_account_roles = [
"roles/logging.logWriter",
"roles/monitoring.metricWriter"
]
tags = ["ssh"]
}
}

View File

@ -20,7 +20,7 @@ output "external_ips" {
var.network_interfaces[0].nat
? [
for name, instance in google_compute_instance.default :
instance.network_interface.0.network_ip
try(instance.network_interface.0.access_config.0.nat_ip, null)
]
: []
)

View File

@ -31,7 +31,7 @@ module "folders-unit" {
| automation_project_id | Project id used for automation service accounts. | <code title="">string</code> | ✓ | |
| billing_account_id | Country billing account account. | <code title="">string</code> | ✓ | |
| name | Top folder name. | <code title="">string</code> | ✓ | |
| organization_id | Organization id. | <code title="">string</code> | ✓ | |
| organization_id | Organization id in organizations/nnnnnn format. | <code title="">string</code> | ✓ | |
| root_node | Root node in folders/folder_id or organizations/org_id format. | <code title="">string</code> | ✓ | |
| short_name | Short name used as GCS bucket and service account prefixes, do not use capital letters or spaces. | <code title="">string</code> | ✓ | |
| *environments* | Unit environments short names. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="&#123;&#10;non-prod &#61; &#34;Non production&#34;&#10;prod &#61; &#34;Production&#34;&#10;&#125;">...</code> |

View File

@ -1,131 +0,0 @@
# On-prem-in-a-box Module
This module allows emulating an on-premise enviroment in a single GCE VM, by connecting a Docker Network to a VPC via a static or dynamic (BGP) VPN connection implemented with Strongswan. It provides a good playground for testing private access and hybrid DNS connectivity between on-premise and Google Cloud.
To see this module in action, please refer to the folowing end-to-end network examples:
- [hub-and-spoke-peerings](../../infrastructure/hub-and-spoke-peerings/)
## TODO
- [ ] describe how to check and troubleshoot the onprem VPN and services
- [ ] add support for service account, scopes and network tags
- [ ] allow passing in arbitrary CoreDNS configurations instead of tweaking a default one via variables
## Examples
### Static VPN Gateway
```hcl
module "cloud-vpn" {
source = "modules/net-vpn-static/"
project_id = "<PROJECT_ID>"
region = "europe-west4"
network = "vpn-network"
name = "cloud-net-to-on-prem"
remote_ranges = ["192.168.192.0/24"]
tunnels = {
remote-0 = {
ike_version = 2
peer_ip = module.on-prem.external_address
shared_secret = ""
traffic_selectors = { local = ["0.0.0.0/0"], remote = null }
}
}
}
module "on-prem" {
source = "modules/on-prem-in-a-box/"
name = "onprem-instance"
project_id = "<PROJECT_ID>"
zone = "europe-west4-b"
network = <NETWORK_NAME>
subnet_self_link = "https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/regions/europe-west4/subnetworks/<SUBNETWORK_NAME>"
vpn_gateway_type = "static"
peer_ip = module.cloud-vpn.address
local_ip_cidr_range = "192.168.192.0/24"
shared_secret = module.cloud-vpn.random_secret
remote_ip_cidr_ranges = "172.16.0.0/24,172.16.1.0/24,172.16.2.0/24"
}
```
### Dynamic VPN Gateway
```hcl
module "cloud-vpn" {
source = "modules/net-vpn-dynamic/"
project_id = "<PROJECT_ID>"
region = "europe-west4"
network = "vpn-network"
name = "cloud-net-to-on-prem"
router_asn = 65001
tunnels = {
remote-1 = {
bgp_peer = {
address = "169.254.0.2"
asn = 65002
}
bgp_session_range = "169.254.0.1/30"
ike_version = 2
peer_ip = module.on-prem.external_address
shared_secret = null
bgp_peer_options = {
advertise_groups = ["ALL_SUBNETS"]
advertise_ip_ranges = {
}
advertise_mode = "DEFAULT"
route_priority = 1000
}
}
}
}
module "on-prem" {
source = "modules/on-prem-in-a-box/"
name = "onprem-instance"
project_id = "<PROJECT_ID>"
zone = "europe-west4-b"
network = "<NETWORK_NAME>"
subnet_self_link = "https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/regions/europe-west4/subnetworks/<SUBNETWORK_NAME>"
vpn_gateway_type = "dynamic"
peer_ip = module.cloud-vpn.address
local_ip_cidr_range = "192.168.192.0/24"
shared_secret = module.cloud-vpn.random_secret
peer_bgp_session_range = "169.254.0.1/30"
local_bgp_session_range = "169.254.0.2/30"
peer_bgp_asn = 65001
local_bgp_asn = 65002
}
```
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
| network | VPC network name. | <code title="">string</code> | ✓ | |
| project_id | Project id. | <code title="">string</code> | ✓ | |
| subnet_self_link | VPC subnet self link. | <code title="">string</code> | ✓ | |
| vpn_config | VPN configuration, type must be one of 'dynamic' or 'static'. | <code title="object&#40;&#123;&#10;peer_ip &#61; string&#10;shared_secret &#61; string&#10;type &#61; string&#10;&#125;&#41;">object({...})</code> | ✓ | |
| zone | Compute zone. | <code title="">string</code> | ✓ | |
| *coredns_config* | CoreDNS configuration, set to null to use default. | <code title="">string</code> | | <code title="">null</code> |
| *dns_domain* | DNS domain used for on-prem host records. | <code title="">string</code> | | <code title="">onprem.example.com</code> |
| *local_ip_cidr_range* | IP CIDR range used for the Docker onprem network. | <code title="">string</code> | | <code title="">192.168.192.0/24</code> |
| *machine_type* | Machine type. | <code title="">string</code> | | <code title="">g1-small</code> |
| *name* | On-prem-in-a-box compute instance name. | <code title="">string</code> | | <code title="">onprem</code> |
| *network_tags* | Network tags. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">["ssh"]</code> |
| *service_account* | Service account customization. | <code title="object&#40;&#123;&#10;email &#61; string&#10;scopes &#61; list&#40;string&#41;&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;email &#61; null&#10;scopes &#61; &#91;&#10;&#34;https:&#47;&#47;www.googleapis.com&#47;auth&#47;devstorage.read_only&#34;,&#10;&#34;https:&#47;&#47;www.googleapis.com&#47;auth&#47;logging.write&#34;,&#10;&#34;https:&#47;&#47;www.googleapis.com&#47;auth&#47;monitoring.write&#34;&#10;&#93;&#10;&#125;">...</code> |
| *vpn_dynamic_config* | BGP configuration for dynamic VPN, ignored if VPN type is 'static'. | <code title="object&#40;&#123;&#10;local_bgp_asn &#61; number&#10;local_bgp_address &#61; string&#10;peer_bgp_asn &#61; number&#10;peer_bgp_address &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;local_bgp_asn &#61; 65002&#10;local_bgp_address &#61; &#34;169.254.0.2&#34;&#10;peer_bgp_asn &#61; 65001&#10;peer_bgp_address &#61; &#34;169.254.0.1&#34;&#10;&#125;">...</code> |
| *vpn_static_ranges* | Remote CIDR ranges for static VPN, ignored if VPN type is 'dynamic'. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">[]</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| dns_ip_address | None | |
| external_address | None | |
| instance_name | None | |
| internal_address | None | |
| toolbox_ip_address | None | |
| vpn_ip_address | None | |
| web_ip_address | None | |
<!-- END TFDOC -->

View File

@ -1,11 +0,0 @@
${dns_domain} {
root /etc/coredns
hosts onprem.hosts
log
errors
}
. {
forward . 8.8.8.8
log
errors
}

View File

@ -1,130 +0,0 @@
/**
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
corefile = (
var.coredns_config == null || var.coredns_config == ""
? data.template_file.corefile.rendered
: var.coredns_config
)
}
resource "google_compute_address" "static" {
project = var.project_id
name = var.name
region = substr(var.zone, 0, length(var.zone) - 2)
address_type = "EXTERNAL"
}
resource "google_compute_instance" "on_prem_in_a_box" {
project = var.project_id
name = var.name
machine_type = var.machine_type
zone = var.zone
tags = concat(var.network_tags, ["onprem"])
boot_disk {
initialize_params {
image = "ubuntu-os-cloud/ubuntu-1804-lts"
}
}
network_interface {
subnetwork = var.subnet_self_link
access_config {
nat_ip = google_compute_address.static.address
}
}
metadata = {
user-data = data.template_file.vpn-gw.rendered
}
service_account {
email = var.service_account.email
scopes = var.service_account.scopes
}
}
data "template_file" "corefile" {
template = file("${path.module}/assets/Corefile")
vars = {
dns_domain = var.dns_domain
}
}
data "template_file" "vpn-gw" {
template = file(format(
"%s/assets/%s-vpn-gw-cloud-init.yaml", path.module, var.vpn_config.type
))
vars = {
coredns_config = indent(4, local.corefile)
dns_domain = var.dns_domain
instance_name = var.name
local_ip_cidr_range = var.local_ip_cidr_range
local_gw_ip = cidrhost(var.local_ip_cidr_range, 1)
vpn_ip_address = cidrhost(var.local_ip_cidr_range, 2)
dns_ip_address = cidrhost(var.local_ip_cidr_range, 3)
web_ip_address = cidrhost(var.local_ip_cidr_range, 4)
toolbox_ip_address = cidrhost(var.local_ip_cidr_range, 5)
# vpn config
peer_ip = var.vpn_config.peer_ip
peer_ip_wildcard = "%${var.vpn_config.peer_ip}"
shared_secret = var.vpn_config.shared_secret
# vpn dynamic config
local_bgp_asn = var.vpn_dynamic_config.local_bgp_asn
local_bgp_address = var.vpn_dynamic_config.local_bgp_address
peer_bgp_asn = var.vpn_dynamic_config.peer_bgp_asn
peer_bgp_address = var.vpn_dynamic_config.peer_bgp_address
# vpn static ranges
vpn_static_ranges = join(",", var.vpn_static_ranges)
}
}
# TODO: use a narrower firewall rule and tie it to the service account
resource "google_compute_firewall" "allow-vpn" {
name = "onprem-in-a-box-allow-vpn"
description = "Allow VPN traffic to the onprem instance"
network = var.network
project = var.project_id
source_ranges = [format("%s/32", var.vpn_config.peer_ip)]
target_tags = ["onprem"]
allow {
protocol = "tcp"
}
allow {
protocol = "udp"
}
allow {
protocol = "icmp"
}
}
resource "google_compute_firewall" "allow-iap" {
name = "onprem-in-a-box-allow-iap"
description = "Allow SSH traffic to the onprem instance from IAP"
network = var.network
project = var.project_id
source_ranges = ["35.235.240.0/20"]
target_tags = ["onprem"]
allow {
protocol = "tcp"
ports = ["22"]
}
}

View File

@ -1,43 +0,0 @@
/**
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "external_address" {
value = google_compute_instance.on_prem_in_a_box.network_interface.0.access_config.0.nat_ip
}
output "internal_address" {
value = google_compute_instance.on_prem_in_a_box.network_interface.0.network_ip
}
output "instance_name" {
value = google_compute_instance.on_prem_in_a_box.name
}
output "vpn_ip_address" {
value = cidrhost(var.local_ip_cidr_range, 2)
}
output "dns_ip_address" {
value = cidrhost(var.local_ip_cidr_range, 3)
}
output "web_ip_address" {
value = cidrhost(var.local_ip_cidr_range, 4)
}
output "toolbox_ip_address" {
value = cidrhost(var.local_ip_cidr_range, 5)
}

View File

@ -15,7 +15,7 @@
*/
module "test" {
source = "../../../../modules/cos-container/coredns"
source = "../../../../modules/cloud-config-container/coredns"
cloud_config = var.cloud_config
config_variables = var.config_variables
coredns_config = var.coredns_config

View File

@ -15,7 +15,7 @@
*/
module "test" {
source = "../../../../modules/cos-container/mysql"
source = "../../../../modules/cloud-config-container/mysql"
cloud_config = var.cloud_config
config_variables = var.config_variables
image = var.image