GKE multitenant
Co-authored-by: Daniel Marzini <dmarzi@google.com>
This commit is contained in:
parent
75abd80d9f
commit
f3f9a4a88c
|
@ -60,6 +60,7 @@ module "automation-project" {
|
||||||
"cloudresourcemanager.googleapis.com",
|
"cloudresourcemanager.googleapis.com",
|
||||||
"container.googleapis.com",
|
"container.googleapis.com",
|
||||||
"compute.googleapis.com",
|
"compute.googleapis.com",
|
||||||
|
"container.googleapis.com",
|
||||||
"essentialcontacts.googleapis.com",
|
"essentialcontacts.googleapis.com",
|
||||||
"iam.googleapis.com",
|
"iam.googleapis.com",
|
||||||
"iamcredentials.googleapis.com",
|
"iamcredentials.googleapis.com",
|
||||||
|
|
|
@ -172,6 +172,14 @@ module "organization" {
|
||||||
"compute.subnetworks.setIamPolicy",
|
"compute.subnetworks.setIamPolicy",
|
||||||
"dns.networks.bindPrivateDNSZone",
|
"dns.networks.bindPrivateDNSZone",
|
||||||
"resourcemanager.projects.get",
|
"resourcemanager.projects.get",
|
||||||
|
|
||||||
|
# if you prefer not granting permissions to create peerings to
|
||||||
|
# service accounts deploying service projects, remove these
|
||||||
|
# permissions and ask you network administrator to create any
|
||||||
|
# needed peerings (e.g. if you need to update routes for a GKE
|
||||||
|
# cluster)
|
||||||
|
"compute.networks.updatePeering",
|
||||||
|
"compute.networks.get",
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
logging_sinks = {
|
logging_sinks = {
|
||||||
|
|
|
@ -159,6 +159,7 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| [billing.tf](./billing.tf) | Billing resources for external billing use cases. | <code>organization</code> | <code>google_billing_account_iam_member</code> |
|
| [billing.tf](./billing.tf) | Billing resources for external billing use cases. | <code>organization</code> | <code>google_billing_account_iam_member</code> |
|
||||||
| [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
| [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
||||||
|
| [branch-gke.tf](./branch-gke.tf) | GKE multitenant stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
||||||
| [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
| [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
||||||
| [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
| [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
||||||
| [branch-security.tf](./branch-security.tf) | Security stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
| [branch-security.tf](./branch-security.tf) | Security stage resources. | <code>folder</code> · <code>gcs</code> · <code>iam-service-account</code> | |
|
||||||
|
@ -194,14 +195,15 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
|
||||||
|
|
||||||
| name | description | sensitive | consumers |
|
| name | description | sensitive | consumers |
|
||||||
|---|---|:---:|---|
|
|---|---|:---:|---|
|
||||||
| [cicd_repositories](outputs.tf#L143) | WIF configuration for CI/CD repositories. | | |
|
| [cicd_repositories](outputs.tf#L157) | WIF configuration for CI/CD repositories. | | |
|
||||||
| [dataplatform](outputs.tf#L155) | Data for the Data Platform stage. | | |
|
| [dataplatform](outputs.tf#L169) | Data for the Data Platform stage. | | |
|
||||||
| [networking](outputs.tf#L171) | Data for the networking stage. | | |
|
| [gke_multitenant](outputs.tf#L237) | Data for the GKE multitenant stage. | | <code>03-gke-multitenant</code> |
|
||||||
| [project_factories](outputs.tf#L180) | Data for the project factories stage. | | |
|
| [networking](outputs.tf#L185) | Data for the networking stage. | | |
|
||||||
| [providers](outputs.tf#L196) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
|
| [project_factories](outputs.tf#L194) | Data for the project factories stage. | | |
|
||||||
| [sandbox](outputs.tf#L203) | Data for the sandbox stage. | | <code>xx-sandbox</code> |
|
| [providers](outputs.tf#L210) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
|
||||||
| [security](outputs.tf#L213) | Data for the networking stage. | | <code>02-security</code> |
|
| [sandbox](outputs.tf#L217) | Data for the sandbox stage. | | <code>xx-sandbox</code> |
|
||||||
| [teams](outputs.tf#L223) | Data for the teams stage. | | |
|
| [security](outputs.tf#L227) | Data for the networking stage. | | <code>02-security</code> |
|
||||||
| [tfvars](outputs.tf#L236) | Terraform variable files for the following stages. | ✓ | |
|
| [teams](outputs.tf#L254) | Data for the teams stage. | | |
|
||||||
|
| [tfvars](outputs.tf#L267) | Terraform variable files for the following stages. | ✓ | |
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -29,6 +29,7 @@ locals {
|
||||||
# for k, v in module.branch-teams-team-sa : v.iam_email
|
# for k, v in module.branch-teams-team-sa : v.iam_email
|
||||||
# ],
|
# ],
|
||||||
local.branch_teams_pf_sa_iam_emails,
|
local.branch_teams_pf_sa_iam_emails,
|
||||||
|
local.branch_gke_multitenant_sa_iam_emails
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,108 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
# tfdoc:file:description GKE multitenant stage resources.
|
||||||
|
|
||||||
|
# top-level gke folder
|
||||||
|
|
||||||
|
module "branch-gke-folder" {
|
||||||
|
source = "../../../modules/folder"
|
||||||
|
parent = "organizations/${var.organization.id}"
|
||||||
|
name = "GKE"
|
||||||
|
# iam = {
|
||||||
|
# "roles/logging.admin" = [module.branch-gke-sa.iam_email]
|
||||||
|
# "roles/owner" = [module.branch-gke-sa.iam_email]
|
||||||
|
# "roles/resourcemanager.folderAdmin" = [module.branch-gke-sa.iam_email]
|
||||||
|
# "roles/resourcemanager.projectCreator" = [module.branch-gke-sa.iam_email]
|
||||||
|
# }
|
||||||
|
}
|
||||||
|
|
||||||
|
# GKE-level folders, service accounts and buckets for each individual environment
|
||||||
|
|
||||||
|
module "branch-gke-multitenant-prod-folder" {
|
||||||
|
source = "../../../modules/folder"
|
||||||
|
parent = module.branch-gke-folder.id
|
||||||
|
name = "prod"
|
||||||
|
iam = {
|
||||||
|
"roles/owner" = [
|
||||||
|
module.branch-gke-multitenant-prod-sa.iam_email
|
||||||
|
]
|
||||||
|
"roles/resourcemanager.projectCreator" = [
|
||||||
|
module.branch-gke-multitenant-prod-sa.iam_email
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "branch-gke-multitenant-prod-sa" {
|
||||||
|
source = "../../../modules/iam-service-account"
|
||||||
|
project_id = var.automation_project_id
|
||||||
|
name = "gke-prod-0"
|
||||||
|
description = "Terraform gke multitenant prod service account."
|
||||||
|
prefix = var.prefix
|
||||||
|
iam = {
|
||||||
|
# FIXME(jccb): who should we use here?
|
||||||
|
"roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "branch-gke-multitenant-prod-gcs" {
|
||||||
|
source = "../../../modules/gcs"
|
||||||
|
project_id = var.automation_project_id
|
||||||
|
name = "gke-prod-0"
|
||||||
|
prefix = var.prefix
|
||||||
|
versioning = true
|
||||||
|
iam = {
|
||||||
|
"roles/storage.objectAdmin" = [module.branch-gke-multitenant-prod-sa.iam_email]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
module "branch-gke-multitenant-dev-folder" {
|
||||||
|
source = "../../../modules/folder"
|
||||||
|
parent = module.branch-gke-folder.id
|
||||||
|
name = "dev"
|
||||||
|
iam = {
|
||||||
|
"roles/owner" = [
|
||||||
|
module.branch-gke-multitenant-dev-sa.iam_email
|
||||||
|
]
|
||||||
|
"roles/resourcemanager.projectCreator" = [
|
||||||
|
module.branch-gke-multitenant-dev-sa.iam_email
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "branch-gke-multitenant-dev-sa" {
|
||||||
|
source = "../../../modules/iam-service-account"
|
||||||
|
project_id = var.automation_project_id
|
||||||
|
name = "gke-dev-0"
|
||||||
|
description = "Terraform gke multitenant dev service account."
|
||||||
|
prefix = var.prefix
|
||||||
|
iam = {
|
||||||
|
# FIXME(jccb): who should we use here?
|
||||||
|
"roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "branch-gke-multitenant-dev-gcs" {
|
||||||
|
source = "../../../modules/gcs"
|
||||||
|
project_id = var.automation_project_id
|
||||||
|
name = "gke-dev-0"
|
||||||
|
prefix = var.prefix
|
||||||
|
versioning = true
|
||||||
|
iam = {
|
||||||
|
"roles/storage.objectAdmin" = [module.branch-gke-multitenant-dev-sa.iam_email]
|
||||||
|
}
|
||||||
|
}
|
|
@ -52,7 +52,8 @@ module "branch-network-prod-folder" {
|
||||||
iam = {
|
iam = {
|
||||||
"roles/compute.xpnAdmin" = [
|
"roles/compute.xpnAdmin" = [
|
||||||
module.branch-dp-prod-sa.iam_email,
|
module.branch-dp-prod-sa.iam_email,
|
||||||
module.branch-teams-prod-pf-sa.iam_email
|
module.branch-teams-prod-pf-sa.iam_email,
|
||||||
|
module.branch-gke-multitenant-prod-sa.iam_email,
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
tag_bindings = {
|
tag_bindings = {
|
||||||
|
@ -69,7 +70,8 @@ module "branch-network-dev-folder" {
|
||||||
iam = {
|
iam = {
|
||||||
(local.custom_roles.service_project_network_admin) = [
|
(local.custom_roles.service_project_network_admin) = [
|
||||||
module.branch-dp-dev-sa.iam_email,
|
module.branch-dp-dev-sa.iam_email,
|
||||||
module.branch-teams-dev-pf-sa.iam_email
|
module.branch-teams-dev-pf-sa.iam_email,
|
||||||
|
module.branch-gke-multitenant-dev-sa.iam_email,
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
tag_bindings = {
|
tag_bindings = {
|
||||||
|
|
|
@ -28,6 +28,10 @@ locals {
|
||||||
module.branch-teams-dev-pf-sa.iam_email,
|
module.branch-teams-dev-pf-sa.iam_email,
|
||||||
module.branch-teams-prod-pf-sa.iam_email
|
module.branch-teams-prod-pf-sa.iam_email
|
||||||
]
|
]
|
||||||
|
branch_gke_multitenant_sa_iam_emails = [
|
||||||
|
module.branch-gke-multitenant-dev-sa.iam_email,
|
||||||
|
module.branch-gke-multitenant-prod-sa.iam_email
|
||||||
|
]
|
||||||
list_allow = {
|
list_allow = {
|
||||||
inherit_from_parent = false
|
inherit_from_parent = false
|
||||||
suggested_value = null
|
suggested_value = null
|
||||||
|
@ -76,7 +80,8 @@ module "organization" {
|
||||||
# [
|
# [
|
||||||
# for k, v in module.branch-teams-team-sa : v.iam_email
|
# for k, v in module.branch-teams-team-sa : v.iam_email
|
||||||
# ],
|
# ],
|
||||||
local.branch_teams_pf_sa_iam_emails
|
local.branch_teams_pf_sa_iam_emails,
|
||||||
|
local.branch_gke_multitenant_sa_iam_emails
|
||||||
)
|
)
|
||||||
} : {}
|
} : {}
|
||||||
)
|
)
|
||||||
|
|
|
@ -61,6 +61,8 @@ locals {
|
||||||
folder_ids = merge(
|
folder_ids = merge(
|
||||||
{
|
{
|
||||||
data-platform = module.branch-dp-dev-folder.id
|
data-platform = module.branch-dp-dev-folder.id
|
||||||
|
gke-multitenant-dev = module.branch-gke-multitenant-dev-folder.id
|
||||||
|
gke-multitenant-prod = module.branch-gke-multitenant-prod-folder.id
|
||||||
networking = module.branch-network-folder.id
|
networking = module.branch-network-folder.id
|
||||||
networking-dev = module.branch-network-dev-folder.id
|
networking-dev = module.branch-network-dev-folder.id
|
||||||
networking-prod = module.branch-network-prod-folder.id
|
networking-prod = module.branch-network-prod-folder.id
|
||||||
|
@ -102,6 +104,16 @@ locals {
|
||||||
name = "dp-prod"
|
name = "dp-prod"
|
||||||
sa = module.branch-dp-prod-sa.email
|
sa = module.branch-dp-prod-sa.email
|
||||||
})
|
})
|
||||||
|
"03-gke-multitenant-dev" = templatefile(local._tpl_providers, {
|
||||||
|
bucket = module.branch-gke-multitenant-dev-gcs.name
|
||||||
|
name = "gke-multitenant-dev"
|
||||||
|
sa = module.branch-gke-multitenant-dev-sa.email
|
||||||
|
})
|
||||||
|
"03-gke-multitenant-prod" = templatefile(local._tpl_providers, {
|
||||||
|
bucket = module.branch-gke-multitenant-prod-gcs.name
|
||||||
|
name = "gke-multitenant-prod"
|
||||||
|
sa = module.branch-gke-multitenant-prod-sa.email
|
||||||
|
})
|
||||||
"03-project-factory-dev" = templatefile(local._tpl_providers, {
|
"03-project-factory-dev" = templatefile(local._tpl_providers, {
|
||||||
bucket = module.branch-teams-dev-pf-gcs.name
|
bucket = module.branch-teams-dev-pf-gcs.name
|
||||||
name = "team-dev"
|
name = "team-dev"
|
||||||
|
@ -122,6 +134,8 @@ locals {
|
||||||
{
|
{
|
||||||
data-platform-dev = module.branch-dp-dev-sa.email
|
data-platform-dev = module.branch-dp-dev-sa.email
|
||||||
data-platform-prod = module.branch-dp-prod-sa.email
|
data-platform-prod = module.branch-dp-prod-sa.email
|
||||||
|
gke-multitenant-dev = module.branch-gke-multitenant-dev-sa.iam_email
|
||||||
|
gke-multitenant-prod = module.branch-gke-multitenant-prod-sa.iam_email
|
||||||
networking = module.branch-network-sa.email
|
networking = module.branch-network-sa.email
|
||||||
project-factory-dev = module.branch-teams-dev-pf-sa.email
|
project-factory-dev = module.branch-teams-dev-pf-sa.email
|
||||||
project-factory-prod = module.branch-teams-prod-pf-sa.email
|
project-factory-prod = module.branch-teams-prod-pf-sa.email
|
||||||
|
@ -220,6 +234,23 @@ output "security" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
output "gke_multitenant" {
|
||||||
|
# tfdoc:output:consumers 03-gke-multitenant
|
||||||
|
description = "Data for the GKE multitenant stage."
|
||||||
|
value = {
|
||||||
|
"dev" = {
|
||||||
|
folder = module.branch-gke-multitenant-dev-folder.id
|
||||||
|
gcs_bucket = module.branch-gke-multitenant-dev-gcs.name
|
||||||
|
service_account = module.branch-gke-multitenant-dev-sa.email
|
||||||
|
}
|
||||||
|
"prod" = {
|
||||||
|
folder = module.branch-gke-multitenant-prod-folder.id
|
||||||
|
gcs_bucket = module.branch-gke-multitenant-prod-gcs.name
|
||||||
|
service_account = module.branch-gke-multitenant-prod-sa.email
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
output "teams" {
|
output "teams" {
|
||||||
description = "Data for the teams stage."
|
description = "Data for the teams stage."
|
||||||
value = {
|
value = {
|
||||||
|
|
|
@ -315,19 +315,20 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS
|
||||||
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||||
| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object({ id = string organization_id = number })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object({ id = string organization_id = number })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||||
| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code title="object({ networking = string networking-dev = string networking-prod = string })">object({…})</code> | ✓ | | <code>01-resman</code> |
|
| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | <code title="object({ networking = string networking-dev = string networking-prod = string })">object({…})</code> | ✓ | | <code>01-resman</code> |
|
||||||
| [organization](variables.tf#L102) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
| [organization](variables.tf#L109) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-bootstrap</code> |
|
||||||
| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
| [prefix](variables.tf#L125) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
|
||||||
| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev = "10.128.32.0/19" gcp_landing = "10.128.0.0/19" gcp_prod = "10.128.64.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | <code>map(string)</code> | | <code title="{ cloud_dns = "35.199.192.0/19" gcp_all = "10.128.0.0/16" gcp_dev = "10.128.32.0/19" gcp_landing = "10.128.0.0/19" gcp_prod = "10.128.64.0/19" googleapis_private = "199.36.153.8/30" googleapis_restricted = "199.36.153.4/30" rfc_1918_10 = "10.0.0.0/8" rfc_1918_172 = "172.16.0.0/12" rfc_1918_192 = "192.168.0.0/16" }">{…}</code> | |
|
||||||
| [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | <code title="object({ service_project_network_admin = string })">object({…})</code> | | <code>null</code> | <code>00-bootstrap</code> |
|
| [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | <code title="object({ service_project_network_admin = string })">object({…})</code> | | <code>null</code> | <code>00-bootstrap</code> |
|
||||||
| [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | <code>string</code> | | <code>"data"</code> | |
|
| [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | <code>string</code> | | <code>"data"</code> | |
|
||||||
| [dns](variables.tf#L66) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
| [dns](variables.tf#L66) | Onprem DNS resolvers. | <code>map(list(string))</code> | | <code title="{ onprem = ["10.0.200.3"] }">{…}</code> | |
|
||||||
| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" } ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" } ] }">{…}</code> | |
|
| [gke_multitenant_sa](variables.tf#L84) | IAM emails for GKE multitenant service accounts. | <code>map(string)</code> | | <code>{}</code> | <code>01-resman</code> |
|
||||||
| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
| [l7ilb_subnets](variables.tf#L91) | Subnets used for L7 ILBs. | <code title="map(list(object({ ip_cidr_range = string region = string })))">map(list(object({…})))</code> | | <code title="{ prod = [ { ip_cidr_range = "10.128.92.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.93.0/24", region = "europe-west4" } ] dev = [ { ip_cidr_range = "10.128.60.0/24", region = "europe-west1" }, { ip_cidr_range = "10.128.61.0/24", region = "europe-west4" } ] }">{…}</code> | |
|
||||||
| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
| [outputs_location](variables.tf#L119) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||||
| [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { asn = "65533" adv = null } }">{…}</code> | |
|
| [psa_ranges](variables.tf#L136) | IP ranges used for Private Service Access (e.g. CloudSQL). | <code title="object({ dev = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) prod = object({ ranges = map(string) routes = object({ export = bool import = bool }) }) })">object({…})</code> | | <code>null</code> | |
|
||||||
|
| [router_onprem_configs](variables.tf#L173) | Configurations for routers used for onprem connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { asn = "65533" adv = null } }">{…}</code> | |
|
||||||
| [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { asn = "64512", adv = null } landing-ew4 = { asn = "64512", adv = null } spoke-dev-ew1 = { asn = "64513", adv = null } spoke-dev-ew4 = { asn = "64513", adv = null } spoke-prod-ew1 = { asn = "64514", adv = null } spoke-prod-ew4 = { asn = "64514", adv = null } }">{…}</code> | |
|
| [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | <code title="map(object({ adv = object({ custom = list(string) default = bool }) asn = number }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { asn = "64512", adv = null } landing-ew4 = { asn = "64512", adv = null } spoke-dev-ew1 = { asn = "64513", adv = null } spoke-dev-ew4 = { asn = "64513", adv = null } spoke-prod-ew1 = { asn = "64514", adv = null } spoke-prod-ew4 = { asn = "64514", adv = null } }">{…}</code> | |
|
||||||
| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>01-resman</code> |
|
| [service_accounts](variables.tf#L191) | Automation service accounts in name => email format. | <code title="object({ data-platform-dev = string data-platform-prod = string project-factory-dev = string project-factory-prod = string })">object({…})</code> | | <code>null</code> | <code>01-resman</code> |
|
||||||
| [vpn_onprem_configs](variables.tf#L196) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(object({ id = number ip_address = string })) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
| [vpn_onprem_configs](variables.tf#L203) | VPN gateway configuration for onprem interconnection. | <code title="map(object({ adv = object({ default = bool custom = list(string) }) peer_external_gateway = object({ redundancy_type = string interfaces = list(object({ id = number ip_address = string })) }) tunnels = list(object({ peer_asn = number peer_external_gateway_interface = number secret = string session_range = string vpn_gateway_interface = number })) }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { adv = { default = false custom = [ "cloud_dns", "googleapis_private", "googleapis_restricted", "gcp_all" ] } peer_external_gateway = { redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" interfaces = [ { id = 0, ip_address = "8.8.8.8" }, ] } tunnels = [ { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.0/30" vpn_gateway_interface = 0 }, { peer_asn = 65534 peer_external_gateway_interface = 0 secret = "foobar" session_range = "169.254.1.4/30" vpn_gateway_interface = 1 } ] } }">{…}</code> | |
|
||||||
| [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | <code title="map(object({ default = bool custom = list(string) }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } landing-ew4 = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } dev-ew1 = { default = false custom = ["gcp_dev"] } prod-ew1 = { default = false custom = ["gcp_prod"] } prod-ew4 = { default = false custom = ["gcp_prod"] } }">{…}</code> | |
|
| [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | <code title="map(object({ default = bool custom = list(string) }))">map(object({…}))</code> | | <code title="{ landing-ew1 = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } landing-ew4 = { default = false custom = ["rfc_1918_10", "rfc_1918_172", "rfc_1918_192"] } dev-ew1 = { default = false custom = ["gcp_dev"] } prod-ew1 = { default = false custom = ["gcp_prod"] } prod-ew4 = { default = false custom = ["gcp_prod"] } }">{…}</code> | |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
# skip boilerplate check
|
||||||
|
|
||||||
|
region: europe-west1
|
||||||
|
description: Default subnet for prod gke nodes
|
||||||
|
ip_cidr_range: 10.64.0.0/24
|
||||||
|
secondary_ip_range:
|
||||||
|
pods: 100.64.0.0/16
|
||||||
|
services: 192.168.1.0/24
|
|
@ -103,6 +103,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" {
|
||||||
members = [
|
members = [
|
||||||
local.service_accounts.data-platform-dev,
|
local.service_accounts.data-platform-dev,
|
||||||
local.service_accounts.project-factory-dev,
|
local.service_accounts.project-factory-dev,
|
||||||
|
local.service_accounts.gke_multitenant_sa,
|
||||||
]
|
]
|
||||||
condition {
|
condition {
|
||||||
title = "dev_stage3_sa_delegated_grants"
|
title = "dev_stage3_sa_delegated_grants"
|
||||||
|
|
|
@ -103,6 +103,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" {
|
||||||
members = [
|
members = [
|
||||||
local.service_accounts.data-platform-prod,
|
local.service_accounts.data-platform-prod,
|
||||||
local.service_accounts.project-factory-prod,
|
local.service_accounts.project-factory-prod,
|
||||||
|
local.service_accounts.gke-multitenant-prod,
|
||||||
]
|
]
|
||||||
condition {
|
condition {
|
||||||
title = "prod_stage3_sa_delegated_grants"
|
title = "prod_stage3_sa_delegated_grants"
|
||||||
|
|
|
@ -81,6 +81,13 @@ variable "folder_ids" {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "gke_multitenant_sa" {
|
||||||
|
# tfdoc:variable:source 01-resman
|
||||||
|
description = "IAM emails for GKE multitenant service accounts."
|
||||||
|
type = map(string)
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
variable "l7ilb_subnets" {
|
variable "l7ilb_subnets" {
|
||||||
description = "Subnets used for L7 ILBs."
|
description = "Subnets used for L7 ILBs."
|
||||||
type = map(list(object({
|
type = map(list(object({
|
||||||
|
|
|
@ -0,0 +1,105 @@
|
||||||
|
## IAM
|
||||||
|
|
||||||
|
- project+infra cluster admin
|
||||||
|
|
||||||
|
- cluster admins
|
||||||
|
they have IAM bindings at proj level
|
||||||
|
|
||||||
|
- namespace admins / users
|
||||||
|
cluster user? only IAM role to fetch GKE creds
|
||||||
|
everything else is in RBAC (second part of this stage)
|
||||||
|
|
||||||
|
- permissions to node service accounts
|
||||||
|
- logging/monitoring
|
||||||
|
- image registry
|
||||||
|
|
||||||
|
- custom role to only allow autoprovisioning of credentials
|
||||||
|
|
||||||
|
## Inputs from previous stages
|
||||||
|
|
||||||
|
- vpc host and subnets
|
||||||
|
|
||||||
|
## Network User role
|
||||||
|
|
||||||
|
- (optional) resman stage 01 creates SA, net stage 02 sets role on net project, dev --> dev, prod --> prod
|
||||||
|
- (always possible) subnet factory assigns role
|
||||||
|
|
||||||
|
## Resources
|
||||||
|
|
||||||
|
### Robot service accounts
|
||||||
|
|
||||||
|
- configured after prject activation (svpc roles, etc.)
|
||||||
|
- gke host service agent
|
||||||
|
- option for security admin
|
||||||
|
- CMEK for disks
|
||||||
|
|
||||||
|
### Service accounts
|
||||||
|
|
||||||
|
- node service accounts (per nodepool?)
|
||||||
|
- config sync service account (start with a single one, +wl identity)
|
||||||
|
- backup for GKE
|
||||||
|
|
||||||
|
### Registry
|
||||||
|
|
||||||
|
- per env?
|
||||||
|
- deep dive on partitioning per team
|
||||||
|
|
||||||
|
### Clusters and nodepools
|
||||||
|
|
||||||
|
- n clusters
|
||||||
|
- no default nodepool (or default nodepool has taints -- infra nodepool)
|
||||||
|
- what can change between clusters?
|
||||||
|
- stays the same
|
||||||
|
- monitoring and logging config
|
||||||
|
- autoprovisioning
|
||||||
|
- private / public
|
||||||
|
- workload identity
|
||||||
|
- cloud dns
|
||||||
|
- can change between clusters
|
||||||
|
- dataplane v2
|
||||||
|
- nodepools
|
||||||
|
- binary auth
|
||||||
|
- nodepools per cluster
|
||||||
|
|
||||||
|
## Addresses for ILBs
|
||||||
|
|
||||||
|
## Filestore
|
||||||
|
|
||||||
|
- later
|
||||||
|
|
||||||
|
## RBAC
|
||||||
|
|
||||||
|
|
||||||
|
org
|
||||||
|
- GKE
|
||||||
|
- prod
|
||||||
|
- dev
|
||||||
|
- core
|
||||||
|
|
||||||
|
branch: gke-stage3
|
||||||
|
|
||||||
|
resman
|
||||||
|
GKE folder + env folders + gcs + sa (extra file to drop in stage 1)
|
||||||
|
|
||||||
|
[TF] supporting infra - stage 3 gke / infra (apply with SA from stage 1)
|
||||||
|
**single environment** (e.g. prod)
|
||||||
|
IAM for admins and users
|
||||||
|
projects for clusters
|
||||||
|
VPC wiring
|
||||||
|
logging
|
||||||
|
monitoring
|
||||||
|
container registries
|
||||||
|
source repos
|
||||||
|
workload identity config
|
||||||
|
Secrets (?)
|
||||||
|
clusters and nodepools and service accounts
|
||||||
|
managed prometheus?
|
||||||
|
|
||||||
|
[Manifests/etc.] multitenant cluster config - stage 3 gke / cluster config (apply/runs with credentials from infra part)
|
||||||
|
config sync
|
||||||
|
gatekeeper
|
||||||
|
team namespaces
|
||||||
|
network policies
|
||||||
|
in-cluster logging and monitoring
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,120 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
locals {
|
||||||
|
clusters = {
|
||||||
|
for name, config in var.clusters :
|
||||||
|
name => merge(config, {
|
||||||
|
overrides = coalesce(config.overrides, var.cluster_defaults)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "gke-cluster" {
|
||||||
|
source = "../../../../modules/gke-cluster"
|
||||||
|
for_each = local.clusters
|
||||||
|
project_id = module.gke-project-0.project_id
|
||||||
|
name = each.key
|
||||||
|
description = each.value.description
|
||||||
|
location = each.value.location
|
||||||
|
network = each.value.net.vpc
|
||||||
|
subnetwork = each.value.net.subnet
|
||||||
|
secondary_range_pods = each.value.net.pods
|
||||||
|
secondary_range_services = each.value.net.services
|
||||||
|
labels = each.value.labels
|
||||||
|
addons = {
|
||||||
|
cloudrun_config = each.value.overrides.cloudrun_config
|
||||||
|
dns_cache_config = true
|
||||||
|
http_load_balancing = true
|
||||||
|
gce_persistent_disk_csi_driver_config = true
|
||||||
|
horizontal_pod_autoscaling = true
|
||||||
|
config_connector_config = true
|
||||||
|
kalm_config = false
|
||||||
|
# enable only if enable_dataplane_v2 is changed to false below
|
||||||
|
network_policy_config = false
|
||||||
|
istio_config = {
|
||||||
|
enabled = false
|
||||||
|
tls = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# change these here for all clusters if absolutely needed
|
||||||
|
# authenticator_security_group = var.authenticator_security_group
|
||||||
|
enable_dataplane_v2 = true
|
||||||
|
enable_l4_ilb_subsetting = false
|
||||||
|
enable_intranode_visibility = true
|
||||||
|
enable_shielded_nodes = true
|
||||||
|
workload_identity = true
|
||||||
|
private_cluster_config = {
|
||||||
|
enable_private_nodes = true
|
||||||
|
enable_private_endpoint = true
|
||||||
|
master_ipv4_cidr_block = each.value.net.master_range
|
||||||
|
master_global_access = true
|
||||||
|
}
|
||||||
|
dns_config = each.value.dns_domain == null ? null : {
|
||||||
|
cluster_dns = "CLOUD_DNS"
|
||||||
|
cluster_dns_scope = "VPC_SCOPE"
|
||||||
|
cluster_dns_domain = "${each.key}.${var.dns_domain}"
|
||||||
|
}
|
||||||
|
logging_config = ["SYSTEM_COMPONENTS", "WORKLOADS"]
|
||||||
|
monitoring_config = ["SYSTEM_COMPONENTS", "WORKLOADS"]
|
||||||
|
|
||||||
|
# if you don't have compute.networks.updatePeering in the host
|
||||||
|
# project, comment out the next line and ask your network admin to
|
||||||
|
# create the peering for you
|
||||||
|
peering_config = {
|
||||||
|
export_routes = true
|
||||||
|
import_routes = false
|
||||||
|
project_id = var.vpc_host_project
|
||||||
|
}
|
||||||
|
# resource_usage_export_config = {
|
||||||
|
# enabled = true
|
||||||
|
# dataset = module.gke-dataset-resource-usage.id
|
||||||
|
# }
|
||||||
|
# TODO: the attributes below are "primed" from project-level defaults
|
||||||
|
# in locals, merge defaults with cluster-level stuff
|
||||||
|
# TODO(jccb): change fabric module
|
||||||
|
database_encryption = (
|
||||||
|
each.value.overrides.database_encryption_key == null ? {
|
||||||
|
enabled = false
|
||||||
|
state = null
|
||||||
|
key_name = null
|
||||||
|
} : {
|
||||||
|
enabled = true
|
||||||
|
state = "ENCRYPTED"
|
||||||
|
key_name = each.value.overrides.database_encryption_key
|
||||||
|
}
|
||||||
|
)
|
||||||
|
default_max_pods_per_node = each.value.overrides.max_pods_per_node
|
||||||
|
enable_binary_authorization = each.value.overrides.enable_binary_authorization
|
||||||
|
master_authorized_ranges = each.value.overrides.master_authorized_ranges
|
||||||
|
pod_security_policy = each.value.overrides.pod_security_policy
|
||||||
|
release_channel = each.value.overrides.release_channel
|
||||||
|
vertical_pod_autoscaling = each.value.overrides.vertical_pod_autoscaling
|
||||||
|
# dynamic "cluster_autoscaling" {
|
||||||
|
# for_each = each.value.cluster_autoscaling == null ? {} : { 1 = 1 }
|
||||||
|
# content {
|
||||||
|
# enabled = true
|
||||||
|
# cpu_min = each.value.cluster_autoscaling.cpu_min
|
||||||
|
# cpu_max = each.value.cluster_autoscaling.cpu_max
|
||||||
|
# memory_min = each.value.cluster_autoscaling.memory_min
|
||||||
|
# memory_max = each.value.cluster_autoscaling.memory_max
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
|
||||||
|
depends_on = [
|
||||||
|
google_project_iam_member.host_project_bindings
|
||||||
|
]
|
||||||
|
}
|
|
@ -0,0 +1,67 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
locals {
|
||||||
|
nodepools = merge([
|
||||||
|
for cluster, nodepools in var.nodepools : {
|
||||||
|
for nodepool, config in nodepools :
|
||||||
|
"${cluster}/${nodepool}" => merge(config, {
|
||||||
|
name = nodepool
|
||||||
|
cluster = cluster
|
||||||
|
overrides = coalesce(config.overrides, var.nodepool_defaults)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
]...)
|
||||||
|
}
|
||||||
|
|
||||||
|
module "gke_1_nodepool" {
|
||||||
|
source = "../../../../modules/gke-nodepool"
|
||||||
|
for_each = local.nodepools
|
||||||
|
name = each.value.name
|
||||||
|
project_id = module.gke-project-0.project_id
|
||||||
|
cluster_name = module.gke-cluster[each.value.cluster].name
|
||||||
|
location = module.gke-cluster[each.value.cluster].location
|
||||||
|
initial_node_count = each.value.node_count
|
||||||
|
node_machine_type = each.value.node_type
|
||||||
|
# TODO(jccb): can we use spot instances here?
|
||||||
|
node_preemptible = each.value.preemptible
|
||||||
|
|
||||||
|
node_count = each.value.node_count
|
||||||
|
# node_count = (
|
||||||
|
# each.value.autoscaling_config == null ? each.value.node_count : null
|
||||||
|
# )
|
||||||
|
# dynamic "autoscaling_config" {
|
||||||
|
# for_each = each.value.autoscaling_config == null ? {} : { 1 = 1 }
|
||||||
|
# content {
|
||||||
|
# min_node_count = each.value.autoscaling_config.min_node_count
|
||||||
|
# max_node_count = each.value.autoscaling_config.max_node_count
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
|
||||||
|
# overrides
|
||||||
|
node_locations = each.value.overrides.node_locations
|
||||||
|
max_pods_per_node = each.value.overrides.max_pods_per_node
|
||||||
|
node_image_type = each.value.overrides.image_type
|
||||||
|
node_tags = each.value.overrides.node_tags
|
||||||
|
node_taints = each.value.overrides.node_taints
|
||||||
|
|
||||||
|
management_config = {
|
||||||
|
auto_repair = true
|
||||||
|
auto_upgrade = true
|
||||||
|
}
|
||||||
|
|
||||||
|
node_service_account_create = true
|
||||||
|
}
|
|
@ -0,0 +1,122 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
locals {
|
||||||
|
labels = merge(var.labels, { environment = var.environment })
|
||||||
|
|
||||||
|
_gke_robot_sa = "serviceAccount:${module.gke-project-0.service_accounts.robots.container-engine}"
|
||||||
|
_cloud_services_sa = "serviceAccount:${module.gke-project-0.service_accounts.cloud_services}"
|
||||||
|
host_project_bindings = [
|
||||||
|
{ role = "roles/container.hostServiceAgentUser", member = local._gke_robot_sa },
|
||||||
|
{ role = "roles/compute.networkUser", member = local._gke_robot_sa },
|
||||||
|
{ role = "roles/compute.networkUser", member = local._cloud_services_sa }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
module "gke-project-0" {
|
||||||
|
source = "../../../../modules/project"
|
||||||
|
billing_account = var.billing_account_id
|
||||||
|
name = "${var.environment}-gke-clusters-0"
|
||||||
|
parent = var.folder_id
|
||||||
|
prefix = var.prefix
|
||||||
|
labels = local.labels
|
||||||
|
services = [
|
||||||
|
"container.googleapis.com",
|
||||||
|
"dns.googleapis.com",
|
||||||
|
"stackdriver.googleapis.com",
|
||||||
|
# uncomment if you need Multi-cluster Ingress / Gateway API
|
||||||
|
# "gkehub.googleapis.com",
|
||||||
|
# "multiclusterservicediscovery.googleapis.com",
|
||||||
|
# "multiclusteringress.googleapis.com",
|
||||||
|
# "trafficdirector.googleapis.com"
|
||||||
|
]
|
||||||
|
# add here any other service ids and keys for robot accounts which are needed
|
||||||
|
# service_encryption_key_ids = {
|
||||||
|
# container = var.project_config.service_encryption_key_ids
|
||||||
|
# }
|
||||||
|
shared_vpc_service_config = {
|
||||||
|
attach = true
|
||||||
|
host_project = var.vpc_host_project
|
||||||
|
}
|
||||||
|
# specify project-level org policies here if you need them
|
||||||
|
|
||||||
|
# policy_boolean = {
|
||||||
|
# "constraints/compute.disableGuestAttributesAccess" = true
|
||||||
|
# }
|
||||||
|
# policy_list = {
|
||||||
|
# "constraints/compute.trustedImageProjects" = {
|
||||||
|
# inherit_from_parent = null
|
||||||
|
# suggested_value = null
|
||||||
|
# status = true
|
||||||
|
# values = ["projects/fl01-prod-iac-core-0"]
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
}
|
||||||
|
|
||||||
|
module "gke-project-1" {
|
||||||
|
source = "../../../../modules/project"
|
||||||
|
billing_account = var.billing_account_id
|
||||||
|
name = "${var.environment}-gke-clusters-2j"
|
||||||
|
parent = var.folder_id
|
||||||
|
prefix = var.prefix
|
||||||
|
labels = local.labels
|
||||||
|
services = [
|
||||||
|
"container.googleapis.com",
|
||||||
|
"dns.googleapis.com",
|
||||||
|
"stackdriver.googleapis.com",
|
||||||
|
# uncomment if you need Multi-cluster Ingress / Gateway API
|
||||||
|
# "gkehub.googleapis.com",
|
||||||
|
# "multiclusterservicediscovery.googleapis.com",
|
||||||
|
# "multiclusteringress.googleapis.com",
|
||||||
|
# "trafficdirector.googleapis.com"
|
||||||
|
]
|
||||||
|
# add here any other service ids and keys for robot accounts which are needed
|
||||||
|
# service_encryption_key_ids = {
|
||||||
|
# container = var.project_config.service_encryption_key_ids
|
||||||
|
# }
|
||||||
|
shared_vpc_service_config = {
|
||||||
|
attach = true
|
||||||
|
host_project = var.vpc_host_project
|
||||||
|
}
|
||||||
|
# specify project-level org policies here if you need them
|
||||||
|
|
||||||
|
# policy_boolean = {
|
||||||
|
# "constraints/compute.disableGuestAttributesAccess" = true
|
||||||
|
# }
|
||||||
|
# policy_list = {
|
||||||
|
# "constraints/compute.trustedImageProjects" = {
|
||||||
|
# inherit_from_parent = null
|
||||||
|
# suggested_value = null
|
||||||
|
# status = true
|
||||||
|
# values = ["projects/fl01-prod-iac-core-0"]
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
module "gke-dataset-resource-usage" {
|
||||||
|
source = "../../../../modules/bigquery-dataset"
|
||||||
|
project_id = module.gke-project-0.project_id
|
||||||
|
id = "resource_usage"
|
||||||
|
friendly_name = "GKE resource usage."
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "google_project_iam_member" "host_project_bindings" {
|
||||||
|
for_each = { for i, v in local.host_project_bindings : i => v }
|
||||||
|
project = var.vpc_host_project
|
||||||
|
role = each.value.role
|
||||||
|
member = each.value.member
|
||||||
|
}
|
|
@ -0,0 +1,15 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
|
@ -0,0 +1,162 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
# we deal with one env here
|
||||||
|
# 1 project, m clusters
|
||||||
|
# cloud dns for gke?
|
||||||
|
|
||||||
|
# variable "authenticator_security_group" {
|
||||||
|
# description = "Optional group used for Groups for GKE."
|
||||||
|
# type = string
|
||||||
|
# default = null
|
||||||
|
# }
|
||||||
|
|
||||||
|
variable "billing_account_id" {
|
||||||
|
# tfdoc:variable:source 00-bootstrap
|
||||||
|
description = "Billing account id."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "cluster_defaults" {
|
||||||
|
description = "Default values for optional cluster configurations."
|
||||||
|
type = object({
|
||||||
|
cloudrun_config = bool
|
||||||
|
database_encryption_key = string
|
||||||
|
enable_binary_authorization = bool
|
||||||
|
master_authorized_ranges = map(string)
|
||||||
|
max_pods_per_node = number
|
||||||
|
pod_security_policy = bool
|
||||||
|
release_channel = string
|
||||||
|
vertical_pod_autoscaling = bool
|
||||||
|
})
|
||||||
|
default = {
|
||||||
|
# TODO: review defaults
|
||||||
|
cloudrun_config = false
|
||||||
|
database_encryption_key = null
|
||||||
|
enable_binary_authorization = false
|
||||||
|
master_authorized_ranges = {
|
||||||
|
rfc1918_1 = "10.0.0.0/8"
|
||||||
|
rfc1918_2 = "172.16.0.0/12"
|
||||||
|
rfc1918_3 = "192.168.0.0/16"
|
||||||
|
}
|
||||||
|
max_pods_per_node = 110
|
||||||
|
pod_security_policy = false
|
||||||
|
release_channel = "STABLE"
|
||||||
|
vertical_pod_autoscaling = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "clusters" {
|
||||||
|
description = ""
|
||||||
|
type = map(object({
|
||||||
|
cluster_autoscaling = object({
|
||||||
|
cpu_min = number
|
||||||
|
cpu_max = number
|
||||||
|
memory_min = number
|
||||||
|
memory_max = number
|
||||||
|
})
|
||||||
|
description = string
|
||||||
|
dns_domain = string
|
||||||
|
labels = map(string)
|
||||||
|
location = string
|
||||||
|
net = object({
|
||||||
|
master_range = string
|
||||||
|
pods = string
|
||||||
|
services = string
|
||||||
|
subnet = string
|
||||||
|
vpc = string
|
||||||
|
})
|
||||||
|
overrides = object({
|
||||||
|
cloudrun_config = bool
|
||||||
|
database_encryption_key = string
|
||||||
|
enable_binary_authorization = bool
|
||||||
|
master_authorized_ranges = map(string)
|
||||||
|
max_pods_per_node = number
|
||||||
|
pod_security_policy = bool
|
||||||
|
release_channel = string
|
||||||
|
vertical_pod_autoscaling = bool
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "dns_domain" {
|
||||||
|
description = "Domain name used for clusters, prefix by each cluster name. Leave null to disable Cloud DNS for GKE."
|
||||||
|
type = string
|
||||||
|
default = null
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "environment" {
|
||||||
|
# tfdoc:variable:source 01-resman
|
||||||
|
description = "Environment abbreviation."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "folder_id" {
|
||||||
|
# tfdoc:variable:source 01-resman
|
||||||
|
description = "Folder to be used for the networking resources in folders/nnnn format."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "labels" {
|
||||||
|
description = "Project-level labels."
|
||||||
|
type = map(string)
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "nodepool_defaults" {
|
||||||
|
description = ""
|
||||||
|
type = object({
|
||||||
|
image_type = string
|
||||||
|
max_pods_per_node = number
|
||||||
|
node_locations = list(string)
|
||||||
|
node_tags = list(string)
|
||||||
|
node_taints = list(string)
|
||||||
|
})
|
||||||
|
default = {
|
||||||
|
image_type = "COS_CONTAINERD"
|
||||||
|
max_pods_per_node = 110
|
||||||
|
node_locations = null
|
||||||
|
node_tags = null
|
||||||
|
node_taints = []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "nodepools" {
|
||||||
|
description = ""
|
||||||
|
type = map(map(object({
|
||||||
|
node_count = number
|
||||||
|
node_type = string
|
||||||
|
overrides = object({
|
||||||
|
image_type = string
|
||||||
|
max_pods_per_node = number
|
||||||
|
node_locations = list(string)
|
||||||
|
node_tags = list(string)
|
||||||
|
node_taints = list(string)
|
||||||
|
})
|
||||||
|
preemptible = bool
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "prefix" {
|
||||||
|
description = "Prefix used for resources that need unique names."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "vpc_host_project" {
|
||||||
|
# tfdoc:variable:source 02-networking
|
||||||
|
description = "Host project for the shared VPC."
|
||||||
|
type = string
|
||||||
|
}
|
Loading…
Reference in New Issue