diff --git a/blueprints/gke/multitenant-fleet/README.md b/blueprints/gke/multitenant-fleet/README.md
index d36faa25..baaf288f 100644
--- a/blueprints/gke/multitenant-fleet/README.md
+++ b/blueprints/gke/multitenant-fleet/README.md
@@ -229,7 +229,6 @@ module "gke" {
-
## Files
| name | description | modules |
@@ -246,20 +245,20 @@ module "gke" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L17) | Billing account id. | string
| ✓ | |
-| [folder_id](variables.tf#L132) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string
| ✓ | |
-| [prefix](variables.tf#L183) | Prefix used for resource names. | string
| ✓ | |
-| [project_id](variables.tf#L192) | ID of the project that will contain all the clusters. | string
| ✓ | |
-| [vpc_config](variables.tf#L204) | Shared VPC project and VPC details. | object({…})
| ✓ | |
-| [clusters](variables.tf#L22) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…}))
| | {}
|
-| [fleet_configmanagement_clusters](variables.tf#L70) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string))
| | {}
|
-| [fleet_configmanagement_templates](variables.tf#L77) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…}))
| | {}
|
-| [fleet_features](variables.tf#L112) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…})
| | null
|
-| [fleet_workload_identity](variables.tf#L125) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool
| | false
|
-| [group_iam](variables.tf#L137) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string))
| | {}
|
-| [iam](variables.tf#L144) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
-| [labels](variables.tf#L151) | Project-level labels. | map(string)
| | {}
|
-| [nodepools](variables.tf#L157) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…})))
| | {}
|
-| [project_services](variables.tf#L197) | Additional project services to enable. | list(string)
| | []
|
+| [folder_id](variables.tf#L138) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string
| ✓ | |
+| [prefix](variables.tf#L189) | Prefix used for resource names. | string
| ✓ | |
+| [project_id](variables.tf#L198) | ID of the project that will contain all the clusters. | string
| ✓ | |
+| [vpc_config](variables.tf#L210) | Shared VPC project and VPC details. | object({…})
| ✓ | |
+| [clusters](variables.tf#L22) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…}))
| | {}
|
+| [fleet_configmanagement_clusters](variables.tf#L76) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string))
| | {}
|
+| [fleet_configmanagement_templates](variables.tf#L83) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…}))
| | {}
|
+| [fleet_features](variables.tf#L118) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…})
| | null
|
+| [fleet_workload_identity](variables.tf#L131) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool
| | false
|
+| [group_iam](variables.tf#L143) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string))
| | {}
|
+| [iam](variables.tf#L150) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
+| [labels](variables.tf#L157) | Project-level labels. | map(string)
| | {}
|
+| [nodepools](variables.tf#L163) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…})))
| | {}
|
+| [project_services](variables.tf#L203) | Additional project services to enable. | list(string)
| | []
|
## Outputs
@@ -268,5 +267,4 @@ module "gke" {
| [cluster_ids](outputs.tf#L17) | Cluster ids. | |
| [clusters](outputs.tf#L24) | Cluster resources. | |
| [project_id](outputs.tf#L29) | GKE project id. | |
-
diff --git a/blueprints/gke/multitenant-fleet/variables.tf b/blueprints/gke/multitenant-fleet/variables.tf
index 37975b60..2461ea8a 100644
--- a/blueprints/gke/multitenant-fleet/variables.tf
+++ b/blueprints/gke/multitenant-fleet/variables.tf
@@ -33,7 +33,13 @@ variable "clusters" {
issue_client_certificate = optional(bool, false)
labels = optional(map(string))
location = string
- logging_config = optional(list(string), ["SYSTEM_COMPONENTS"])
+ logging_config = optional(object({
+ enable_system_logs = optional(bool, true)
+ enable_workloads_logs = optional(bool, true)
+ enable_api_server_logs = optional(bool, false)
+ enable_scheduler_logs = optional(bool, false)
+ enable_controller_manager_logs = optional(bool, false)
+ }), {})
maintenance_config = optional(any, {
daily_window_start_time = "03:00"
recurring_window = null
diff --git a/fast/stages/3-gke-multitenant/dev/README.md b/fast/stages/3-gke-multitenant/dev/README.md
index 35dc7372..23572297 100644
--- a/fast/stages/3-gke-multitenant/dev/README.md
+++ b/fast/stages/3-gke-multitenant/dev/README.md
@@ -149,7 +149,6 @@ Leave all these variables unset (or set to `null`) to disable fleet management.
-
## Files
| name | description | modules | resources |
@@ -164,21 +163,21 @@ Leave all these variables unset (or set to `null`) to disable fleet management.
|---|---|:---:|:---:|:---:|:---:|
| [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…})
| ✓ | | 0-bootstrap
|
| [billing_account](variables.tf#L29) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…})
| ✓ | | 0-bootstrap
|
-| [folder_ids](variables.tf#L153) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…})
| ✓ | | 1-resman
|
-| [host_project_ids](variables.tf#L168) | Host project for the shared VPC. | object({…})
| ✓ | | 2-networking
|
-| [prefix](variables.tf#L221) | Prefix used for resources that need unique names. | string
| ✓ | | |
-| [vpc_self_links](variables.tf#L237) | Self link for the shared VPC. | object({…})
| ✓ | | 2-networking
|
-| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…}))
| | {}
| |
-| [fleet_configmanagement_clusters](variables.tf#L90) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string))
| | {}
| |
-| [fleet_configmanagement_templates](variables.tf#L98) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…}))
| | {}
| |
-| [fleet_features](variables.tf#L133) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…})
| | null
| |
-| [fleet_workload_identity](variables.tf#L146) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool
| | false
| |
-| [group_iam](variables.tf#L161) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string))
| | {}
| |
-| [iam](variables.tf#L176) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
| |
-| [labels](variables.tf#L183) | Project-level labels. | map(string)
| | {}
| |
-| [nodepools](variables.tf#L189) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…})))
| | {}
| |
-| [outputs_location](variables.tf#L215) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string
| | null
| |
-| [project_services](variables.tf#L230) | Additional project services to enable. | list(string)
| | []
| |
+| [folder_ids](variables.tf#L159) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…})
| ✓ | | 1-resman
|
+| [host_project_ids](variables.tf#L174) | Host project for the shared VPC. | object({…})
| ✓ | | 2-networking
|
+| [prefix](variables.tf#L227) | Prefix used for resources that need unique names. | string
| ✓ | | |
+| [vpc_self_links](variables.tf#L243) | Self link for the shared VPC. | object({…})
| ✓ | | 2-networking
|
+| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…}))
| | {}
| |
+| [fleet_configmanagement_clusters](variables.tf#L96) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string))
| | {}
| |
+| [fleet_configmanagement_templates](variables.tf#L104) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…}))
| | {}
| |
+| [fleet_features](variables.tf#L139) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…})
| | null
| |
+| [fleet_workload_identity](variables.tf#L152) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool
| | false
| |
+| [group_iam](variables.tf#L167) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string))
| | {}
| |
+| [iam](variables.tf#L182) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
| |
+| [labels](variables.tf#L189) | Project-level labels. | map(string)
| | {}
| |
+| [nodepools](variables.tf#L195) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…})))
| | {}
| |
+| [outputs_location](variables.tf#L221) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string
| | null
| |
+| [project_services](variables.tf#L236) | Additional project services to enable. | list(string)
| | []
| |
## Outputs
@@ -187,5 +186,4 @@ Leave all these variables unset (or set to `null`) to disable fleet management.
| [cluster_ids](outputs.tf#L57) | Cluster ids. | | |
| [clusters](outputs.tf#L62) | Cluster resources. | ✓ | |
| [project_id](outputs.tf#L68) | GKE project id. | | |
-
diff --git a/fast/stages/3-gke-multitenant/dev/variables.tf b/fast/stages/3-gke-multitenant/dev/variables.tf
index 66936db6..11e32ed6 100644
--- a/fast/stages/3-gke-multitenant/dev/variables.tf
+++ b/fast/stages/3-gke-multitenant/dev/variables.tf
@@ -53,7 +53,13 @@ variable "clusters" {
issue_client_certificate = optional(bool, false)
labels = optional(map(string))
location = string
- logging_config = optional(list(string), ["SYSTEM_COMPONENTS"])
+ logging_config = optional(object({
+ enable_system_logs = optional(bool, true)
+ enable_workloads_logs = optional(bool, true)
+ enable_api_server_logs = optional(bool, false)
+ enable_scheduler_logs = optional(bool, false)
+ enable_controller_manager_logs = optional(bool, false)
+ }), {})
maintenance_config = optional(any, {
daily_window_start_time = "03:00"
recurring_window = null
diff --git a/modules/gke-cluster-standard/README.md b/modules/gke-cluster-standard/README.md
index 40548f62..e80a4e6d 100644
--- a/modules/gke-cluster-standard/README.md
+++ b/modules/gke-cluster-standard/README.md
@@ -71,6 +71,58 @@ module "cluster-1" {
# tftest modules=1 resources=1 inventory=dataplane-v2.yaml
```
+### Managing GKE logs
+
+This example shows you how to [control which logs are sent from your GKE cluster to Cloud Logging](https://cloud.google.com/stackdriver/docs/solutions/gke/installing).
+
+When you create a new GKE cluster, [Cloud Operations for GKE](https://cloud.google.com/stackdriver/docs/solutions/gke) integration with Cloud Logging is enabled by default and [System logs](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#what_logs) are collected. You can enable collection of several other [types of logs](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#what_logs). The following example enables collection of *all* optional logs.
+
+```hcl
+module "cluster-1" {
+ source = "./fabric/modules/gke-cluster-standard"
+ project_id = "myproject"
+ name = "cluster-1"
+ location = "europe-west1-b"
+ vpc_config = {
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }
+ logging_config = {
+ enable_workloads_logs = true
+ enable_api_server_logs = true
+ enable_scheduler_logs = true
+ enable_controller_manager_logs = true
+ }
+}
+# tftest modules=1 resources=1 inventory=logging-config-enable-all.yaml
+```
+
+### Disable GKE logs collection
+
+This example shows how to fully disable logs collection on a GKE Standard cluster. This is not recommended.
+
+> **Warning**
+> If you've disabled Cloud Logging or Cloud Monitoring, GKE customer support
+> is offered on a best-effort basis and might require additional effort
+> from your engineering team.
+
+```hcl
+module "cluster-1" {
+ source = "./fabric/modules/gke-cluster-standard"
+ project_id = "myproject"
+ name = "cluster-1"
+ location = "europe-west1-b"
+ vpc_config = {
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }
+ logging_config = {
+ enable_system_logs = false
+ }
+}
+# tftest modules=1 resources=1 inventory=logging-config-disable-all.yaml
+```
+
### Cloud DNS
This example shows how to [use Cloud DNS as a Kubernetes DNS provider](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns) for GKE Standard clusters.
@@ -97,7 +149,6 @@ module "cluster-1" {
# tftest modules=1 resources=1 inventory=dns.yaml
```
-
### Backup for GKE
This example shows how to [enable the Backup for GKE agent and configure a Backup Plan](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke) for GKE Standard clusters.
@@ -131,9 +182,9 @@ module "cluster-1" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [location](variables.tf#L138) | Cluster zone or region. | string
| ✓ | |
-| [name](variables.tf#L195) | Cluster name. | string
| ✓ | |
-| [project_id](variables.tf#L221) | Cluster project id. | string
| ✓ | |
-| [vpc_config](variables.tf#L238) | VPC-level configuration. | object({…})
| ✓ | |
+| [name](variables.tf#L210) | Cluster name. | string
| ✓ | |
+| [project_id](variables.tf#L236) | Cluster project id. | string
| ✓ | |
+| [vpc_config](variables.tf#L253) | VPC-level configuration. | object({…})
| ✓ | |
| [backup_configs](variables.tf#L17) | Configuration for Backup for GKE. | object({…})
| | {}
|
| [cluster_autoscaling](variables.tf#L37) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…})
| | null
|
| [description](variables.tf#L58) | Cluster description. | string
| | null
|
@@ -141,15 +192,15 @@ module "cluster-1" {
| [enable_features](variables.tf#L87) | Enable cluster-level features. Certain features allow configuration. | object({…})
| | {…}
|
| [issue_client_certificate](variables.tf#L126) | Enable issuing client certificate. | bool
| | false
|
| [labels](variables.tf#L132) | Cluster resource labels. | map(string)
| | null
|
-| [logging_config](variables.tf#L143) | Logging configuration. | list(string)
| | ["SYSTEM_COMPONENTS"]
|
-| [maintenance_config](variables.tf#L149) | Maintenance window configuration. | object({…})
| | {…}
|
-| [max_pods_per_node](variables.tf#L172) | Maximum number of pods per node in this cluster. | number
| | 110
|
-| [min_master_version](variables.tf#L178) | Minimum version of the master, defaults to the version of the most recent official release. | string
| | null
|
-| [monitoring_config](variables.tf#L184) | Monitoring components. | object({…})
| | {…}
|
-| [node_locations](variables.tf#L200) | Zones in which the cluster's nodes are located. | list(string)
| | []
|
-| [private_cluster_config](variables.tf#L207) | Private cluster configuration. | object({…})
| | null
|
-| [release_channel](variables.tf#L226) | Release channel for GKE upgrades. | string
| | null
|
-| [tags](variables.tf#L232) | Network tags applied to nodes. | list(string)
| | null
|
+| [logging_config](variables.tf#L143) | Logging configuration. | object({…})
| | {}
|
+| [maintenance_config](variables.tf#L164) | Maintenance window configuration. | object({…})
| | {…}
|
+| [max_pods_per_node](variables.tf#L187) | Maximum number of pods per node in this cluster. | number
| | 110
|
+| [min_master_version](variables.tf#L193) | Minimum version of the master, defaults to the version of the most recent official release. | string
| | null
|
+| [monitoring_config](variables.tf#L199) | Monitoring components. | object({…})
| | {…}
|
+| [node_locations](variables.tf#L215) | Zones in which the cluster's nodes are located. | list(string)
| | []
|
+| [private_cluster_config](variables.tf#L222) | Private cluster configuration. | object({…})
| | null
|
+| [release_channel](variables.tf#L241) | Release channel for GKE upgrades. | string
| | null
|
+| [tags](variables.tf#L247) | Network tags applied to nodes. | list(string)
| | null
|
## Outputs
diff --git a/modules/gke-cluster-standard/main.tf b/modules/gke-cluster-standard/main.tf
index bbf0d106..8f0df84f 100644
--- a/modules/gke-cluster-standard/main.tf
+++ b/modules/gke-cluster-standard/main.tf
@@ -181,10 +181,27 @@ resource "google_container_cluster" "cluster" {
}
}
+ # Send GKE cluster logs from chosen sources to Cloud Logging.
+ # System logs must be enabled if any other source is enabled.
+ # This is validated by input variable validation rules.
dynamic "logging_config" {
- for_each = var.logging_config != null ? [""] : []
+ for_each = var.logging_config.enable_system_logs ? [""] : []
content {
- enable_components = var.logging_config
+ enable_components = toset(compact([
+ var.logging_config.enable_api_server_logs ? "APISERVER" : null,
+ var.logging_config.enable_controller_manager_logs ? "CONTROLLER_MANAGER" : null,
+ var.logging_config.enable_scheduler_logs ? "SCHEDULER" : null,
+ "SYSTEM_COMPONENTS",
+ var.logging_config.enable_workloads_logs ? "WORKLOADS" : null,
+ ]))
+ }
+ }
+ # Don't send any GKE cluster logs to Cloud Logging. Input variable validation
+ # makes sure every other log source is false when enable_system_logs is false.
+ dynamic "logging_config" {
+ for_each = var.logging_config.enable_system_logs == false ? [""] : []
+ content {
+ enable_components = []
}
}
diff --git a/modules/gke-cluster-standard/variables.tf b/modules/gke-cluster-standard/variables.tf
index 15bce469..b9c4a113 100644
--- a/modules/gke-cluster-standard/variables.tf
+++ b/modules/gke-cluster-standard/variables.tf
@@ -142,8 +142,23 @@ variable "location" {
variable "logging_config" {
description = "Logging configuration."
- type = list(string)
- default = ["SYSTEM_COMPONENTS"]
+ type = object({
+ enable_system_logs = optional(bool, true)
+ enable_workloads_logs = optional(bool, false)
+ enable_api_server_logs = optional(bool, false)
+ enable_scheduler_logs = optional(bool, false)
+ enable_controller_manager_logs = optional(bool, false)
+ })
+ default = {}
+ nullable = false
+ # System logs are the minimum required component for enabling log collection.
+ # So either everything is off (false), or enable_system_logs must be true.
+ validation {
+ condition = (
+ !anytrue(values(var.logging_config)) || var.logging_config.enable_system_logs
+ )
+ error_message = "System logs are the minimum required component for enabling log collection."
+ }
}
variable "maintenance_config" {
diff --git a/tests/modules/gke_cluster_standard/examples/logging-config-disable-all.yaml b/tests/modules/gke_cluster_standard/examples/logging-config-disable-all.yaml
new file mode 100644
index 00000000..0481b8f2
--- /dev/null
+++ b/tests/modules/gke_cluster_standard/examples/logging-config-disable-all.yaml
@@ -0,0 +1,21 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.cluster-1.google_container_cluster.cluster:
+ logging_config:
+ - enable_components: []
+
+counts:
+ google_container_cluster: 1
diff --git a/tests/modules/gke_cluster_standard/examples/logging-config-enable-all.yaml b/tests/modules/gke_cluster_standard/examples/logging-config-enable-all.yaml
new file mode 100644
index 00000000..c6dad6eb
--- /dev/null
+++ b/tests/modules/gke_cluster_standard/examples/logging-config-enable-all.yaml
@@ -0,0 +1,26 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+ module.cluster-1.google_container_cluster.cluster:
+ logging_config:
+ - enable_components:
+ - APISERVER
+ - CONTROLLER_MANAGER
+ - SCHEDULER
+ - SYSTEM_COMPONENTS
+ - WORKLOADS
+
+counts:
+ google_container_cluster: 1