diff --git a/blueprints/gke/multitenant-fleet/README.md b/blueprints/gke/multitenant-fleet/README.md index d36faa25..baaf288f 100644 --- a/blueprints/gke/multitenant-fleet/README.md +++ b/blueprints/gke/multitenant-fleet/README.md @@ -229,7 +229,6 @@ module "gke" { - ## Files | name | description | modules | @@ -246,20 +245,20 @@ module "gke" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | -| [folder_id](variables.tf#L132) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | -| [prefix](variables.tf#L183) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L192) | ID of the project that will contain all the clusters. | string | ✓ | | -| [vpc_config](variables.tf#L204) | Shared VPC project and VPC details. | object({…}) | ✓ | | -| [clusters](variables.tf#L22) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | -| [fleet_configmanagement_clusters](variables.tf#L70) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | -| [fleet_configmanagement_templates](variables.tf#L77) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | -| [fleet_features](variables.tf#L112) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | -| [fleet_workload_identity](variables.tf#L125) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | -| [group_iam](variables.tf#L137) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | -| [iam](variables.tf#L144) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [labels](variables.tf#L151) | Project-level labels. | map(string) | | {} | -| [nodepools](variables.tf#L157) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | -| [project_services](variables.tf#L197) | Additional project services to enable. | list(string) | | [] | +| [folder_id](variables.tf#L138) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | +| [prefix](variables.tf#L189) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L198) | ID of the project that will contain all the clusters. | string | ✓ | | +| [vpc_config](variables.tf#L210) | Shared VPC project and VPC details. | object({…}) | ✓ | | +| [clusters](variables.tf#L22) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | +| [fleet_configmanagement_clusters](variables.tf#L76) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | +| [fleet_configmanagement_templates](variables.tf#L83) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | +| [fleet_features](variables.tf#L118) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | +| [fleet_workload_identity](variables.tf#L131) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | +| [group_iam](variables.tf#L143) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | +| [iam](variables.tf#L150) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L157) | Project-level labels. | map(string) | | {} | +| [nodepools](variables.tf#L163) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | +| [project_services](variables.tf#L203) | Additional project services to enable. | list(string) | | [] | ## Outputs @@ -268,5 +267,4 @@ module "gke" { | [cluster_ids](outputs.tf#L17) | Cluster ids. | | | [clusters](outputs.tf#L24) | Cluster resources. | | | [project_id](outputs.tf#L29) | GKE project id. | | - diff --git a/blueprints/gke/multitenant-fleet/variables.tf b/blueprints/gke/multitenant-fleet/variables.tf index 37975b60..2461ea8a 100644 --- a/blueprints/gke/multitenant-fleet/variables.tf +++ b/blueprints/gke/multitenant-fleet/variables.tf @@ -33,7 +33,13 @@ variable "clusters" { issue_client_certificate = optional(bool, false) labels = optional(map(string)) location = string - logging_config = optional(list(string), ["SYSTEM_COMPONENTS"]) + logging_config = optional(object({ + enable_system_logs = optional(bool, true) + enable_workloads_logs = optional(bool, true) + enable_api_server_logs = optional(bool, false) + enable_scheduler_logs = optional(bool, false) + enable_controller_manager_logs = optional(bool, false) + }), {}) maintenance_config = optional(any, { daily_window_start_time = "03:00" recurring_window = null diff --git a/fast/stages/3-gke-multitenant/dev/README.md b/fast/stages/3-gke-multitenant/dev/README.md index 35dc7372..23572297 100644 --- a/fast/stages/3-gke-multitenant/dev/README.md +++ b/fast/stages/3-gke-multitenant/dev/README.md @@ -149,7 +149,6 @@ Leave all these variables unset (or set to `null`) to disable fleet management. - ## Files | name | description | modules | resources | @@ -164,21 +163,21 @@ Leave all these variables unset (or set to `null`) to disable fleet management. |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | | [billing_account](variables.tf#L29) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | -| [folder_ids](variables.tf#L153) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman | -| [host_project_ids](variables.tf#L168) | Host project for the shared VPC. | object({…}) | ✓ | | 2-networking | -| [prefix](variables.tf#L221) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L237) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking | -| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | | -| [fleet_configmanagement_clusters](variables.tf#L90) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L98) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L133) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L146) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | -| [group_iam](variables.tf#L161) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [iam](variables.tf#L176) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [labels](variables.tf#L183) | Project-level labels. | map(string) | | {} | | -| [nodepools](variables.tf#L189) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | -| [outputs_location](variables.tf#L215) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L230) | Additional project services to enable. | list(string) | | [] | | +| [folder_ids](variables.tf#L159) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman | +| [host_project_ids](variables.tf#L174) | Host project for the shared VPC. | object({…}) | ✓ | | 2-networking | +| [prefix](variables.tf#L227) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L243) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking | +| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | | +| [fleet_configmanagement_clusters](variables.tf#L96) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L104) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L139) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L152) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | +| [group_iam](variables.tf#L167) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L182) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L189) | Project-level labels. | map(string) | | {} | | +| [nodepools](variables.tf#L195) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | +| [outputs_location](variables.tf#L221) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L236) | Additional project services to enable. | list(string) | | [] | | ## Outputs @@ -187,5 +186,4 @@ Leave all these variables unset (or set to `null`) to disable fleet management. | [cluster_ids](outputs.tf#L57) | Cluster ids. | | | | [clusters](outputs.tf#L62) | Cluster resources. | ✓ | | | [project_id](outputs.tf#L68) | GKE project id. | | | - diff --git a/fast/stages/3-gke-multitenant/dev/variables.tf b/fast/stages/3-gke-multitenant/dev/variables.tf index 66936db6..11e32ed6 100644 --- a/fast/stages/3-gke-multitenant/dev/variables.tf +++ b/fast/stages/3-gke-multitenant/dev/variables.tf @@ -53,7 +53,13 @@ variable "clusters" { issue_client_certificate = optional(bool, false) labels = optional(map(string)) location = string - logging_config = optional(list(string), ["SYSTEM_COMPONENTS"]) + logging_config = optional(object({ + enable_system_logs = optional(bool, true) + enable_workloads_logs = optional(bool, true) + enable_api_server_logs = optional(bool, false) + enable_scheduler_logs = optional(bool, false) + enable_controller_manager_logs = optional(bool, false) + }), {}) maintenance_config = optional(any, { daily_window_start_time = "03:00" recurring_window = null diff --git a/modules/gke-cluster-standard/README.md b/modules/gke-cluster-standard/README.md index 40548f62..e80a4e6d 100644 --- a/modules/gke-cluster-standard/README.md +++ b/modules/gke-cluster-standard/README.md @@ -71,6 +71,58 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=dataplane-v2.yaml ``` +### Managing GKE logs + +This example shows you how to [control which logs are sent from your GKE cluster to Cloud Logging](https://cloud.google.com/stackdriver/docs/solutions/gke/installing). + +When you create a new GKE cluster, [Cloud Operations for GKE](https://cloud.google.com/stackdriver/docs/solutions/gke) integration with Cloud Logging is enabled by default and [System logs](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#what_logs) are collected. You can enable collection of several other [types of logs](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#what_logs). The following example enables collection of *all* optional logs. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + logging_config = { + enable_workloads_logs = true + enable_api_server_logs = true + enable_scheduler_logs = true + enable_controller_manager_logs = true + } +} +# tftest modules=1 resources=1 inventory=logging-config-enable-all.yaml +``` + +### Disable GKE logs collection + +This example shows how to fully disable logs collection on a GKE Standard cluster. This is not recommended. + +> **Warning** +> If you've disabled Cloud Logging or Cloud Monitoring, GKE customer support +> is offered on a best-effort basis and might require additional effort +> from your engineering team. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + logging_config = { + enable_system_logs = false + } +} +# tftest modules=1 resources=1 inventory=logging-config-disable-all.yaml +``` + ### Cloud DNS This example shows how to [use Cloud DNS as a Kubernetes DNS provider](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns) for GKE Standard clusters. @@ -97,7 +149,6 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=dns.yaml ``` - ### Backup for GKE This example shows how to [enable the Backup for GKE agent and configure a Backup Plan](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke) for GKE Standard clusters. @@ -131,9 +182,9 @@ module "cluster-1" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [location](variables.tf#L138) | Cluster zone or region. | string | ✓ | | -| [name](variables.tf#L195) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L221) | Cluster project id. | string | ✓ | | -| [vpc_config](variables.tf#L238) | VPC-level configuration. | object({…}) | ✓ | | +| [name](variables.tf#L210) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L236) | Cluster project id. | string | ✓ | | +| [vpc_config](variables.tf#L253) | VPC-level configuration. | object({…}) | ✓ | | | [backup_configs](variables.tf#L17) | Configuration for Backup for GKE. | object({…}) | | {} | | [cluster_autoscaling](variables.tf#L37) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | null | | [description](variables.tf#L58) | Cluster description. | string | | null | @@ -141,15 +192,15 @@ module "cluster-1" { | [enable_features](variables.tf#L87) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {…} | | [issue_client_certificate](variables.tf#L126) | Enable issuing client certificate. | bool | | false | | [labels](variables.tf#L132) | Cluster resource labels. | map(string) | | null | -| [logging_config](variables.tf#L143) | Logging configuration. | list(string) | | ["SYSTEM_COMPONENTS"] | -| [maintenance_config](variables.tf#L149) | Maintenance window configuration. | object({…}) | | {…} | -| [max_pods_per_node](variables.tf#L172) | Maximum number of pods per node in this cluster. | number | | 110 | -| [min_master_version](variables.tf#L178) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | -| [monitoring_config](variables.tf#L184) | Monitoring components. | object({…}) | | {…} | -| [node_locations](variables.tf#L200) | Zones in which the cluster's nodes are located. | list(string) | | [] | -| [private_cluster_config](variables.tf#L207) | Private cluster configuration. | object({…}) | | null | -| [release_channel](variables.tf#L226) | Release channel for GKE upgrades. | string | | null | -| [tags](variables.tf#L232) | Network tags applied to nodes. | list(string) | | null | +| [logging_config](variables.tf#L143) | Logging configuration. | object({…}) | | {} | +| [maintenance_config](variables.tf#L164) | Maintenance window configuration. | object({…}) | | {…} | +| [max_pods_per_node](variables.tf#L187) | Maximum number of pods per node in this cluster. | number | | 110 | +| [min_master_version](variables.tf#L193) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L199) | Monitoring components. | object({…}) | | {…} | +| [node_locations](variables.tf#L215) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [private_cluster_config](variables.tf#L222) | Private cluster configuration. | object({…}) | | null | +| [release_channel](variables.tf#L241) | Release channel for GKE upgrades. | string | | null | +| [tags](variables.tf#L247) | Network tags applied to nodes. | list(string) | | null | ## Outputs diff --git a/modules/gke-cluster-standard/main.tf b/modules/gke-cluster-standard/main.tf index bbf0d106..8f0df84f 100644 --- a/modules/gke-cluster-standard/main.tf +++ b/modules/gke-cluster-standard/main.tf @@ -181,10 +181,27 @@ resource "google_container_cluster" "cluster" { } } + # Send GKE cluster logs from chosen sources to Cloud Logging. + # System logs must be enabled if any other source is enabled. + # This is validated by input variable validation rules. dynamic "logging_config" { - for_each = var.logging_config != null ? [""] : [] + for_each = var.logging_config.enable_system_logs ? [""] : [] content { - enable_components = var.logging_config + enable_components = toset(compact([ + var.logging_config.enable_api_server_logs ? "APISERVER" : null, + var.logging_config.enable_controller_manager_logs ? "CONTROLLER_MANAGER" : null, + var.logging_config.enable_scheduler_logs ? "SCHEDULER" : null, + "SYSTEM_COMPONENTS", + var.logging_config.enable_workloads_logs ? "WORKLOADS" : null, + ])) + } + } + # Don't send any GKE cluster logs to Cloud Logging. Input variable validation + # makes sure every other log source is false when enable_system_logs is false. + dynamic "logging_config" { + for_each = var.logging_config.enable_system_logs == false ? [""] : [] + content { + enable_components = [] } } diff --git a/modules/gke-cluster-standard/variables.tf b/modules/gke-cluster-standard/variables.tf index 15bce469..b9c4a113 100644 --- a/modules/gke-cluster-standard/variables.tf +++ b/modules/gke-cluster-standard/variables.tf @@ -142,8 +142,23 @@ variable "location" { variable "logging_config" { description = "Logging configuration." - type = list(string) - default = ["SYSTEM_COMPONENTS"] + type = object({ + enable_system_logs = optional(bool, true) + enable_workloads_logs = optional(bool, false) + enable_api_server_logs = optional(bool, false) + enable_scheduler_logs = optional(bool, false) + enable_controller_manager_logs = optional(bool, false) + }) + default = {} + nullable = false + # System logs are the minimum required component for enabling log collection. + # So either everything is off (false), or enable_system_logs must be true. + validation { + condition = ( + !anytrue(values(var.logging_config)) || var.logging_config.enable_system_logs + ) + error_message = "System logs are the minimum required component for enabling log collection." + } } variable "maintenance_config" { diff --git a/tests/modules/gke_cluster_standard/examples/logging-config-disable-all.yaml b/tests/modules/gke_cluster_standard/examples/logging-config-disable-all.yaml new file mode 100644 index 00000000..0481b8f2 --- /dev/null +++ b/tests/modules/gke_cluster_standard/examples/logging-config-disable-all.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.cluster-1.google_container_cluster.cluster: + logging_config: + - enable_components: [] + +counts: + google_container_cluster: 1 diff --git a/tests/modules/gke_cluster_standard/examples/logging-config-enable-all.yaml b/tests/modules/gke_cluster_standard/examples/logging-config-enable-all.yaml new file mode 100644 index 00000000..c6dad6eb --- /dev/null +++ b/tests/modules/gke_cluster_standard/examples/logging-config-enable-all.yaml @@ -0,0 +1,26 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.cluster-1.google_container_cluster.cluster: + logging_config: + - enable_components: + - APISERVER + - CONTROLLER_MANAGER + - SCHEDULER + - SYSTEM_COMPONENTS + - WORKLOADS + +counts: + google_container_cluster: 1