From 2c0f949f07d0563b621cb433c36cf13e117ea561 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Sat, 5 Dec 2020 08:31:35 +0100 Subject: [PATCH] Logging sinks and exclusions (#178) * Add sink support to folder module * Make folder creation optional. * Add logging sinks to the organization module * Add logging sink support to project module * Update readme --- CHANGELOG.md | 1 + modules/folder/README.md | 62 +++++++++++++++++++++- modules/folder/main.tf | 86 ++++++++++++++++++++++++++++--- modules/folder/outputs.tf | 13 +++-- modules/folder/variables.tf | 33 +++++++++++- modules/organization/README.md | 56 ++++++++++++++++++++ modules/organization/main.tf | 63 ++++++++++++++++++++++ modules/organization/outputs.tf | 7 +++ modules/organization/variables.tf | 17 ++++++ modules/project/README.md | 57 ++++++++++++++++++++ modules/project/main.tf | 63 ++++++++++++++++++++++ modules/project/outputs.tf | 7 +++ modules/project/variables.tf | 17 ++++++ 13 files changed, 470 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c858f5d..18c11ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- add support for creating logging sinks and logging exclusions in the `project`, `folder` and `organization` modules ## [4.2.0] - 2020-11-25 diff --git a/modules/folder/README.md b/modules/folder/README.md index 430db3dc..dede3fd2 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -41,6 +41,59 @@ module "folder" { # tftest:modules=1:resources=4 ``` +### Logging Sinks + +```hcl +module "gcs" { + source = "./modules/gcs" + project_id = "my-project" + name = "gcs_sink" + force_destroy = true +} + +module "dataset" { + source = "./modules/bigquery-dataset" + project_id = "my-project" + id = "bq_sink" +} + +module "pubsub" { + source = "./modules/pubsub" + project_id = "my-project" + name = "pubsub_sink" +} + +module "folder-sink" { + source = "./modules/folder" + parent = "folders/657104291943" + name = "my-folder" + logging_sinks = { + warnings = { + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + grant = false + } + info = { + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + grant = false + } + notice = { + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + grant = true + } + } + logging_exclusions = { + no-gce-instances = "resource.type=gce_instance" + } +} +# tftest:modules=4:resources=9 +``` + ### Hierarchical firewall policies ```hcl @@ -88,11 +141,15 @@ module "folder2" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | Folder name. | string | ✓ | | -| parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ | | | *firewall_policies* | Hierarchical firewall policies to *create* in this folder. | map(map(object({...}))) | | {} | | *firewall_policy_attachments* | List of hierarchical firewall policy IDs to *attach* to this folder. | map(string) | | {} | +| *folder_create* | Create folder. When set to false, uses id to reference an existing folder. | bool | | true | | *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | map(set(string)) | | {} | +| *id* | Folder ID in case you use folder_create=false | string | | null | +| *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | +| *logging_sinks* | Logging sinks to create for this folder. | map(object({...})) | | {} | +| *name* | Folder name. | string | | null | +| *parent* | Parent in folders/folder_id or organizations/org_id format. | string | | ... | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | | *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...})) | | {} | @@ -105,4 +162,5 @@ module "folder2" { | folder | Folder resource. | | | id | Folder id. | | | name | Folder name. | | +| sink_writer_identities | None | | diff --git a/modules/folder/main.tf b/modules/folder/main.tf index 6fb50acb..b46eed4f 100644 --- a/modules/folder/main.tf +++ b/modules/folder/main.tf @@ -25,23 +25,50 @@ locals { for rule in local.extended_rules : "${rule.policy}-${rule.name}" => rule } + logging_sinks = coalesce(var.logging_sinks, {}) + sink_type_destination = { + gcs = "storage.googleapis.com" + bigquery = "bigquery.googleapis.com" + pubsub = "pubsub.googleapis.com" + # TODO: add logging buckets support + # logging = "logging.googleapis.com" + } + sink_bindings = { + for type in ["gcs", "bigquery", "pubsub", "logging"] : + type => { + for name, sink in local.logging_sinks : + name => sink + if sink.grant && sink.type == type + } + } + folder = ( + var.folder_create + ? try(google_folder.folder.0, null) + : try(data.google_folder.folder.0, null) + ) +} + +data "google_folder" "folder" { + count = var.folder_create ? 0 : 1 + folder = var.id } resource "google_folder" "folder" { + count = var.folder_create ? 1 : 0 display_name = var.name parent = var.parent } resource "google_folder_iam_binding" "authoritative" { for_each = var.iam - folder = google_folder.folder.name + folder = local.folder.name role = each.key members = each.value } resource "google_folder_organization_policy" "boolean" { for_each = var.policy_boolean - folder = google_folder.folder.name + folder = local.folder.name constraint = each.key dynamic boolean_policy { @@ -62,7 +89,7 @@ resource "google_folder_organization_policy" "boolean" { resource "google_folder_organization_policy" "list" { for_each = var.policy_list - folder = google_folder.folder.name + folder = local.folder.name constraint = each.key dynamic list_policy { @@ -117,7 +144,7 @@ resource "google_compute_organization_security_policy" "policy" { for_each = var.firewall_policies display_name = each.key - parent = google_folder.folder.id + parent = local.folder.id } resource "google_compute_organization_security_policy_rule" "rule" { @@ -152,7 +179,54 @@ resource "google_compute_organization_security_policy_rule" "rule" { resource "google_compute_organization_security_policy_association" "attachment" { provider = google-beta for_each = var.firewall_policy_attachments - name = "${google_folder.folder.id}-${each.key}" - attachment_id = google_folder.folder.id + name = "${local.folder.id}-${each.key}" + attachment_id = local.folder.id policy_id = each.value } + +resource "google_logging_folder_sink" "sink" { + for_each = local.logging_sinks + name = each.key + #description = "${each.key} (Terraform-managed)" + folder = local.folder.name + destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" + filter = each.value.filter +} + +resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { + for_each = local.sink_bindings["gcs"] + bucket = each.value.destination + role = "roles/storage.objectCreator" + members = [google_logging_folder_sink.sink[each.key].writer_identity] +} + +resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" { + for_each = local.sink_bindings["bigquery"] + project = split("/", each.value.destination)[1] + dataset_id = split("/", each.value.destination)[3] + role = "roles/bigquery.dataEditor" + members = [google_logging_folder_sink.sink[each.key].writer_identity] +} + +resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" { + for_each = local.sink_bindings["pubsub"] + project = split("/", each.value.destination)[1] + topic = split("/", each.value.destination)[3] + role = "roles/pubsub.publisher" + members = [google_logging_folder_sink.sink[each.key].writer_identity] +} + +# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" { +# for_each = local.sink_grants["gcs"] +# bucket = each.value.destination +# role = "roles/storage.objectCreator" +# members = [google_logging_folder_sink.sink[each.key].writer_identity] +# } + +resource "google_logging_folder_exclusion" "logging-exclusion" { + for_each = coalesce(var.logging_exclusions, {}) + name = each.key + folder = local.folder.name + description = "${each.key} (Terraform-managed)" + filter = each.value +} diff --git a/modules/folder/outputs.tf b/modules/folder/outputs.tf index c521367f..6af634ab 100644 --- a/modules/folder/outputs.tf +++ b/modules/folder/outputs.tf @@ -16,12 +16,12 @@ output "folder" { description = "Folder resource." - value = google_folder.folder + value = local.folder } output "id" { description = "Folder id." - value = google_folder.folder.name + value = local.folder.name depends_on = [ google_folder_iam_binding.authoritative, google_folder_organization_policy.boolean, @@ -31,7 +31,7 @@ output "id" { output "name" { description = "Folder name." - value = google_folder.folder.display_name + value = local.folder.display_name } output "firewall_policies" { @@ -49,3 +49,10 @@ output "firewall_policy_id" { name => google_compute_organization_security_policy.policy[name].id } } + +output "sink_writer_identities" { + description = "" + value = { + for name, sink in google_logging_folder_sink.sink : name => sink.writer_identity + } +} diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf index aba267e6..fc5ff7eb 100644 --- a/modules/folder/variables.tf +++ b/modules/folder/variables.tf @@ -23,13 +23,15 @@ variable "iam" { variable "name" { description = "Folder name." type = string + default = null } variable "parent" { description = "Parent in folders/folder_id or organizations/org_id format." type = string + default = null validation { - condition = can(regex("(organizations|folders)/[0-9]+", var.parent)) + condition = var.parent == null || can(regex("(organizations|folders)/[0-9]+", var.parent)) error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." } } @@ -72,3 +74,32 @@ variable "firewall_policy_attachments" { type = map(string) default = {} } + +variable "logging_sinks" { + description = "Logging sinks to create for this folder." + type = map(object({ + destination = string + type = string + filter = string + grant = bool + })) + default = {} +} + +variable "logging_exclusions" { + description = "Logging exclusions for this folder in the form {NAME -> FILTER}." + type = map(string) + default = {} +} + +variable "folder_create" { + description = "Create folder. When set to false, uses id to reference an existing folder." + type = bool + default = true +} + +variable "id" { + description = "Folder ID in case you use folder_create=false" + type = string + default = null +} diff --git a/modules/organization/README.md b/modules/organization/README.md index a30b161d..f8b500fb 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -59,6 +59,59 @@ module "org" { # tftest:modules=1:resources=3 ``` +## Logging Sinks +```hcl +module "gcs" { + source = "./modules/gcs" + project_id = var.project_id + name = "gcs_sink" + force_destroy = true +} + +module "dataset" { + source = "./modules/bigquery-dataset" + project_id = var.project_id + id = "bq_sink" +} + +module "pubsub" { + source = "./modules/pubsub" + project_id = var.project_id + name = "pubsub_sink" +} + +module "org" { + source = "./modules/organization" + organization_id = var.organization_id + + logging_sinks = { + warnings = { + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + grant = false + } + info = { + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + grant = false + } + notice = { + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + grant = true + } + } + logging_exclusions = { + no-gce-instances = "resource.type=gce_instance" + } +} +# tftest:modules=4:resources=8 +``` + + ## Variables @@ -72,6 +125,8 @@ module "org" { | *iam_additive* | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | | *iam_audit_config* | Service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. | map(map(list(string))) | | {} | +| *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | +| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | | *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...})) | | {} | @@ -82,4 +137,5 @@ module "org" { | firewall_policies | Map of firewall policy resources created in the organization. | | | firewall_policy_id | Map of firewall policy ids created in the organization. | | | organization_id | Organization id dependent on module resources. | | +| sink_writer_identities | None | | diff --git a/modules/organization/main.tf b/modules/organization/main.tf index f3b75166..a764710c 100644 --- a/modules/organization/main.tf +++ b/modules/organization/main.tf @@ -40,6 +40,22 @@ locals { for rule in local.extended_rules : "${rule.policy}-${rule.name}" => rule } + logging_sinks = coalesce(var.logging_sinks, {}) + sink_type_destination = { + gcs = "storage.googleapis.com" + bigquery = "bigquery.googleapis.com" + pubsub = "pubsub.googleapis.com" + # TODO: add logging buckets support + # logging = "logging.googleapis.com" + } + sink_bindings = { + for type in ["gcs", "bigquery", "pubsub", "logging"] : + type => { + for name, sink in local.logging_sinks : + name => sink + if sink.grant && sink.type == type + } + } } resource "google_organization_iam_custom_role" "roles" { @@ -200,3 +216,50 @@ resource "google_compute_organization_security_policy_association" "attachment" attachment_id = var.organization_id policy_id = each.value } + +resource "google_logging_organization_sink" "sink" { + for_each = local.logging_sinks + name = each.key + #description = "${each.key} (Terraform-managed)" + org_id = local.organization_id_numeric + destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" + filter = each.value.filter +} + +resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { + for_each = local.sink_bindings["gcs"] + bucket = each.value.destination + role = "roles/storage.objectCreator" + members = [google_logging_organization_sink.sink[each.key].writer_identity] +} + +resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" { + for_each = local.sink_bindings["bigquery"] + project = split("/", each.value.destination)[1] + dataset_id = split("/", each.value.destination)[3] + role = "roles/bigquery.dataEditor" + members = [google_logging_organization_sink.sink[each.key].writer_identity] +} + +resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" { + for_each = local.sink_bindings["pubsub"] + project = split("/", each.value.destination)[1] + topic = split("/", each.value.destination)[3] + role = "roles/pubsub.publisher" + members = [google_logging_organization_sink.sink[each.key].writer_identity] +} + +# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" { +# for_each = local.sink_grants["gcs"] +# bucket = each.value.destination +# role = "roles/storage.objectCreator" +# members = [google_logging_organization_sink.sink[each.key].writer_identity] +# } + +resource "google_logging_organization_exclusion" "logging-exclusion" { + for_each = coalesce(var.logging_exclusions, {}) + name = each.key + org_id = local.organization_id_numeric + description = "${each.key} (Terraform-managed)" + filter = each.value +} diff --git a/modules/organization/outputs.tf b/modules/organization/outputs.tf index dd0d0294..869f8185 100644 --- a/modules/organization/outputs.tf +++ b/modules/organization/outputs.tf @@ -42,3 +42,10 @@ output "firewall_policy_id" { name => google_compute_organization_security_policy.policy[name].id } } + +output "sink_writer_identities" { + description = "" + value = { + for name, sink in google_logging_organization_sink.sink : name => sink.writer_identity + } +} diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index ea7b5f52..5c426cf8 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -98,3 +98,20 @@ variable "firewall_policy_attachments" { type = map(string) default = {} } + +variable "logging_sinks" { + description = "Logging sinks to create for this organization." + type = map(object({ + destination = string + type = string + filter = string + grant = bool + })) + default = {} +} + +variable "logging_exclusions" { + description = "Logging exclusions for this organization in the form {NAME -> FILTER}." + type = map(string) + default = {} +} diff --git a/modules/project/README.md b/modules/project/README.md index 39871625..f8d238d1 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -84,6 +84,60 @@ module "project" { # tftest:modules=1:resources=6 ``` +## Logging Sinks +```hcl +module "gcs" { + source = "./modules/gcs" + project_id = var.project_id + name = "gcs_sink" + force_destroy = true +} + +module "dataset" { + source = "./modules/bigquery-dataset" + project_id = var.project_id + id = "bq_sink" +} + +module "pubsub" { + source = "./modules/pubsub" + project_id = var.project_id + name = "pubsub_sink" +} + +module "project-host" { + source = "./modules/project" + name = "my-project" + billing_account = "123456-123456-123456" + parent = "folders/1234567890" + logging_sinks = { + warnings = { + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + grant = false + } + info = { + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + grant = false + } + notice = { + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + grant = true + } + } + logging_exclusions = { + no-gce-instances = "resource.type=gce_instance" + } +} +# tftest:modules=4:resources=9 +``` + + ## Variables @@ -98,6 +152,8 @@ module "project" { | *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | | *labels* | Resource labels. | map(string) | | {} | | *lien_reason* | If non-empty, creates a project lien with this description. | string | | | +| *logging_exclusions* | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| *logging_sinks* | Logging sinks to create for this project. | map(object({...})) | | {} | | *oslogin* | Enable OS Login. | bool | | false | | *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | | *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | @@ -120,5 +176,6 @@ module "project" { | number | Project number. | | | project_id | Project id. | | | service_accounts | Product robot service accounts in project. | | +| sink_writer_identities | None | | diff --git a/modules/project/main.tf b/modules/project/main.tf index 9e5502ae..0606fa24 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -37,6 +37,22 @@ locals { ? try(google_project.project.0, null) : try(data.google_project.project.0, null) ) + logging_sinks = coalesce(var.logging_sinks, {}) + sink_type_destination = { + gcs = "storage.googleapis.com" + bigquery = "bigquery.googleapis.com" + pubsub = "pubsub.googleapis.com" + # TODO: add logging buckets support + # logging = "logging.googleapis.com" + } + sink_bindings = { + for type in ["gcs", "bigquery", "pubsub", "logging"] : + type => { + for name, sink in local.logging_sinks : + name => sink + if sink.grant && sink.type == type + } + } } data "google_project" "project" { @@ -242,3 +258,50 @@ resource "google_compute_shared_vpc_service_project" "shared_vpc_service" { host_project = var.shared_vpc_service_config.host_project service_project = local.project.project_id } + +resource "google_logging_project_sink" "sink" { + for_each = local.logging_sinks + name = each.key + #description = "${each.key} (Terraform-managed)" + project = local.project.project_id + destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" + filter = each.value.filter +} + +resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { + for_each = local.sink_bindings["gcs"] + bucket = each.value.destination + role = "roles/storage.objectCreator" + members = [google_logging_project_sink.sink[each.key].writer_identity] +} + +resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" { + for_each = local.sink_bindings["bigquery"] + project = split("/", each.value.destination)[1] + dataset_id = split("/", each.value.destination)[3] + role = "roles/bigquery.dataEditor" + members = [google_logging_project_sink.sink[each.key].writer_identity] +} + +resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" { + for_each = local.sink_bindings["pubsub"] + project = split("/", each.value.destination)[1] + topic = split("/", each.value.destination)[3] + role = "roles/pubsub.publisher" + members = [google_logging_project_sink.sink[each.key].writer_identity] +} + +# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" { +# for_each = local.sink_grants["gcs"] +# bucket = each.value.destination +# role = "roles/storage.objectCreator" +# members = [google_logging_project_sink.sink[each.key].writer_identity] +# } + +resource "google_logging_project_exclusion" "logging-exclusion" { + for_each = coalesce(var.logging_exclusions, {}) + name = each.key + project = local.project.project_id + description = "${each.key} (Terraform-managed)" + filter = each.value +} diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf index f20c78b0..b11d362c 100644 --- a/modules/project/outputs.tf +++ b/modules/project/outputs.tf @@ -64,3 +64,10 @@ output "custom_roles" { name => role.id } } + +output "sink_writer_identities" { + description = "" + value = { + for name, sink in google_logging_project_sink.sink : name => sink.writer_identity + } +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf index feb3b21a..d8e06a9e 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -165,3 +165,20 @@ variable "shared_vpc_service_config" { host_project = "" } } + +variable "logging_sinks" { + description = "Logging sinks to create for this project." + type = map(object({ + destination = string + type = string + filter = string + grant = bool + })) + default = {} +} + +variable "logging_exclusions" { + description = "Logging exclusions for this project in the form {NAME -> FILTER}." + type = map(string) + default = {} +}