diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7c858f5d..18c11ff0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
All notable changes to this project will be documented in this file.
## [Unreleased]
+- add support for creating logging sinks and logging exclusions in the `project`, `folder` and `organization` modules
## [4.2.0] - 2020-11-25
diff --git a/modules/folder/README.md b/modules/folder/README.md
index 430db3dc..dede3fd2 100644
--- a/modules/folder/README.md
+++ b/modules/folder/README.md
@@ -41,6 +41,59 @@ module "folder" {
# tftest:modules=1:resources=4
```
+### Logging Sinks
+
+```hcl
+module "gcs" {
+ source = "./modules/gcs"
+ project_id = "my-project"
+ name = "gcs_sink"
+ force_destroy = true
+}
+
+module "dataset" {
+ source = "./modules/bigquery-dataset"
+ project_id = "my-project"
+ id = "bq_sink"
+}
+
+module "pubsub" {
+ source = "./modules/pubsub"
+ project_id = "my-project"
+ name = "pubsub_sink"
+}
+
+module "folder-sink" {
+ source = "./modules/folder"
+ parent = "folders/657104291943"
+ name = "my-folder"
+ logging_sinks = {
+ warnings = {
+ type = "gcs"
+ destination = module.gcs.name
+ filter = "severity=WARNING"
+ grant = false
+ }
+ info = {
+ type = "bigquery"
+ destination = module.dataset.id
+ filter = "severity=INFO"
+ grant = false
+ }
+ notice = {
+ type = "pubsub"
+ destination = module.pubsub.id
+ filter = "severity=NOTICE"
+ grant = true
+ }
+ }
+ logging_exclusions = {
+ no-gce-instances = "resource.type=gce_instance"
+ }
+}
+# tftest:modules=4:resources=9
+```
+
### Hierarchical firewall policies
```hcl
@@ -88,11 +141,15 @@ module "folder2" {
| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
-| name | Folder name. | string
| ✓ | |
-| parent | Parent in folders/folder_id or organizations/org_id format. | string
| ✓ | |
| *firewall_policies* | Hierarchical firewall policies to *create* in this folder. | map(map(object({...})))
| | {}
|
| *firewall_policy_attachments* | List of hierarchical firewall policy IDs to *attach* to this folder. | map(string)
| | {}
|
+| *folder_create* | Create folder. When set to false, uses id to reference an existing folder. | bool
| | true
|
| *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | map(set(string))
| | {}
|
+| *id* | Folder ID in case you use folder_create=false | string
| | null
|
+| *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string)
| | {}
|
+| *logging_sinks* | Logging sinks to create for this folder. | map(object({...}))
| | {}
|
+| *name* | Folder name. | string
| | null
|
+| *parent* | Parent in folders/folder_id or organizations/org_id format. | string
| | ...
|
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool)
| | {}
|
| *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...}))
| | {}
|
@@ -105,4 +162,5 @@ module "folder2" {
| folder | Folder resource. | |
| id | Folder id. | |
| name | Folder name. | |
+| sink_writer_identities | None | |
diff --git a/modules/folder/main.tf b/modules/folder/main.tf
index 6fb50acb..b46eed4f 100644
--- a/modules/folder/main.tf
+++ b/modules/folder/main.tf
@@ -25,23 +25,50 @@ locals {
for rule in local.extended_rules :
"${rule.policy}-${rule.name}" => rule
}
+ logging_sinks = coalesce(var.logging_sinks, {})
+ sink_type_destination = {
+ gcs = "storage.googleapis.com"
+ bigquery = "bigquery.googleapis.com"
+ pubsub = "pubsub.googleapis.com"
+ # TODO: add logging buckets support
+ # logging = "logging.googleapis.com"
+ }
+ sink_bindings = {
+ for type in ["gcs", "bigquery", "pubsub", "logging"] :
+ type => {
+ for name, sink in local.logging_sinks :
+ name => sink
+ if sink.grant && sink.type == type
+ }
+ }
+ folder = (
+ var.folder_create
+ ? try(google_folder.folder.0, null)
+ : try(data.google_folder.folder.0, null)
+ )
+}
+
+data "google_folder" "folder" {
+ count = var.folder_create ? 0 : 1
+ folder = var.id
}
resource "google_folder" "folder" {
+ count = var.folder_create ? 1 : 0
display_name = var.name
parent = var.parent
}
resource "google_folder_iam_binding" "authoritative" {
for_each = var.iam
- folder = google_folder.folder.name
+ folder = local.folder.name
role = each.key
members = each.value
}
resource "google_folder_organization_policy" "boolean" {
for_each = var.policy_boolean
- folder = google_folder.folder.name
+ folder = local.folder.name
constraint = each.key
dynamic boolean_policy {
@@ -62,7 +89,7 @@ resource "google_folder_organization_policy" "boolean" {
resource "google_folder_organization_policy" "list" {
for_each = var.policy_list
- folder = google_folder.folder.name
+ folder = local.folder.name
constraint = each.key
dynamic list_policy {
@@ -117,7 +144,7 @@ resource "google_compute_organization_security_policy" "policy" {
for_each = var.firewall_policies
display_name = each.key
- parent = google_folder.folder.id
+ parent = local.folder.id
}
resource "google_compute_organization_security_policy_rule" "rule" {
@@ -152,7 +179,54 @@ resource "google_compute_organization_security_policy_rule" "rule" {
resource "google_compute_organization_security_policy_association" "attachment" {
provider = google-beta
for_each = var.firewall_policy_attachments
- name = "${google_folder.folder.id}-${each.key}"
- attachment_id = google_folder.folder.id
+ name = "${local.folder.id}-${each.key}"
+ attachment_id = local.folder.id
policy_id = each.value
}
+
+resource "google_logging_folder_sink" "sink" {
+ for_each = local.logging_sinks
+ name = each.key
+ #description = "${each.key} (Terraform-managed)"
+ folder = local.folder.name
+ destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
+ filter = each.value.filter
+}
+
+resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
+ for_each = local.sink_bindings["gcs"]
+ bucket = each.value.destination
+ role = "roles/storage.objectCreator"
+ members = [google_logging_folder_sink.sink[each.key].writer_identity]
+}
+
+resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" {
+ for_each = local.sink_bindings["bigquery"]
+ project = split("/", each.value.destination)[1]
+ dataset_id = split("/", each.value.destination)[3]
+ role = "roles/bigquery.dataEditor"
+ members = [google_logging_folder_sink.sink[each.key].writer_identity]
+}
+
+resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" {
+ for_each = local.sink_bindings["pubsub"]
+ project = split("/", each.value.destination)[1]
+ topic = split("/", each.value.destination)[3]
+ role = "roles/pubsub.publisher"
+ members = [google_logging_folder_sink.sink[each.key].writer_identity]
+}
+
+# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" {
+# for_each = local.sink_grants["gcs"]
+# bucket = each.value.destination
+# role = "roles/storage.objectCreator"
+# members = [google_logging_folder_sink.sink[each.key].writer_identity]
+# }
+
+resource "google_logging_folder_exclusion" "logging-exclusion" {
+ for_each = coalesce(var.logging_exclusions, {})
+ name = each.key
+ folder = local.folder.name
+ description = "${each.key} (Terraform-managed)"
+ filter = each.value
+}
diff --git a/modules/folder/outputs.tf b/modules/folder/outputs.tf
index c521367f..6af634ab 100644
--- a/modules/folder/outputs.tf
+++ b/modules/folder/outputs.tf
@@ -16,12 +16,12 @@
output "folder" {
description = "Folder resource."
- value = google_folder.folder
+ value = local.folder
}
output "id" {
description = "Folder id."
- value = google_folder.folder.name
+ value = local.folder.name
depends_on = [
google_folder_iam_binding.authoritative,
google_folder_organization_policy.boolean,
@@ -31,7 +31,7 @@ output "id" {
output "name" {
description = "Folder name."
- value = google_folder.folder.display_name
+ value = local.folder.display_name
}
output "firewall_policies" {
@@ -49,3 +49,10 @@ output "firewall_policy_id" {
name => google_compute_organization_security_policy.policy[name].id
}
}
+
+output "sink_writer_identities" {
+ description = ""
+ value = {
+ for name, sink in google_logging_folder_sink.sink : name => sink.writer_identity
+ }
+}
diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf
index aba267e6..fc5ff7eb 100644
--- a/modules/folder/variables.tf
+++ b/modules/folder/variables.tf
@@ -23,13 +23,15 @@ variable "iam" {
variable "name" {
description = "Folder name."
type = string
+ default = null
}
variable "parent" {
description = "Parent in folders/folder_id or organizations/org_id format."
type = string
+ default = null
validation {
- condition = can(regex("(organizations|folders)/[0-9]+", var.parent))
+ condition = var.parent == null || can(regex("(organizations|folders)/[0-9]+", var.parent))
error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
}
}
@@ -72,3 +74,32 @@ variable "firewall_policy_attachments" {
type = map(string)
default = {}
}
+
+variable "logging_sinks" {
+ description = "Logging sinks to create for this folder."
+ type = map(object({
+ destination = string
+ type = string
+ filter = string
+ grant = bool
+ }))
+ default = {}
+}
+
+variable "logging_exclusions" {
+ description = "Logging exclusions for this folder in the form {NAME -> FILTER}."
+ type = map(string)
+ default = {}
+}
+
+variable "folder_create" {
+ description = "Create folder. When set to false, uses id to reference an existing folder."
+ type = bool
+ default = true
+}
+
+variable "id" {
+ description = "Folder ID in case you use folder_create=false"
+ type = string
+ default = null
+}
diff --git a/modules/organization/README.md b/modules/organization/README.md
index a30b161d..f8b500fb 100644
--- a/modules/organization/README.md
+++ b/modules/organization/README.md
@@ -59,6 +59,59 @@ module "org" {
# tftest:modules=1:resources=3
```
+## Logging Sinks
+```hcl
+module "gcs" {
+ source = "./modules/gcs"
+ project_id = var.project_id
+ name = "gcs_sink"
+ force_destroy = true
+}
+
+module "dataset" {
+ source = "./modules/bigquery-dataset"
+ project_id = var.project_id
+ id = "bq_sink"
+}
+
+module "pubsub" {
+ source = "./modules/pubsub"
+ project_id = var.project_id
+ name = "pubsub_sink"
+}
+
+module "org" {
+ source = "./modules/organization"
+ organization_id = var.organization_id
+
+ logging_sinks = {
+ warnings = {
+ type = "gcs"
+ destination = module.gcs.name
+ filter = "severity=WARNING"
+ grant = false
+ }
+ info = {
+ type = "bigquery"
+ destination = module.dataset.id
+ filter = "severity=INFO"
+ grant = false
+ }
+ notice = {
+ type = "pubsub"
+ destination = module.pubsub.id
+ filter = "severity=NOTICE"
+ grant = true
+ }
+ }
+ logging_exclusions = {
+ no-gce-instances = "resource.type=gce_instance"
+ }
+}
+# tftest:modules=4:resources=8
+```
+
+
## Variables
@@ -72,6 +125,8 @@ module "org" {
| *iam_additive* | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
| *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string))
| | {}
|
| *iam_audit_config* | Service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. | map(map(list(string)))
| | {}
|
+| *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string)
| | {}
|
+| *logging_sinks* | Logging sinks to create for this organization. | map(object({...}))
| | {}
|
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool)
| | {}
|
| *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...}))
| | {}
|
@@ -82,4 +137,5 @@ module "org" {
| firewall_policies | Map of firewall policy resources created in the organization. | |
| firewall_policy_id | Map of firewall policy ids created in the organization. | |
| organization_id | Organization id dependent on module resources. | |
+| sink_writer_identities | None | |
diff --git a/modules/organization/main.tf b/modules/organization/main.tf
index f3b75166..a764710c 100644
--- a/modules/organization/main.tf
+++ b/modules/organization/main.tf
@@ -40,6 +40,22 @@ locals {
for rule in local.extended_rules :
"${rule.policy}-${rule.name}" => rule
}
+ logging_sinks = coalesce(var.logging_sinks, {})
+ sink_type_destination = {
+ gcs = "storage.googleapis.com"
+ bigquery = "bigquery.googleapis.com"
+ pubsub = "pubsub.googleapis.com"
+ # TODO: add logging buckets support
+ # logging = "logging.googleapis.com"
+ }
+ sink_bindings = {
+ for type in ["gcs", "bigquery", "pubsub", "logging"] :
+ type => {
+ for name, sink in local.logging_sinks :
+ name => sink
+ if sink.grant && sink.type == type
+ }
+ }
}
resource "google_organization_iam_custom_role" "roles" {
@@ -200,3 +216,50 @@ resource "google_compute_organization_security_policy_association" "attachment"
attachment_id = var.organization_id
policy_id = each.value
}
+
+resource "google_logging_organization_sink" "sink" {
+ for_each = local.logging_sinks
+ name = each.key
+ #description = "${each.key} (Terraform-managed)"
+ org_id = local.organization_id_numeric
+ destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
+ filter = each.value.filter
+}
+
+resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
+ for_each = local.sink_bindings["gcs"]
+ bucket = each.value.destination
+ role = "roles/storage.objectCreator"
+ members = [google_logging_organization_sink.sink[each.key].writer_identity]
+}
+
+resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" {
+ for_each = local.sink_bindings["bigquery"]
+ project = split("/", each.value.destination)[1]
+ dataset_id = split("/", each.value.destination)[3]
+ role = "roles/bigquery.dataEditor"
+ members = [google_logging_organization_sink.sink[each.key].writer_identity]
+}
+
+resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" {
+ for_each = local.sink_bindings["pubsub"]
+ project = split("/", each.value.destination)[1]
+ topic = split("/", each.value.destination)[3]
+ role = "roles/pubsub.publisher"
+ members = [google_logging_organization_sink.sink[each.key].writer_identity]
+}
+
+# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" {
+# for_each = local.sink_grants["gcs"]
+# bucket = each.value.destination
+# role = "roles/storage.objectCreator"
+# members = [google_logging_organization_sink.sink[each.key].writer_identity]
+# }
+
+resource "google_logging_organization_exclusion" "logging-exclusion" {
+ for_each = coalesce(var.logging_exclusions, {})
+ name = each.key
+ org_id = local.organization_id_numeric
+ description = "${each.key} (Terraform-managed)"
+ filter = each.value
+}
diff --git a/modules/organization/outputs.tf b/modules/organization/outputs.tf
index dd0d0294..869f8185 100644
--- a/modules/organization/outputs.tf
+++ b/modules/organization/outputs.tf
@@ -42,3 +42,10 @@ output "firewall_policy_id" {
name => google_compute_organization_security_policy.policy[name].id
}
}
+
+output "sink_writer_identities" {
+ description = ""
+ value = {
+ for name, sink in google_logging_organization_sink.sink : name => sink.writer_identity
+ }
+}
diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf
index ea7b5f52..5c426cf8 100644
--- a/modules/organization/variables.tf
+++ b/modules/organization/variables.tf
@@ -98,3 +98,20 @@ variable "firewall_policy_attachments" {
type = map(string)
default = {}
}
+
+variable "logging_sinks" {
+ description = "Logging sinks to create for this organization."
+ type = map(object({
+ destination = string
+ type = string
+ filter = string
+ grant = bool
+ }))
+ default = {}
+}
+
+variable "logging_exclusions" {
+ description = "Logging exclusions for this organization in the form {NAME -> FILTER}."
+ type = map(string)
+ default = {}
+}
diff --git a/modules/project/README.md b/modules/project/README.md
index 39871625..f8d238d1 100644
--- a/modules/project/README.md
+++ b/modules/project/README.md
@@ -84,6 +84,60 @@ module "project" {
# tftest:modules=1:resources=6
```
+## Logging Sinks
+```hcl
+module "gcs" {
+ source = "./modules/gcs"
+ project_id = var.project_id
+ name = "gcs_sink"
+ force_destroy = true
+}
+
+module "dataset" {
+ source = "./modules/bigquery-dataset"
+ project_id = var.project_id
+ id = "bq_sink"
+}
+
+module "pubsub" {
+ source = "./modules/pubsub"
+ project_id = var.project_id
+ name = "pubsub_sink"
+}
+
+module "project-host" {
+ source = "./modules/project"
+ name = "my-project"
+ billing_account = "123456-123456-123456"
+ parent = "folders/1234567890"
+ logging_sinks = {
+ warnings = {
+ type = "gcs"
+ destination = module.gcs.name
+ filter = "severity=WARNING"
+ grant = false
+ }
+ info = {
+ type = "bigquery"
+ destination = module.dataset.id
+ filter = "severity=INFO"
+ grant = false
+ }
+ notice = {
+ type = "pubsub"
+ destination = module.pubsub.id
+ filter = "severity=NOTICE"
+ grant = true
+ }
+ }
+ logging_exclusions = {
+ no-gce-instances = "resource.type=gce_instance"
+ }
+}
+# tftest:modules=4:resources=9
+```
+
+
## Variables
@@ -98,6 +152,8 @@ module "project" {
| *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string))
| | {}
|
| *labels* | Resource labels. | map(string)
| | {}
|
| *lien_reason* | If non-empty, creates a project lien with this description. | string
| |
|
+| *logging_exclusions* | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string)
| | {}
|
+| *logging_sinks* | Logging sinks to create for this project. | map(object({...}))
| | {}
|
| *oslogin* | Enable OS Login. | bool
| | false
|
| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string)
| | []
|
| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string)
| | []
|
@@ -120,5 +176,6 @@ module "project" {
| number | Project number. | |
| project_id | Project id. | |
| service_accounts | Product robot service accounts in project. | |
+| sink_writer_identities | None | |
diff --git a/modules/project/main.tf b/modules/project/main.tf
index 9e5502ae..0606fa24 100644
--- a/modules/project/main.tf
+++ b/modules/project/main.tf
@@ -37,6 +37,22 @@ locals {
? try(google_project.project.0, null)
: try(data.google_project.project.0, null)
)
+ logging_sinks = coalesce(var.logging_sinks, {})
+ sink_type_destination = {
+ gcs = "storage.googleapis.com"
+ bigquery = "bigquery.googleapis.com"
+ pubsub = "pubsub.googleapis.com"
+ # TODO: add logging buckets support
+ # logging = "logging.googleapis.com"
+ }
+ sink_bindings = {
+ for type in ["gcs", "bigquery", "pubsub", "logging"] :
+ type => {
+ for name, sink in local.logging_sinks :
+ name => sink
+ if sink.grant && sink.type == type
+ }
+ }
}
data "google_project" "project" {
@@ -242,3 +258,50 @@ resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
host_project = var.shared_vpc_service_config.host_project
service_project = local.project.project_id
}
+
+resource "google_logging_project_sink" "sink" {
+ for_each = local.logging_sinks
+ name = each.key
+ #description = "${each.key} (Terraform-managed)"
+ project = local.project.project_id
+ destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
+ filter = each.value.filter
+}
+
+resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
+ for_each = local.sink_bindings["gcs"]
+ bucket = each.value.destination
+ role = "roles/storage.objectCreator"
+ members = [google_logging_project_sink.sink[each.key].writer_identity]
+}
+
+resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" {
+ for_each = local.sink_bindings["bigquery"]
+ project = split("/", each.value.destination)[1]
+ dataset_id = split("/", each.value.destination)[3]
+ role = "roles/bigquery.dataEditor"
+ members = [google_logging_project_sink.sink[each.key].writer_identity]
+}
+
+resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" {
+ for_each = local.sink_bindings["pubsub"]
+ project = split("/", each.value.destination)[1]
+ topic = split("/", each.value.destination)[3]
+ role = "roles/pubsub.publisher"
+ members = [google_logging_project_sink.sink[each.key].writer_identity]
+}
+
+# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" {
+# for_each = local.sink_grants["gcs"]
+# bucket = each.value.destination
+# role = "roles/storage.objectCreator"
+# members = [google_logging_project_sink.sink[each.key].writer_identity]
+# }
+
+resource "google_logging_project_exclusion" "logging-exclusion" {
+ for_each = coalesce(var.logging_exclusions, {})
+ name = each.key
+ project = local.project.project_id
+ description = "${each.key} (Terraform-managed)"
+ filter = each.value
+}
diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf
index f20c78b0..b11d362c 100644
--- a/modules/project/outputs.tf
+++ b/modules/project/outputs.tf
@@ -64,3 +64,10 @@ output "custom_roles" {
name => role.id
}
}
+
+output "sink_writer_identities" {
+ description = ""
+ value = {
+ for name, sink in google_logging_project_sink.sink : name => sink.writer_identity
+ }
+}
diff --git a/modules/project/variables.tf b/modules/project/variables.tf
index feb3b21a..d8e06a9e 100644
--- a/modules/project/variables.tf
+++ b/modules/project/variables.tf
@@ -165,3 +165,20 @@ variable "shared_vpc_service_config" {
host_project = ""
}
}
+
+variable "logging_sinks" {
+ description = "Logging sinks to create for this project."
+ type = map(object({
+ destination = string
+ type = string
+ filter = string
+ grant = bool
+ }))
+ default = {}
+}
+
+variable "logging_exclusions" {
+ description = "Logging exclusions for this project in the form {NAME -> FILTER}."
+ type = map(string)
+ default = {}
+}