zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Logging sinks and exclusions (#178) 

* Add sink support to folder module

* Make folder creation optional.

* Add logging sinks to the organization module

* Add logging sink support to project module

* Update readme
This commit is contained in:
Julio Castillo 2020-12-05 08:31:35 +01:00 committed by GitHub
parent 830216ac9b
commit 2c0f949f07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 470 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -3,6 +3,7 @@
All notable changes to this project will be documented in this file.
## [Unreleased]
- add support for creating logging sinks and logging exclusions in the `project`, `folder` and `organization` modules
## [4.2.0] - 2020-11-25

62
modules/folder/README.md
View File

@ -41,6 +41,59 @@ module "folder" {
# tftest:modules=1:resources=4
```
### Logging Sinks
```hcl
module "gcs" {
source = "./modules/gcs"
project_id = "my-project"
name = "gcs_sink"
force_destroy = true
}
module "dataset" {
source = "./modules/bigquery-dataset"
project_id = "my-project"
id = "bq_sink"
}
module "pubsub" {
source = "./modules/pubsub"
project_id = "my-project"
name = "pubsub_sink"
}
module "folder-sink" {
source = "./modules/folder"
parent = "folders/657104291943"
name = "my-folder"
logging_sinks = {
warnings = {
type = "gcs"
destination = module.gcs.name
filter = "severity=WARNING"
grant = false
}
info = {
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
grant = false
}
notice = {
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
grant = true
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=9
```
### Hierarchical firewall policies
```hcl
@ -88,11 +141,15 @@ module "folder2" {
| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
| name | Folder name. | <code title="">string</code> | ✓ | |
| parent | Parent in folders/folder_id or organizations/org_id format. | <code title="string&#10;validation &#123;&#10;condition &#61; can&#40;regex&#40;&#34;&#40;organizations&#124;folders&#41;&#47;&#91;0-9&#93;&#43;&#34;, var.parent&#41;&#41;&#10;error_message &#61; &#34;Parent must be of the form folders&#47;folder_id or organizations&#47;organization_id.&#34;&#10;&#125;">string</code> | ✓ | |
| *firewall_policies* | Hierarchical firewall policies to *create* in this folder. | <code title="map&#40;map&#40;object&#40;&#123;&#10;description &#61; string&#10;direction &#61; string&#10;action &#61; string&#10;priority &#61; number&#10;ranges &#61; list&#40;string&#41;&#10;ports &#61; map&#40;list&#40;string&#41;&#41;&#10;target_service_accounts &#61; list&#40;string&#41;&#10;target_resources &#61; list&#40;string&#41;&#10;logging &#61; bool&#10;&#125;&#41;&#41;&#41;">map(map(object({...})))</code> | | <code title="">{}</code> |
| *firewall_policy_attachments* | List of hierarchical firewall policy IDs to *attach* to this folder. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *folder_create* | Create folder. When set to false, uses id to reference an existing folder. | <code title="">bool</code> | | <code title="">true</code> |
| *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | <code title="map&#40;set&#40;string&#41;&#41;">map(set(string))</code> | | <code title="">{}</code> |
| *id* | Folder ID in case you use folder_create=false | <code title="">string</code> | | <code title="">null</code> |
| *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;grant &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *name* | Folder name. | <code title="">string</code> | | <code title="">null</code> |
| *parent* | Parent in folders/folder_id or organizations/org_id format. | <code title="">string</code> | | <code title="null&#10;validation &#123;&#10;condition &#61; var.parent &#61;&#61; null &#124;&#124; can&#40;regex&#40;&#34;&#40;organizations&#124;folders&#41;&#47;&#91;0-9&#93;&#43;&#34;, var.parent&#41;&#41;&#10;error_message &#61; &#34;Parent must be of the form folders&#47;folder_id or organizations&#47;organization_id.&#34;&#10;&#125;">...</code> |
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | <code title="map&#40;bool&#41;">map(bool)</code> | | <code title="">{}</code> |
| *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | <code title="map&#40;object&#40;&#123;&#10;inherit_from_parent &#61; bool&#10;suggested_value &#61; string&#10;status &#61; bool&#10;values &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
@ -105,4 +162,5 @@ module "folder2" {
| folder | Folder resource. | |
| id | Folder id. | |
| name | Folder name. | |
| sink_writer_identities | None | |
<!-- END TFDOC -->

86
modules/folder/main.tf
View File

@ -25,23 +25,50 @@ locals {
for rule in local.extended_rules :
"${rule.policy}-${rule.name}" => rule
}
logging_sinks = coalesce(var.logging_sinks, {})
sink_type_destination = {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
type => {
for name, sink in local.logging_sinks :
name => sink
if sink.grant && sink.type == type
}
}
folder = (
var.folder_create
? try(google_folder.folder.0, null)
: try(data.google_folder.folder.0, null)
)
}
data "google_folder" "folder" {
count = var.folder_create ? 0 : 1
folder = var.id
}
resource "google_folder" "folder" {
count = var.folder_create ? 1 : 0
display_name = var.name
parent = var.parent
}
resource "google_folder_iam_binding" "authoritative" {
for_each = var.iam
folder = google_folder.folder.name
folder = local.folder.name
role = each.key
members = each.value
}
resource "google_folder_organization_policy" "boolean" {
for_each = var.policy_boolean
folder = google_folder.folder.name
folder = local.folder.name
constraint = each.key
dynamic boolean_policy {
@ -62,7 +89,7 @@ resource "google_folder_organization_policy" "boolean" {
resource "google_folder_organization_policy" "list" {
for_each = var.policy_list
folder = google_folder.folder.name
folder = local.folder.name
constraint = each.key
dynamic list_policy {
@ -117,7 +144,7 @@ resource "google_compute_organization_security_policy" "policy" {
for_each = var.firewall_policies
display_name = each.key
parent = google_folder.folder.id
parent = local.folder.id
}
resource "google_compute_organization_security_policy_rule" "rule" {
@ -152,7 +179,54 @@ resource "google_compute_organization_security_policy_rule" "rule" {
resource "google_compute_organization_security_policy_association" "attachment" {
provider = google-beta
for_each = var.firewall_policy_attachments
name = "${google_folder.folder.id}-${each.key}"
attachment_id = google_folder.folder.id
name = "${local.folder.id}-${each.key}"
attachment_id = local.folder.id
policy_id = each.value
}
resource "google_logging_folder_sink" "sink" {
for_each = local.logging_sinks
name = each.key
#description = "${each.key} (Terraform-managed)"
folder = local.folder.name
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
}
resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
for_each = local.sink_bindings["gcs"]
bucket = each.value.destination
role = "roles/storage.objectCreator"
members = [google_logging_folder_sink.sink[each.key].writer_identity]
}
resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" {
for_each = local.sink_bindings["bigquery"]
project = split("/", each.value.destination)[1]
dataset_id = split("/", each.value.destination)[3]
role = "roles/bigquery.dataEditor"
members = [google_logging_folder_sink.sink[each.key].writer_identity]
}
resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" {
for_each = local.sink_bindings["pubsub"]
project = split("/", each.value.destination)[1]
topic = split("/", each.value.destination)[3]
role = "roles/pubsub.publisher"
members = [google_logging_folder_sink.sink[each.key].writer_identity]
}
# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" {
# for_each = local.sink_grants["gcs"]
# bucket = each.value.destination
# role = "roles/storage.objectCreator"
# members = [google_logging_folder_sink.sink[each.key].writer_identity]
# }
resource "google_logging_folder_exclusion" "logging-exclusion" {
for_each = coalesce(var.logging_exclusions, {})
name = each.key
folder = local.folder.name
description = "${each.key} (Terraform-managed)"
filter = each.value
}

13
modules/folder/outputs.tf
View File

@ -16,12 +16,12 @@
output "folder" {
description = "Folder resource."
value = google_folder.folder
value = local.folder
}
output "id" {
description = "Folder id."
value = google_folder.folder.name
value = local.folder.name
depends_on = [
google_folder_iam_binding.authoritative,
google_folder_organization_policy.boolean,
@ -31,7 +31,7 @@ output "id" {
output "name" {
description = "Folder name."
value = google_folder.folder.display_name
value = local.folder.display_name
}
output "firewall_policies" {
@ -49,3 +49,10 @@ output "firewall_policy_id" {
name => google_compute_organization_security_policy.policy[name].id
}
}
output "sink_writer_identities" {
description = ""
value = {
for name, sink in google_logging_folder_sink.sink : name => sink.writer_identity
}
}

33
modules/folder/variables.tf
View File

@ -23,13 +23,15 @@ variable "iam" {
variable "name" {
description = "Folder name."
type = string
default = null
}
variable "parent" {
description = "Parent in folders/folder_id or organizations/org_id format."
type = string
default = null
validation {
condition = can(regex("(organizations|folders)/[0-9]+", var.parent))
condition = var.parent == null || can(regex("(organizations|folders)/[0-9]+", var.parent))
error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
}
}
@ -72,3 +74,32 @@ variable "firewall_policy_attachments" {
type = map(string)
default = {}
}
variable "logging_sinks" {
description = "Logging sinks to create for this folder."
type = map(object({
destination = string
type = string
filter = string
grant = bool
}))
default = {}
}
variable "logging_exclusions" {
description = "Logging exclusions for this folder in the form {NAME -> FILTER}."
type = map(string)
default = {}
}
variable "folder_create" {
description = "Create folder. When set to false, uses id to reference an existing folder."
type = bool
default = true
}
variable "id" {
description = "Folder ID in case you use folder_create=false"
type = string
default = null
}

56
modules/organization/README.md
View File

@ -59,6 +59,59 @@ module "org" {
# tftest:modules=1:resources=3
```
## Logging Sinks
```hcl
module "gcs" {
source = "./modules/gcs"
project_id = var.project_id
name = "gcs_sink"
force_destroy = true
}
module "dataset" {
source = "./modules/bigquery-dataset"
project_id = var.project_id
id = "bq_sink"
}
module "pubsub" {
source = "./modules/pubsub"
project_id = var.project_id
name = "pubsub_sink"
}
module "org" {
source = "./modules/organization"
organization_id = var.organization_id
logging_sinks = {
warnings = {
type = "gcs"
destination = module.gcs.name
filter = "severity=WARNING"
grant = false
}
info = {
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
grant = false
}
notice = {
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
grant = true
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=8
```
<!-- BEGIN TFDOC -->
## Variables
@ -72,6 +125,8 @@ module "org" {
| *iam_additive* | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | <code title="map&#40;list&#40;string&#41;&#41;">map(list(string))</code> | | <code title="">{}</code> |
| *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | <code title="map&#40;list&#40;string&#41;&#41;">map(list(string))</code> | | <code title="">{}</code> |
| *iam_audit_config* | Service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. | <code title="map&#40;map&#40;list&#40;string&#41;&#41;&#41;">map(map(list(string)))</code> | | <code title="">{}</code> |
| *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this organization. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;grant &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | <code title="map&#40;bool&#41;">map(bool)</code> | | <code title="">{}</code> |
| *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | <code title="map&#40;object&#40;&#123;&#10;inherit_from_parent &#61; bool&#10;suggested_value &#61; string&#10;status &#61; bool&#10;values &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
@ -82,4 +137,5 @@ module "org" {
| firewall_policies | Map of firewall policy resources created in the organization. | |
| firewall_policy_id | Map of firewall policy ids created in the organization. | |
| organization_id | Organization id dependent on module resources. | |
| sink_writer_identities | None | |
<!-- END TFDOC -->

63
modules/organization/main.tf
View File

@ -40,6 +40,22 @@ locals {
for rule in local.extended_rules :
"${rule.policy}-${rule.name}" => rule
}
logging_sinks = coalesce(var.logging_sinks, {})
sink_type_destination = {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
type => {
for name, sink in local.logging_sinks :
name => sink
if sink.grant && sink.type == type
}
}
}
resource "google_organization_iam_custom_role" "roles" {
@ -200,3 +216,50 @@ resource "google_compute_organization_security_policy_association" "attachment"
attachment_id = var.organization_id
policy_id = each.value
}
resource "google_logging_organization_sink" "sink" {
for_each = local.logging_sinks
name = each.key
#description = "${each.key} (Terraform-managed)"
org_id = local.organization_id_numeric
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
}
resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
for_each = local.sink_bindings["gcs"]
bucket = each.value.destination
role = "roles/storage.objectCreator"
members = [google_logging_organization_sink.sink[each.key].writer_identity]
}
resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" {
for_each = local.sink_bindings["bigquery"]
project = split("/", each.value.destination)[1]
dataset_id = split("/", each.value.destination)[3]
role = "roles/bigquery.dataEditor"
members = [google_logging_organization_sink.sink[each.key].writer_identity]
}
resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" {
for_each = local.sink_bindings["pubsub"]
project = split("/", each.value.destination)[1]
topic = split("/", each.value.destination)[3]
role = "roles/pubsub.publisher"
members = [google_logging_organization_sink.sink[each.key].writer_identity]
}
# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" {
# for_each = local.sink_grants["gcs"]
# bucket = each.value.destination
# role = "roles/storage.objectCreator"
# members = [google_logging_organization_sink.sink[each.key].writer_identity]
# }
resource "google_logging_organization_exclusion" "logging-exclusion" {
for_each = coalesce(var.logging_exclusions, {})
name = each.key
org_id = local.organization_id_numeric
description = "${each.key} (Terraform-managed)"
filter = each.value
}

7
modules/organization/outputs.tf
View File

@ -42,3 +42,10 @@ output "firewall_policy_id" {
name => google_compute_organization_security_policy.policy[name].id
}
}
output "sink_writer_identities" {
description = ""
value = {
for name, sink in google_logging_organization_sink.sink : name => sink.writer_identity
}
}

17
modules/organization/variables.tf
View File

@ -98,3 +98,20 @@ variable "firewall_policy_attachments" {
type = map(string)
default = {}
}
variable "logging_sinks" {
description = "Logging sinks to create for this organization."
type = map(object({
destination = string
type = string
filter = string
grant = bool
}))
default = {}
}
variable "logging_exclusions" {
description = "Logging exclusions for this organization in the form {NAME -> FILTER}."
type = map(string)
default = {}
}

57
modules/project/README.md
View File

@ -84,6 +84,60 @@ module "project" {
# tftest:modules=1:resources=6
```
## Logging Sinks
```hcl
module "gcs" {
source = "./modules/gcs"
project_id = var.project_id
name = "gcs_sink"
force_destroy = true
}
module "dataset" {
source = "./modules/bigquery-dataset"
project_id = var.project_id
id = "bq_sink"
}
module "pubsub" {
source = "./modules/pubsub"
project_id = var.project_id
name = "pubsub_sink"
}
module "project-host" {
source = "./modules/project"
name = "my-project"
billing_account = "123456-123456-123456"
parent = "folders/1234567890"
logging_sinks = {
warnings = {
type = "gcs"
destination = module.gcs.name
filter = "severity=WARNING"
grant = false
}
info = {
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
grant = false
}
notice = {
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
grant = true
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=9
```
<!-- BEGIN TFDOC -->
## Variables
@ -98,6 +152,8 @@ module "project" {
| *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | <code title="map&#40;list&#40;string&#41;&#41;">map(list(string))</code> | | <code title="">{}</code> |
| *labels* | Resource labels. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *lien_reason* | If non-empty, creates a project lien with this description. | <code title="">string</code> | | <code title=""></code> |
| *logging_exclusions* | Logging exclusions for this project in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this project. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;grant &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *oslogin* | Enable OS Login. | <code title="">bool</code> | | <code title="">false</code> |
| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">[]</code> |
| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">[]</code> |
@ -120,5 +176,6 @@ module "project" {
| number | Project number. | |
| project_id | Project id. | |
| service_accounts | Product robot service accounts in project. | |
| sink_writer_identities | None | |
<!-- END TFDOC -->

63
modules/project/main.tf
View File

@ -37,6 +37,22 @@ locals {
? try(google_project.project.0, null)
: try(data.google_project.project.0, null)
)
logging_sinks = coalesce(var.logging_sinks, {})
sink_type_destination = {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
type => {
for name, sink in local.logging_sinks :
name => sink
if sink.grant && sink.type == type
}
}
}
data "google_project" "project" {
@ -242,3 +258,50 @@ resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
host_project = var.shared_vpc_service_config.host_project
service_project = local.project.project_id
}
resource "google_logging_project_sink" "sink" {
for_each = local.logging_sinks
name = each.key
#description = "${each.key} (Terraform-managed)"
project = local.project.project_id
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
}
resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
for_each = local.sink_bindings["gcs"]
bucket = each.value.destination
role = "roles/storage.objectCreator"
members = [google_logging_project_sink.sink[each.key].writer_identity]
}
resource "google_bigquery_dataset_iam_binding" "bq-sinks-binding" {
for_each = local.sink_bindings["bigquery"]
project = split("/", each.value.destination)[1]
dataset_id = split("/", each.value.destination)[3]
role = "roles/bigquery.dataEditor"
members = [google_logging_project_sink.sink[each.key].writer_identity]
}
resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" {
for_each = local.sink_bindings["pubsub"]
project = split("/", each.value.destination)[1]
topic = split("/", each.value.destination)[3]
role = "roles/pubsub.publisher"
members = [google_logging_project_sink.sink[each.key].writer_identity]
}
# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" {
# for_each = local.sink_grants["gcs"]
# bucket = each.value.destination
# role = "roles/storage.objectCreator"
# members = [google_logging_project_sink.sink[each.key].writer_identity]
# }
resource "google_logging_project_exclusion" "logging-exclusion" {
for_each = coalesce(var.logging_exclusions, {})
name = each.key
project = local.project.project_id
description = "${each.key} (Terraform-managed)"
filter = each.value
}

7
modules/project/outputs.tf
View File

@ -64,3 +64,10 @@ output "custom_roles" {
name => role.id
}
}
output "sink_writer_identities" {
description = ""
value = {
for name, sink in google_logging_project_sink.sink : name => sink.writer_identity
}
}

17
modules/project/variables.tf
View File

@ -165,3 +165,20 @@ variable "shared_vpc_service_config" {
host_project = ""
}
}
variable "logging_sinks" {
description = "Logging sinks to create for this project."
type = map(object({
destination = string
type = string
filter = string
grant = bool
}))
default = {}
}
variable "logging_exclusions" {
description = "Logging exclusions for this project in the form {NAME -> FILTER}."
type = map(string)
default = {}
}