zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support for cloud logging buckets

This commit is contained in:
Julio Castillo 2021-03-03 14:19:08 +01:00
parent 0f469a22a1
commit ad68fc4dfa
23 changed files with 847 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
modules/folder/README.md
View File

@ -63,6 +63,13 @@ module "pubsub" {
name = "pubsub_sink"
}
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = "my-project"
id = "bucket"
}
module "folder-sink" {
source = "./modules/folder"
parent = "folders/657104291943"
@ -74,6 +81,7 @@ module "folder-sink" {
filter = "severity=WARNING"
iam = false
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
@ -81,6 +89,7 @@ module "folder-sink" {
filter = "severity=INFO"
iam = false
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
@ -88,13 +97,24 @@ module "folder-sink" {
filter = "severity=NOTICE"
iam = true
include_children = true
exclusions = {}
}
debug = {
type = "logging"
destination = module.bucket.id
filter = "severity=DEBUG"
iam = true
include_children = true
exclusions = {
no-compute = "logName:compute"
}
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=9
# tftest:modules=5:resources=11
```
### Hierarchical firewall policies
@ -151,7 +171,7 @@ module "folder2" {
| *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | <code title="map&#40;set&#40;string&#41;&#41;">map(set(string))</code> | | <code title="">{}</code> |
| *id* | Folder ID in case you use folder_create=false | <code title="">string</code> | | <code title="">null</code> |
| *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;exclusions &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *name* | Folder name. | <code title="">string</code> | | <code title="">null</code> |
| *parent* | Parent in folders/folder_id or organizations/org_id format. | <code title="">string</code> | | <code title="null&#10;validation &#123;&#10;condition &#61; var.parent &#61;&#61; null &#124;&#124; can&#40;regex&#40;&#34;&#40;organizations&#124;folders&#41;&#47;&#91;0-9&#93;&#43;&#34;, var.parent&#41;&#41;&#10;error_message &#61; &#34;Parent must be of the form folders&#47;folder_id or organizations&#47;organization_id.&#34;&#10;&#125;">...</code> |
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | <code title="map&#40;bool&#41;">map(bool)</code> | | <code title="">{}</code> |

12
modules/folder/main.tf
View File

@ -30,8 +30,7 @@ locals {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
@ -192,6 +191,15 @@ resource "google_logging_folder_sink" "sink" {
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
include_children = each.value.include_children
dynamic "exclusions" {
for_each = each.value.exclusions
iterator = exclusion
content {
name = exclusion.key
filter = exclusion.value
}
}
}
resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {

2
modules/folder/variables.tf
View File

@ -83,6 +83,8 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
# TODO exclusions also support description and disabled
exclusions = map(string)
}))
default = {}
}

61
modules/logging-bucket/README.md Normal file
View File

@ -0,0 +1,61 @@
# Google Cloud Logging Buckets Module
This module manages [logging buckets](https://cloud.google.com/logging/docs/storage#logs-buckets) for a project, folder, organization or billing account.
Note that some logging buckets are automatically created for a given folder, project, organization, and billing account cannot be deleted. Creating a resource of this type will acquire and update the resource that already exists at the desired location. These buckets cannot be removed so deleting this resource will remove the bucket config from your terraform state but will leave the logging bucket unchanged. The buckets that are currently automatically created are "_Default" and "_Required".
See also the `logging_sinks` argument within the [project](../project/), [folder](../folder/) and [organization](../organization) modules.
## Examples
### Create custom logging bucket in a project
```hcl
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = var.project_id
id = "mybucket"
}
# tftest:modules=1:resources=1
```
### Change retention period of a folder's _Default bucket
```hcl
module "folder" {
source = "./modules/folder"
parent = "folders/657104291943"
name = "my folder"
}
module "bucket-default" {
source = "./modules/logging-bucket"
parent_type = "folder"
parent = module.folder.id
id = "_Default"
retention = 10
}
# tftest:modules=2:resources=2
```
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
| id | Name of the logging bucket. | <code title="">string</code> | ✓ | |
| parent | ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'. | <code title="">string</code> | ✓ | |
| parent_type | Parent object type for the bucket (project, folder, organization, billing_account). | <code title="">string</code> | ✓ | |
| *description* | Human-readable description for the logging bucket. | <code title="">string</code> | | <code title="">null</code> |
| *location* | Location of the bucket. | <code title="">string</code> | | <code title="">global</code> |
| *retention* | Retention time in days for the logging bucket. | <code title="">number</code> | | <code title="">30</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| id | None | |
<!-- END TFDOC -->

51
modules/logging-bucket/main.tf Normal file
View File

@ -0,0 +1,51 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
resource "google_logging_project_bucket_config" "bucket" {
count = var.parent_type == "project" ? 1 : 0
project = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}
resource "google_logging_folder_bucket_config" "bucket" {
count = var.parent_type == "folder" ? 1 : 0
folder = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}
resource "google_logging_organization_bucket_config" "bucket" {
count = var.parent_type == "organization" ? 1 : 0
organization = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}
resource "google_logging_billing_account_bucket_config" "bucket" {
count = var.parent_type == "billing_account" ? 1 : 0
billing_account = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}

24
modules/logging-bucket/outputs.tf Normal file
View File

@ -0,0 +1,24 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "id" {
value = try(
google_logging_project_bucket_config.bucket.0.id,
google_logging_folder_bucket_config.bucket.0.id,
google_logging_organization_bucket_config.bucket.0.id,
google_logging_billing_account_bucket_config.bucket.0.id,
)
}

48
modules/logging-bucket/variables.tf Normal file
View File

@ -0,0 +1,48 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "parent_type" {
description = "Parent object type for the bucket (project, folder, organization, billing_account)."
type = string
}
variable "parent" {
description = "ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'."
type = string
}
variable "location" {
description = "Location of the bucket."
type = string
default = "global"
}
variable "id" {
description = "Name of the logging bucket."
type = string
}
variable "description" {
description = "Human-readable description for the logging bucket."
type = string
default = null
}
variable "retention" {
description = "Retention time in days for the logging bucket."
type = number
default = 30
}

24
modules/organization/README.md
View File

@ -80,6 +80,13 @@ module "pubsub" {
name = "pubsub_sink"
}
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = "my-project"
id = "bucket"
}
module "org" {
source = "./modules/organization"
organization_id = var.organization_id
@ -91,6 +98,7 @@ module "org" {
filter = "severity=WARNING"
iam = false
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
@ -98,6 +106,7 @@ module "org" {
filter = "severity=INFO"
iam = false
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
@ -105,13 +114,24 @@ module "org" {
filter = "severity=NOTICE"
iam = true
include_children = true
exclusions = {}
}
debug = {
type = "logging"
destination = module.bucket.id
filter = "severity=DEBUG"
iam = true
include_children = false
exclusions = {
no-compute = "logName:compute"
}
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=8
# tftest:modules=5:resources=10
```
@ -132,7 +152,7 @@ module "org" {
| *iam_audit_config_authoritative* | IAM Authoritative service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. Audit config should also be authoritative when using authoritative bindings. Use with caution. | <code title="map&#40;map&#40;list&#40;string&#41;&#41;&#41;">map(map(list(string)))</code> | | <code title="">null</code> |
| *iam_bindings_authoritative* | IAM authoritative bindings, in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared. Bindings should also be authoritative when using authoritative audit config. Use with caution. | <code title="map&#40;list&#40;string&#41;&#41;">map(list(string))</code> | | <code title="">null</code> |
| *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this organization. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this organization. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;exclusions &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | <code title="map&#40;bool&#41;">map(bool)</code> | | <code title="">{}</code> |
| *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | <code title="map&#40;object&#40;&#123;&#10;inherit_from_parent &#61; bool&#10;suggested_value &#61; string&#10;status &#61; bool&#10;values &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |

12
modules/organization/main.tf
View File

@ -45,8 +45,7 @@ locals {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
@ -256,6 +255,15 @@ resource "google_logging_organization_sink" "sink" {
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
include_children = each.value.include_children
dynamic "exclusions" {
for_each = each.value.exclusions
iterator = exclusion
content {
name = exclusion.key
filter = exclusion.value
}
}
}
resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {

2
modules/organization/variables.tf
View File

@ -124,6 +124,8 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
# TODO exclusions also support description and disabled
exclusions = map(string)
}))
default = {}
}

51
modules/project/README.md
View File

@ -94,6 +94,13 @@ module "pubsub" {
name = "pubsub_sink"
}
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = "my-project"
id = "bucket"
}
module "project-host" {
source = "./modules/project"
name = "my-project"
@ -101,29 +108,45 @@ module "project-host" {
parent = "folders/1234567890"
logging_sinks = {
warnings = {
type = "gcs"
destination = module.gcs.name
filter = "severity=WARNING"
iam = false
type = "gcs"
destination = module.gcs.name
filter = "severity=WARNING"
iam = false
unique_writer = false
exclusions = {}
}
info = {
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
iam = false
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
iam = false
unique_writer = false
exclusions = {}
}
notice = {
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
iam = true
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
iam = true
unique_writer = false
exclusions = {}
}
debug = {
type = "logging"
destination = module.bucket.id
filter = "severity=DEBUG"
iam = true
unique_writer = false
exclusions = {
no-compute = "logName:compute"
}
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=9
# tftest:modules=5:resources=11
```
@ -143,7 +166,7 @@ module "project-host" {
| *labels* | Resource labels. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *lien_reason* | If non-empty, creates a project lien with this description. | <code title="">string</code> | | <code title=""></code> |
| *logging_exclusions* | Logging exclusions for this project in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this project. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this project. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;unique_writer &#61; bool&#10;exclusions &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *oslogin* | Enable OS Login. | <code title="">bool</code> | | <code title="">false</code> |
| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">[]</code> |
| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | <code title="list&#40;string&#41;">list(string)</code> | | <code title="">[]</code> |

19
modules/project/main.tf
View File

@ -42,8 +42,7 @@ locals {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
@ -263,9 +262,19 @@ resource "google_logging_project_sink" "sink" {
for_each = local.logging_sinks
name = each.key
#description = "${each.key} (Terraform-managed)"
project = local.project.project_id
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
project = local.project.project_id
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
unique_writer_identity = each.value.unique_writer
dynamic "exclusions" {
for_each = each.value.exclusions
iterator = exclusion
content {
name = exclusion.key
filter = exclusion.value
}
}
}
resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {

11
modules/project/variables.tf
View File

@ -169,10 +169,13 @@ variable "shared_vpc_service_config" {
variable "logging_sinks" {
description = "Logging sinks to create for this project."
type = map(object({
destination = string
type = string
filter = string
iam = bool
destination = string
type = string
filter = string
iam = bool
unique_writer = bool
# TODO exclusions also support description and disabled
exclusions = map(string)
}))
default = {}
}

1
tests/modules/folder/fixture/variables.tf
View File

@ -61,6 +61,7 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
exclusions = map(string)
}))
default = {}
}

183
tests/modules/folder/test_plan_logging.py
View File

@ -18,18 +18,19 @@ import pytest
from collections import Counter
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_sinks(plan_runner):
"Test folder-level sinks."
logging_sinks = """ {
"Test folder-level sinks."
logging_sinks = """ {
warning = {
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
@ -37,6 +38,7 @@ def test_sinks(plan_runner):
filter = "severity=INFO"
iam = true
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
@ -44,72 +46,123 @@ def test_sinks(plan_runner):
filter = "severity=NOTICE"
iam = true
include_children = false
exclusions = {}
}
debug = {
type = "logging"
destination = "projects/myproject/locations/global/buckets/mybucket"
filter = "severity=DEBUG"
iam = true
include_children = false
exclusions = {
no-compute = "logName:compute"
no-container = "logName:container"
}
}
}
"""
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 7
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 8
resource_types = Counter([r['type'] for r in resources])
assert resource_types == {
'google_bigquery_dataset_iam_binding': 1,
'google_folder': 1,
'google_logging_folder_sink': 3,
'google_pubsub_topic_iam_binding': 1,
'google_storage_bucket_iam_binding': 1
}
sinks = [r for r in resources
if r['type'] == 'google_logging_folder_sink']
assert sorted([r['index'] for r in sinks]) == [
'info',
'notice',
'warning',
]
values = [(r['index'], r['values']['filter'], r['values']['destination'],
r['values']['include_children'])
for r in sinks]
assert sorted(values) == [
('info',
'severity=INFO',
'bigquery.googleapis.com/projects/myproject/datasets/mydataset',
True),
('notice',
'severity=NOTICE',
'pubsub.googleapis.com/projects/myproject/topics/mytopic',
False),
('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)]
resource_types = Counter([r["type"] for r in resources])
assert resource_types == {
"google_bigquery_dataset_iam_binding": 1,
"google_folder": 1,
"google_logging_folder_sink": 4,
"google_pubsub_topic_iam_binding": 1,
"google_storage_bucket_iam_binding": 1,
}
sinks = [r for r in resources if r["type"] == "google_logging_folder_sink"]
assert sorted([r["index"] for r in sinks]) == [
"debug",
"info",
"notice",
"warning",
]
values = [
(
r["index"],
r["values"]["filter"],
r["values"]["destination"],
r["values"]["include_children"],
)
for r in sinks
]
assert sorted(values) == [
(
"debug",
"severity=DEBUG",
"logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket",
False,
),
(
"info",
"severity=INFO",
"bigquery.googleapis.com/projects/myproject/datasets/mydataset",
True,
),
(
"notice",
"severity=NOTICE",
"pubsub.googleapis.com/projects/myproject/topics/mytopic",
False,
),
("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True),
]
bindings = [r for r in resources if "binding" in r["type"]]
values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings]
assert sorted(values) == [
("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"),
("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"),
("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"),
]
exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks]
assert sorted(exclusions) == [
(
"debug",
[
{
"description": None,
"disabled": False,
"filter": "logName:compute",
"name": "no-compute",
},
{
"description": None,
"disabled": False,
"filter": "logName:container",
"name": "no-container",
},
],
),
("info", []),
("notice", []),
("warning", []),
]
bindings = [r for r in resources
if 'binding' in r['type']]
values = [(r['index'], r['type'], r['values']['role'])
for r in bindings]
assert sorted(values) == [
('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
]
def test_exclusions(plan_runner):
"Test folder-level logging exclusions."
logging_exclusions = (
'{'
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
'}'
)
_, resources = plan_runner(FIXTURES_DIR,
logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [r for r in resources
if r['type'] == 'google_logging_folder_exclusion']
assert sorted([r['index'] for r in exclusions]) == [
'exclusion1',
'exclusion2',
]
values = [(r['index'], r['values']['filter']) for r in exclusions]
assert sorted(values) == [
('exclusion1', 'resource.type=gce_instance'),
('exclusion2', 'severity=NOTICE')
]
"Test folder-level logging exclusions."
logging_exclusions = (
"{"
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
"}"
)
_, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [
r for r in resources if r["type"] == "google_logging_folder_exclusion"
]
assert sorted([r["index"] for r in exclusions]) == [
"exclusion1",
"exclusion2",
]
values = [(r["index"], r["values"]["filter"]) for r in exclusions]
assert sorted(values) == [
("exclusion1", "resource.type=gce_instance"),
("exclusion2", "severity=NOTICE"),
]

13
tests/modules/logging_bucket/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

24
tests/modules/logging_bucket/fixture/main.tf Normal file
View File

@ -0,0 +1,24 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
module "test" {
source = "../../../../modules/logging-bucket"
parent_type = var.parent_type
parent = var.parent
id = var.id
retention = var.retention
location = var.location
}

42
tests/modules/logging_bucket/fixture/variables.tf Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "parent" {
type = string
}
variable "parent_type" {
type = string
validation {
condition = contains(["project", "folder", "organization", "billing_account"], var.parent_type)
error_message = "Parent type must be project, folder, organization or billing_account."
}
}
variable "location" {
type = string
default = "global"
}
variable "id" {
type = string
default = "mybucket"
}
variable "retention" {
type = number
default = 30
}

86
tests/modules/logging_bucket/test_plan.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import pytest
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_project_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(FIXTURES_DIR, parent_type="project", parent="myproject")
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_project_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"project": "myproject",
"location": "global",
"retention_days": 30,
}
def test_folder_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(
FIXTURES_DIR, parent_type="folder", parent="folders/0123456789"
)
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_folder_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"folder": "folders/0123456789",
"location": "global",
"retention_days": 30,
}
def test_organization_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(
FIXTURES_DIR, parent_type="organization", parent="organizations/0123456789"
)
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_organization_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"organization": "organizations/0123456789",
"location": "global",
"retention_days": 30,
}
def test_billing_account_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(
FIXTURES_DIR, parent_type="billing_account", parent="0123456789"
)
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_billing_account_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"billing_account": "0123456789",
"location": "global",
"retention_days": 30,
}

1
tests/modules/organization/fixture/variables.tf
View File

@ -81,6 +81,7 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
exclusions = map(string)
}))
default = {}
}

181
tests/modules/organization/test_plan_logging.py
View File

@ -18,18 +18,19 @@ import pytest
from collections import Counter
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_sinks(plan_runner):
"Test folder-level sinks."
logging_sinks = """ {
"Test folder-level sinks."
logging_sinks = """ {
warning = {
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
@ -37,6 +38,7 @@ def test_sinks(plan_runner):
filter = "severity=INFO"
iam = true
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
@ -44,71 +46,122 @@ def test_sinks(plan_runner):
filter = "severity=NOTICE"
iam = true
include_children = false
exclusions = {}
}
debug = {
type = "logging"
destination = "projects/myproject/locations/global/buckets/mybucket"
filter = "severity=DEBUG"
iam = true
include_children = false
exclusions = {
no-compute = "logName:compute"
no-container = "logName:container"
}
}
}
"""
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 6
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 7
resource_types = Counter([r['type'] for r in resources])
assert resource_types == {
'google_bigquery_dataset_iam_binding': 1,
'google_logging_organization_sink': 3,
'google_pubsub_topic_iam_binding': 1,
'google_storage_bucket_iam_binding': 1
}
sinks = [r for r in resources
if r['type'] == 'google_logging_organization_sink']
assert sorted([r['index'] for r in sinks]) == [
'info',
'notice',
'warning',
]
values = [(r['index'], r['values']['filter'], r['values']['destination'],
r['values']['include_children'])
for r in sinks]
assert sorted(values) == [
('info',
'severity=INFO',
'bigquery.googleapis.com/projects/myproject/datasets/mydataset',
True),
('notice',
'severity=NOTICE',
'pubsub.googleapis.com/projects/myproject/topics/mytopic',
False),
('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)]
resource_types = Counter([r["type"] for r in resources])
assert resource_types == {
"google_bigquery_dataset_iam_binding": 1,
"google_logging_organization_sink": 4,
"google_pubsub_topic_iam_binding": 1,
"google_storage_bucket_iam_binding": 1,
}
sinks = [r for r in resources if r["type"] == "google_logging_organization_sink"]
assert sorted([r["index"] for r in sinks]) == [
"debug",
"info",
"notice",
"warning",
]
values = [
(
r["index"],
r["values"]["filter"],
r["values"]["destination"],
r["values"]["include_children"],
)
for r in sinks
]
assert sorted(values) == [
(
"debug",
"severity=DEBUG",
"logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket",
False,
),
(
"info",
"severity=INFO",
"bigquery.googleapis.com/projects/myproject/datasets/mydataset",
True,
),
(
"notice",
"severity=NOTICE",
"pubsub.googleapis.com/projects/myproject/topics/mytopic",
False,
),
("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True),
]
bindings = [r for r in resources if "binding" in r["type"]]
values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings]
assert sorted(values) == [
("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"),
("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"),
("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"),
]
exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks]
assert sorted(exclusions) == [
(
"debug",
[
{
"description": None,
"disabled": False,
"filter": "logName:compute",
"name": "no-compute",
},
{
"description": None,
"disabled": False,
"filter": "logName:container",
"name": "no-container",
},
],
),
("info", []),
("notice", []),
("warning", []),
]
bindings = [r for r in resources
if 'binding' in r['type']]
values = [(r['index'], r['type'], r['values']['role'])
for r in bindings]
assert sorted(values) == [
('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
]
def test_exclusions(plan_runner):
"Test folder-level logging exclusions."
logging_exclusions = (
'{'
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
'}'
)
_, resources = plan_runner(FIXTURES_DIR,
logging_exclusions=logging_exclusions)
assert len(resources) == 2
exclusions = [r for r in resources
if r['type'] == 'google_logging_organization_exclusion']
assert sorted([r['index'] for r in exclusions]) == [
'exclusion1',
'exclusion2',
]
values = [(r['index'], r['values']['filter']) for r in exclusions]
assert sorted(values) == [
('exclusion1', 'resource.type=gce_instance'),
('exclusion2', 'severity=NOTICE')
]
"Test folder-level logging exclusions."
logging_exclusions = (
"{"
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
"}"
)
_, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions)
assert len(resources) == 2
exclusions = [
r for r in resources if r["type"] == "google_logging_organization_exclusion"
]
assert sorted([r["index"] for r in exclusions]) == [
"exclusion1",
"exclusion2",
]
values = [(r["index"], r["values"]["filter"]) for r in exclusions]
assert sorted(values) == [
("exclusion1", "resource.type=gce_instance"),
("exclusion2", "severity=NOTICE"),
]

10
tests/modules/project/fixture/variables.tf
View File

@ -96,10 +96,12 @@ variable "services" {
variable "logging_sinks" {
type = map(object({
destination = string
type = string
filter = string
iam = bool
destination = string
type = string
filter = string
iam = bool
exclusions = map(string)
unique_writer = bool
}))
default = {}
}

199
tests/modules/project/test_plan_logging.py
View File

@ -18,92 +18,151 @@ import pytest
from collections import Counter
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_sinks(plan_runner):
"Test folder-level sinks."
logging_sinks = """ {
"Test folder-level sinks."
logging_sinks = """ {
warning = {
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
exclusions = {}
unique_writer = false
}
info = {
type = "bigquery"
destination = "projects/myproject/datasets/mydataset"
filter = "severity=INFO"
iam = true
exclusions = {}
unique_writer = false
}
notice = {
type = "pubsub"
destination = "projects/myproject/topics/mytopic"
filter = "severity=NOTICE"
iam = true
type = "pubsub"
destination = "projects/myproject/topics/mytopic"
filter = "severity=NOTICE"
iam = true
exclusions = {}
unique_writer = false
}
debug = {
type = "logging"
destination = "projects/myproject/locations/global/buckets/mybucket"
filter = "severity=DEBUG"
iam = true
exclusions = {
no-compute = "logName:compute"
no-container = "logName:container"
}
unique_writer = true
}
}
"""
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 7
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 8
resource_types = Counter([r['type'] for r in resources])
assert resource_types == {
'google_bigquery_dataset_iam_binding': 1,
'google_logging_project_sink': 3,
'google_project': 1,
'google_pubsub_topic_iam_binding': 1,
'google_storage_bucket_iam_binding': 1
}
sinks = [r for r in resources
if r['type'] == 'google_logging_project_sink']
assert sorted([r['index'] for r in sinks]) == [
'info',
'notice',
'warning',
]
values = [(r['index'], r['values']['filter'], r['values']['destination'])
for r in sinks]
assert sorted(values) == [
('info',
'severity=INFO',
'bigquery.googleapis.com/projects/myproject/datasets/mydataset'),
('notice',
'severity=NOTICE',
'pubsub.googleapis.com/projects/myproject/topics/mytopic'),
('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')]
resource_types = Counter([r["type"] for r in resources])
assert resource_types == {
"google_bigquery_dataset_iam_binding": 1,
"google_logging_project_sink": 4,
"google_project": 1,
"google_pubsub_topic_iam_binding": 1,
"google_storage_bucket_iam_binding": 1,
}
sinks = [r for r in resources if r["type"] == "google_logging_project_sink"]
assert sorted([r["index"] for r in sinks]) == [
"debug",
"info",
"notice",
"warning",
]
values = [
(
r["index"],
r["values"]["filter"],
r["values"]["destination"],
r["values"]["unique_writer_identity"],
)
for r in sinks
]
assert sorted(values) == [
(
"debug",
"severity=DEBUG",
"logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket",
True,
),
(
"info",
"severity=INFO",
"bigquery.googleapis.com/projects/myproject/datasets/mydataset",
False,
),
(
"notice",
"severity=NOTICE",
"pubsub.googleapis.com/projects/myproject/topics/mytopic",
False,
),
("warning", "severity=WARNING", "storage.googleapis.com/mybucket", False),
]
bindings = [r for r in resources if "binding" in r["type"]]
values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings]
assert sorted(values) == [
("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"),
("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"),
("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"),
]
exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks]
assert sorted(exclusions) == [
(
"debug",
[
{
"description": None,
"disabled": False,
"filter": "logName:compute",
"name": "no-compute",
},
{
"description": None,
"disabled": False,
"filter": "logName:container",
"name": "no-container",
},
],
),
("info", []),
("notice", []),
("warning", []),
]
bindings = [r for r in resources
if 'binding' in r['type']]
values = [(r['index'], r['type'], r['values']['role'])
for r in bindings]
assert sorted(values) == [
('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
]
def test_exclusions(plan_runner):
"Test folder-level logging exclusions."
logging_exclusions = (
'{'
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
'}'
)
_, resources = plan_runner(FIXTURES_DIR,
logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [r for r in resources
if r['type'] == 'google_logging_project_exclusion']
assert sorted([r['index'] for r in exclusions]) == [
'exclusion1',
'exclusion2',
]
values = [(r['index'], r['values']['filter']) for r in exclusions]
assert sorted(values) == [
('exclusion1', 'resource.type=gce_instance'),
('exclusion2', 'severity=NOTICE')
]
"Test folder-level logging exclusions."
logging_exclusions = (
"{"
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
"}"
)
_, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [
r for r in resources if r["type"] == "google_logging_project_exclusion"
]
assert sorted([r["index"] for r in exclusions]) == [
"exclusion1",
"exclusion2",
]
values = [(r["index"], r["values"]["filter"]) for r in exclusions]
assert sorted(values) == [
("exclusion1", "resource.type=gce_instance"),
("exclusion2", "severity=NOTICE"),
]