Add tests for logging sinks

tests/foundations/business_units/test_plan.py

@@ -23,5 +23,5 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
 def test_resources(e2e_plan_runner):
   "Test that plan works and the numbers of resources is as expected."
   modules, resources = e2e_plan_runner(FIXTURES_DIR)
-  assert len(modules) == 9
-  assert len(resources) == 84
+  assert len(modules) == 8
+  assert len(resources) == 82

tests/modules/folder/fixture/main.tf

@@ -23,4 +23,6 @@ module "test" {
   policy_list                 = var.policy_list
   firewall_policies           = var.firewall_policies
   firewall_policy_attachments = var.firewall_policy_attachments
+  logging_sinks               = var.logging_sinks
+  logging_exclusions          = var.logging_exclusions
 }

tests/modules/folder/fixture/variables.tf

@@ -53,3 +53,18 @@ variable "firewall_policy_attachments" {
   type    = map(string)
   default = {}
 }
+
+variable "logging_sinks" {
+  type = map(object({
+    destination = string
+    type        = string
+    filter      = string
+    iam         = bool
+  }))
+  default = {}
+}
+
+variable "logging_exclusions" {
+  type    = map(string)
+  default = {}
+}

tests/modules/folder/test_plan_logging.py

@@ -0,0 +1,109 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import pytest
+
+from collections import Counter
+
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
+
+
+def test_sinks(plan_runner):
+  "Test folder-level sinks."
+  logging_sinks = """ {
+    warning = {
+      type        = "gcs"
+      destination = "mybucket"
+      filter      = "severity=WARNING"
+      iam         = true
+    }
+    info = {
+      type        = "bigquery"
+      destination = "projects/myproject/datasets/mydataset"
+      filter      = "severity=INFO"
+      iam         = true
+    }
+    notice = {
+      type        = "pubsub"
+      destination = "projects/myproject/topics/mytopic"
+      filter      = "severity=NOTICE"
+      iam         = true
+    }
+  }
+  """
+  _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
+  assert len(resources) == 7
+
+  resource_types = Counter([r['type'] for r in resources])
+  assert resource_types == {
+    'google_bigquery_dataset_iam_binding': 1,
+    'google_folder': 1,
+    'google_logging_folder_sink': 3,
+    'google_pubsub_topic_iam_binding': 1,
+    'google_storage_bucket_iam_binding': 1
+  }
+  
+  sinks = [r for r in resources
+           if r['type'] == 'google_logging_folder_sink']
+  assert sorted([r['index'] for r in sinks]) == [
+      'info',
+      'notice',
+      'warning',
+  ]
+  values = [(r['index'], r['values']['filter'], r['values']['destination'])
+            for r in sinks]
+  assert sorted(values) == [
+    ('info',
+     'severity=INFO',
+     'bigquery.googleapis.com/projects/myproject/datasets/mydataset'),
+    ('notice',
+     'severity=NOTICE',
+     'pubsub.googleapis.com/projects/myproject/topics/mytopic'),
+    ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')]
+
+  bindings = [r for r in resources
+              if 'binding' in r['type']]
+  values = [(r['index'], r['type'], r['values']['role'])
+            for r in bindings]
+  assert sorted(values) == [
+    ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
+    ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
+    ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
+  ]
+
+  
+def test_exclusions(plan_runner):
+  "Test folder-level logging exclusions."
+  logging_exclusions = (
+      '{'
+      'exclusion1 = "resource.type=gce_instance", '
+      'exclusion2 = "severity=NOTICE", '
+      '}'
+  )
+  _, resources = plan_runner(FIXTURES_DIR,
+                             logging_exclusions=logging_exclusions)
+  assert len(resources) == 3
+  exclusions = [r for r in resources
+                if r['type'] == 'google_logging_folder_exclusion']
+  assert sorted([r['index'] for r in exclusions]) == [
+      'exclusion1',
+      'exclusion2',
+  ]
+  values = [(r['index'], r['values']['filter']) for r in exclusions]
+  assert sorted(values) == [
+    ('exclusion1', 'resource.type=gce_instance'),
+    ('exclusion2', 'severity=NOTICE')
+  ]

tests/modules/folder/test_plan_org_policies.py

@@ -20,8 +20,8 @@ import pytest
 FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
 
 
-def test_policy_boolean(plan_runner):
-  "Test boolean folder policy."
+def test_sink(plan_runner):
+  "Test folder-level sink."
   policy_boolean = '{policy-a = true, policy-b = false, policy-c = null}'
   _, resources = plan_runner(FIXTURES_DIR, policy_boolean=policy_boolean)
 
@@ -46,8 +46,8 @@ def test_policy_boolean(plan_runner):
   ]
 
 
-def test_policy_list(plan_runner):
-  "Test list org policy."
+def test_exclussions(plan_runner):
+  "Test folder-level logging exclusions."
   policy_list = (
       '{'
       'policy-a = {inherit_from_parent = true, suggested_value = null, status = true, values = []}, '

tests/modules/organization/fixture/main.tf

@@ -26,4 +26,6 @@ module "test" {
   policy_list                 = var.policy_list
   firewall_policies           = var.firewall_policies
   firewall_policy_attachments = var.firewall_policy_attachments
+  logging_sinks               = var.logging_sinks
+  logging_exclusions          = var.logging_exclusions
 }

tests/modules/organization/fixture/variables.tf

@@ -73,3 +73,18 @@ variable "firewall_policy_attachments" {
   type    = map(string)
   default = {}
 }
+
+variable "logging_sinks" {
+  type = map(object({
+    destination = string
+    type        = string
+    filter      = string
+    iam         = bool
+  }))
+  default = {}
+}
+
+variable "logging_exclusions" {
+  type    = map(string)
+  default = {}
+}

tests/modules/organization/test_plan_logging.py

@@ -0,0 +1,108 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import pytest
+
+from collections import Counter
+
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
+
+
+def test_sinks(plan_runner):
+  "Test folder-level sinks."
+  logging_sinks = """ {
+    warning = {
+      type        = "gcs"
+      destination = "mybucket"
+      filter      = "severity=WARNING"
+      iam         = true
+    }
+    info = {
+      type        = "bigquery"
+      destination = "projects/myproject/datasets/mydataset"
+      filter      = "severity=INFO"
+      iam         = true
+    }
+    notice = {
+      type        = "pubsub"
+      destination = "projects/myproject/topics/mytopic"
+      filter      = "severity=NOTICE"
+      iam         = true
+    }
+  }
+  """
+  _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
+  assert len(resources) == 6
+
+  resource_types = Counter([r['type'] for r in resources])
+  assert resource_types == {
+    'google_bigquery_dataset_iam_binding': 1,
+    'google_logging_organization_sink': 3,
+    'google_pubsub_topic_iam_binding': 1,
+    'google_storage_bucket_iam_binding': 1
+  }
+  
+  sinks = [r for r in resources
+           if r['type'] == 'google_logging_organization_sink']
+  assert sorted([r['index'] for r in sinks]) == [
+      'info',
+      'notice',
+      'warning',
+  ]
+  values = [(r['index'], r['values']['filter'], r['values']['destination'])
+            for r in sinks]
+  assert sorted(values) == [
+    ('info',
+     'severity=INFO',
+     'bigquery.googleapis.com/projects/myproject/datasets/mydataset'),
+    ('notice',
+     'severity=NOTICE',
+     'pubsub.googleapis.com/projects/myproject/topics/mytopic'),
+    ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')]
+
+  bindings = [r for r in resources
+              if 'binding' in r['type']]
+  values = [(r['index'], r['type'], r['values']['role'])
+            for r in bindings]
+  assert sorted(values) == [
+    ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
+    ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
+    ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
+  ]
+
+  
+def test_exclusions(plan_runner):
+  "Test folder-level logging exclusions."
+  logging_exclusions = (
+      '{'
+      'exclusion1 = "resource.type=gce_instance", '
+      'exclusion2 = "severity=NOTICE", '
+      '}'
+  )
+  _, resources = plan_runner(FIXTURES_DIR,
+                             logging_exclusions=logging_exclusions)
+  assert len(resources) == 2
+  exclusions = [r for r in resources
+                if r['type'] == 'google_logging_organization_exclusion']
+  assert sorted([r['index'] for r in exclusions]) == [
+      'exclusion1',
+      'exclusion2',
+  ]
+  values = [(r['index'], r['values']['filter']) for r in exclusions]
+  assert sorted(values) == [
+    ('exclusion1', 'resource.type=gce_instance'),
+    ('exclusion2', 'severity=NOTICE')
+  ]

tests/modules/project/fixture/main.tf

@@ -33,4 +33,6 @@ module "test" {
   policy_list          = var.policy_list
   prefix               = var.prefix
   services             = var.services
+  logging_sinks        = var.logging_sinks
+  logging_exclusions   = var.logging_exclusions
 }

tests/modules/project/fixture/variables.tf

@@ -93,3 +93,18 @@ variable "services" {
   type    = list(string)
   default = []
 }
+
+variable "logging_sinks" {
+  type = map(object({
+    destination = string
+    type        = string
+    filter      = string
+    iam         = bool
+  }))
+  default = {}
+}
+
+variable "logging_exclusions" {
+  type    = map(string)
+  default = {}
+}

tests/modules/project/test_plan_logging.py

@@ -0,0 +1,109 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import pytest
+
+from collections import Counter
+
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
+
+
+def test_sinks(plan_runner):
+  "Test folder-level sinks."
+  logging_sinks = """ {
+    warning = {
+      type        = "gcs"
+      destination = "mybucket"
+      filter      = "severity=WARNING"
+      iam         = true
+    }
+    info = {
+      type        = "bigquery"
+      destination = "projects/myproject/datasets/mydataset"
+      filter      = "severity=INFO"
+      iam         = true
+    }
+    notice = {
+      type        = "pubsub"
+      destination = "projects/myproject/topics/mytopic"
+      filter      = "severity=NOTICE"
+      iam         = true
+    }
+  }
+  """
+  _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
+  assert len(resources) == 7
+
+  resource_types = Counter([r['type'] for r in resources])
+  assert resource_types == {
+    'google_bigquery_dataset_iam_binding': 1,
+    'google_logging_project_sink': 3,
+    'google_project': 1,
+    'google_pubsub_topic_iam_binding': 1,
+    'google_storage_bucket_iam_binding': 1
+  }
+  
+  sinks = [r for r in resources
+           if r['type'] == 'google_logging_project_sink']
+  assert sorted([r['index'] for r in sinks]) == [
+      'info',
+      'notice',
+      'warning',
+  ]
+  values = [(r['index'], r['values']['filter'], r['values']['destination'])
+            for r in sinks]
+  assert sorted(values) == [
+    ('info',
+     'severity=INFO',
+     'bigquery.googleapis.com/projects/myproject/datasets/mydataset'),
+    ('notice',
+     'severity=NOTICE',
+     'pubsub.googleapis.com/projects/myproject/topics/mytopic'),
+    ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')]
+
+  bindings = [r for r in resources
+              if 'binding' in r['type']]
+  values = [(r['index'], r['type'], r['values']['role'])
+            for r in bindings]
+  assert sorted(values) == [
+    ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
+    ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
+    ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
+  ]
+
+  
+def test_exclusions(plan_runner):
+  "Test folder-level logging exclusions."
+  logging_exclusions = (
+      '{'
+      'exclusion1 = "resource.type=gce_instance", '
+      'exclusion2 = "severity=NOTICE", '
+      '}'
+  )
+  _, resources = plan_runner(FIXTURES_DIR,
+                             logging_exclusions=logging_exclusions)
+  assert len(resources) == 3
+  exclusions = [r for r in resources
+                if r['type'] == 'google_logging_project_exclusion']
+  assert sorted([r['index'] for r in exclusions]) == [
+      'exclusion1',
+      'exclusion2',
+  ]
+  values = [(r['index'], r['values']['filter']) for r in exclusions]
+  assert sorted(values) == [
+    ('exclusion1', 'resource.type=gce_instance'),
+    ('exclusion2', 'severity=NOTICE')
+  ]