Merge pull request #987 from GoogleCloudPlatform/jccb/test-factory-examples

Add tests to factory examples

blueprints/cloud-operations/iam-delegated-role-grants/README.md

@@ -27,13 +27,12 @@ By changing the `restricted_role_grant`, the blueprint can be used to grant admi
 
 You can easily configure the blueprint for this use case:
 
-```hcl
+```tfvars
 # terraform.tfvars
 
 delegated_role_grants = ["roles/compute.networkUser"]
-direct_role_grants = []
+direct_role_grants    = []
 restricted_role_grant = "roles/compute.networkAdmin"
-# tftest skip
 ```
 
 This diagram shows the resources and expected behaviour:

blueprints/data-solutions/data-platform-foundations/README.md

@@ -135,7 +135,6 @@ service_encryption_keys = {
     storage  = "KEY_URL_MULTIREGIONAL"
     pubsub   = "KEY_URL_MULTIREGIONAL"
 }
-# tftest skip
 ```
 
 This step is optional and depends on customer policies and security best practices.
@@ -198,8 +197,7 @@ billing_account_id  = "111111-222222-333333"
 older_id            = "folders/123456789012"
 organization_domain = "domain.com"
 prefix              = "myco"
-# tftest skip`
+```
-``
 
 For more fine details check variables on [`variables.tf`](./variables.tf) and update according to the desired configuration. Remember to create team groups described [below](#groups).
 

blueprints/networking/hub-and-spoke-vpn/README.md

@@ -35,7 +35,7 @@ You can easily create such a project by commenting turning on project creation i
 
 ```hcl
 module "project" {
-  source           = "../../..//modules/project"
+  source           = "../../../modules/project"
   name             = var.project_id
   # comment or remove this line to enable project creation
   # project_create = false

modules/__experimental/net-dns-policy-address/README.md

@@ -12,7 +12,7 @@ module "dns-policy-addresses" {
   project_id = "myproject"
   regions    = ["europe-west1", "europe-west3"]
 }
-# tftest skip
+# tftest skip (uses data sources)
 ```
 
 The output is a map with lists of addresses of type `DNS_RESOLVER` for each region specified in variables.

modules/endpoints/README.md

@@ -11,14 +11,24 @@ module "endpoint" {
   source         = "./fabric/modules/endpoints"
   project_id     = "my-project"
   service_name   = "YOUR-API.endpoints.YOUR-PROJECT-ID.cloud.goog"
-  openapi_config = { "yaml_path" = "openapi.yaml" }
+  openapi_config = { "yaml_path" = "configs/endpoints/openapi.yaml" }
   iam = {
     "servicemanagement.serviceController" = [
       "serviceAccount:123456890-compute@developer.gserviceaccount.com"
     ]
   }
 }
-# tftest skip
+# tftest modules=1 resources=2 files=openapi
+```
+
+```yaml
+# tftest file openapi configs/endpoints/openapi.yaml
+swagger: "2.0"
+info:
+  description: "A simple Google Cloud Endpoints API example."
+  title: "Endpoints Example"
+  version: "1.0.0"
+host: "echo-api.endpoints.YOUR-PROJECT-ID.cloud.goog"
 ```
 
 [Here](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/endpoints/getting-started/openapi.yaml) you can find an example of an openapi.yaml file. Once created the endpoint, remember to activate the service at project level.

modules/folder/README.md

@@ -89,20 +89,19 @@ module "folder" {
   parent = "organizations/1234567890"
   name  = "Folder name"
   firewall_policy_factory = {
-    cidr_file   = "data/cidrs.yaml"
+    cidr_file   = "configs/firewall-policies/cidrs.yaml"
     policy_name = null
-    rules_file  = "data/rules.yaml"
+    rules_file  = "configs/firewall-policies/rules.yaml"
   }
   firewall_policy_association = {
     factory-policy = module.folder.firewall_policy_id["factory"]
   }
 }
-# tftest skip
+# tftest modules=1 resources=5 files=cidrs,rules
 ```
 
 ```yaml
-# cidrs.yaml
+# tftest file cidrs configs/firewall-policies/cidrs.yaml
-
 rfc1918:
   - 10.0.0.0/8
   - 172.16.0.0/12
@@ -110,8 +109,7 @@ rfc1918:
 ```
 
 ```yaml
-# rules.yaml
+# tftest file rules configs/firewall-policies/rules.yaml
-
 allow-admins:
   description: Access from the admin subnet to all subnets
   direction: INGRESS

modules/folder/organization-policies.tf

@@ -17,14 +17,10 @@
 # tfdoc:file:description Folder-level organization policies.
 
 locals {
-  _factory_data_raw = (
+  _factory_data_raw = merge([
-    var.org_policies_data_path == null
+    for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) :
-    ? tomap({})
+    yamldecode(file("${var.org_policies_data_path}/${f}"))
-    : merge([
+  ]...)
-      for f in fileset(var.org_policies_data_path, "*.yaml") :
-      yamldecode(file("${var.org_policies_data_path}/${f}"))
-    ]...)
-  )
 
   # simulate applying defaults to data coming from yaml files
   _factory_data = {

modules/kms/README.md

@@ -23,7 +23,7 @@ module "kms" {
   keyring_create = false
   keys           = { key-a = null, key-b = null, key-c = null }
 }
-# tftest skip
+# tftest skip (uses data sources)
 ```
 
 ### Keyring creation and crypto key rotation and IAM roles

modules/net-vpc-firewall/README.md

@@ -138,16 +138,16 @@ module "firewall" {
   project_id       = "my-project"
   network          = "my-network"
   factories_config = {
-    rules_folder  = "config/firewall"
+    rules_folder  = "configs/firewal/rules"
-    cidr_tpl_file = "config/cidr_template.yaml"
+    cidr_tpl_file = "configs/firewal/cidr_template.yaml"
   }
 
 }
-# tftest skip
+# tftest modules=1 resources=3
 ```
 
 ```yaml
-# ./config/firewall/load_balancers.yaml
+# tftest file configs/firewall/rules/load_balancers.yaml
 allow-healthchecks:
   description: Allow ingress from healthchecks.
   ranges:
@@ -161,13 +161,12 @@ allow-healthchecks:
 ```
 
 ```yaml
-# ./config/cidr_template.yaml
+# tftest file configs/firewall/cidr_template.yaml
 healthchecks:
   - 35.191.0.0/16
   - 130.211.0.0/22
   - 209.85.152.0/22
   - 209.85.204.0/22
-
 ```
 <!-- BEGIN TFDOC -->
 

modules/net-vpc/README.md

@@ -233,11 +233,11 @@ module "vpc" {
   name        = "my-network"
   data_folder = "config/subnets"
 }
-# tftest skip
+# tftest modules=1 resources=1 file=subnets
 ```
 
 ```yaml
-# ./config/subnets/subnet-name.yaml
+# tftest file subnets ./config/subnets/subnet-name.yaml
 region: europe-west1
 description: Sample description
 ip_cidr_range: 10.0.0.0/24

modules/organization/README.md

@@ -137,16 +137,15 @@ module "org" {
   source          = "./fabric/modules/organization"
   organization_id = var.organization_id
   
-  org_policy_custom_constraints_data_path = "/my/path"
+  org_policy_custom_constraints_data_path = "configs/custom-constraints"
-
 }
-# tftest skip
+# tftest modules=1 resources=3 files=gke,dataproc
 ```
 
 ```yaml
-# /my/path/gke.yaml
+# tftest file gke configs/custom-constraints/gke.yaml
 custom.gkeEnableLogging:
-  resource_types: 
+  resource_types:
   - container.googleapis.com/Cluster
   method_types:
   - CREATE
@@ -155,7 +154,7 @@ custom.gkeEnableLogging:
   action_type: DENY
   display_name: Do not disable Cloud Logging
 custom.gkeEnableAutoUpgrade:
-  resource_types: 
+  resource_types:
   - container.googleapis.com/NodePool
   method_types:
   - CREATE
@@ -166,10 +165,9 @@ custom.gkeEnableAutoUpgrade:
 ```
 
 ```yaml
-# /my/path/dataproc.yaml
+# tftest file dataproc configs/custom-constraints/dataproc.yaml
-
+custom.dataprocNoMoreThan10Workers:
-custom.dataprocNoMoreThan10Workers
+  resource_types:
-  resource_types: 
   - dataproc.googleapis.com/Cluster
   method_types:
   - CREATE
@@ -228,20 +226,19 @@ module "org" {
   source          = "./fabric/modules/organization"
   organization_id = var.organization_id
   firewall_policy_factory = {
-    cidr_file   = "data/cidrs.yaml"
+    cidr_file   = "configs/firewall-policies/cidrs.yaml"
     policy_name = null
-    rules_file  = "data/rules.yaml"
+    rules_file  = "configs/firewall-policies/rules.yaml"
   }
   firewall_policy_association = {
     factory-policy = module.org.firewall_policy_id["factory"]
   }
 }
-# tftest skip
+# tftest modules=1 resources=4 files=cidrs,rules
 ```
 
 ```yaml
-# cidrs.yaml
+# tftest file cidrs configs/firewall-policies/cidrs.yaml
-
 rfc1918:
   - 10.0.0.0/8
   - 172.16.0.0/12
@@ -249,8 +246,7 @@ rfc1918:
 ```
 
 ```yaml
-# rules.yaml
+# tftest file rules configs/firewall-policies/rules.yaml
-
 allow-admins:
   description: Access from the admin subnet to all subnets
   direction: INGRESS

modules/organization/org-policy-custom-constraints.tf

@@ -15,14 +15,11 @@
  */
 
 locals {
-  _custom_constraints_factory_data_raw = (
+  _custom_constraints_factory_data_raw = merge([
-    var.org_policy_custom_constraints_data_path == null
+    for f in try(fileset(var.org_policy_custom_constraints_data_path, "*.yaml"), []) :
-    ? tomap({})
+    yamldecode(file("${var.org_policy_custom_constraints_data_path}/${f}"))
-    : tomap(merge([
+  ]...)
-      for f in fileset(var.org_policy_custom_constraints_data_path, "*.yaml") :
+
-      yamldecode(file("${var.org_policy_custom_constraints_data_path}/${f}"))
-    ]...))
-  )
 
   _custom_constraints_factory_data = {
     for k, v in local._custom_constraints_factory_data_raw :

modules/organization/organization-policies.tf

@@ -17,14 +17,10 @@
 # tfdoc:file:description Organization-level organization policies.
 
 locals {
-  _factory_data_raw = (
+  _factory_data_raw = merge([
-    var.org_policies_data_path == null
+    for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) :
-    ? tomap({})
+    yamldecode(file("${var.org_policies_data_path}/${f}"))
-    : merge([
+  ]...)
-      for f in fileset(var.org_policies_data_path, "*.yaml") :
-      yamldecode(file("${var.org_policies_data_path}/${f}"))
-    ]...)
-  )
 
   # simulate applying defaults to data coming from yaml files
   _factory_data = {

modules/project/README.md

@@ -224,13 +224,13 @@ module "folder" {
   source                 = "./fabric/modules/folder"
   parent                 = "organizations/1234567890"
   name                   = "Folder name"
-  org_policies_data_path = "/my/path"
+  org_policies_data_path = "configs/org-policies/"
 }
-# tftest skip
+# tftest modules=1 resources=6 files=boolean,list
 ```
 
 ```yaml
-# /my/path/boolean.yaml
+# tftest file boolean configs/org-policies/boolean.yaml
 iam.disableServiceAccountKeyCreation:
   enforce: true
 
@@ -246,7 +246,7 @@ iam.disableServiceAccountKeyUpload:
 ```
 
 ```yaml
-# /my/path/list.yaml
+# tftest file list configs/org-policies/list.yaml
 compute.vmExternalIpAccess:
   deny:
     all: true

modules/project/organization-policies.tf

@@ -17,14 +17,10 @@
 # tfdoc:file:description Project-level organization policies.
 
 locals {
-  _factory_data_raw = (
+  _factory_data_raw = merge([
-    var.org_policies_data_path == null
+    for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) :
-    ? tomap({})
+    yamldecode(file("${var.org_policies_data_path}/${f}"))
-    : merge([
+  ]...)
-      for f in fileset(var.org_policies_data_path, "*.yaml") :
-      yamldecode(file("${var.org_policies_data_path}/${f}"))
-    ]...)
-  )
 
   # simulate applying defaults to data coming from yaml files
   _factory_data = {

modules/projects-data-source/README.md

@@ -22,7 +22,7 @@ output "folders" {
   value = module.my-org.folders
 }
 
-# tftest skip
+# tftest skip (uses data sources)
 ```
 
 ### My dev projects based on parent and label
@@ -42,7 +42,7 @@ output "dev-folders" {
   value = module.my-dev.folders
 }
 
-# tftest skip
+# tftest skip (uses data sources)
 ```
 <!-- BEGIN TFDOC -->
 

tests/examples/conftest.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
+import re
 from pathlib import Path
 
 import marko
@@ -21,6 +23,11 @@ FABRIC_ROOT = Path(__file__).parents[2]
 MODULES_PATH = FABRIC_ROOT / 'modules/'
 BLUEPRINTS_PATH = FABRIC_ROOT / 'blueprints/'
 
+FILE_TEST_RE = re.compile(r'# tftest file (\w+) ([\S]+)')
+
+Example = collections.namedtuple('Example', 'code files')
+File = collections.namedtuple('File', 'path content')
+
 
 def pytest_generate_tests(metafunc):
   if 'example' in metafunc.fixturenames:
@@ -37,19 +44,30 @@ def pytest_generate_tests(metafunc):
       doc = marko.parse(readme.read_text())
       index = 0
       last_header = None
-      mark = pytest.mark.xdist_group(name=module.name)
+      files = {}
+
+      #first pass: collect all tftest tagged files
       for child in doc.children:
-        if isinstance(child, marko.block.FencedCode) and child.lang == 'hcl':
+        if isinstance(child, marko.block.FencedCode):
+          code = child.children[0].children
+          match = FILE_TEST_RE.search(code)
+          if match:
+            name, path = match.groups()
+            files[name] = File(path, code)
+
+      for child in doc.children:
+        if isinstance(child, marko.block.FencedCode):
           index += 1
           code = child.children[0].children
           if 'tftest skip' in code:
             continue
-          examples.append(pytest.param(code, marks=mark))
+          if child.lang == 'hcl':
-          path = module.relative_to(FABRIC_ROOT)
+            examples.append(Example(code, files))
-          name = f'{path}:{last_header}'
+            path = module.relative_to(FABRIC_ROOT)
-          if index > 1:
+            name = f'{path}:{last_header}'
-            name += f' {index}'
+            if index > 1:
-          ids.append(name)
+              name += f' {index}'
+            ids.append(name)
         elif isinstance(child, marko.block.Heading):
           last_header = child.children[0].children
           index = 0

tests/examples/test_plan.py

@@ -16,20 +16,30 @@ import re
 from pathlib import Path
 
 BASE_PATH = Path(__file__).parent
-EXPECTED_RESOURCES_RE = re.compile(r'# tftest modules=(\d+) resources=(\d+)')
+COUNT_TEST_RE = re.compile(
+    r'# tftest modules=(\d+) resources=(\d+)(?: files=([\w,]+))?')
 
 
 def test_example(recursive_e2e_plan_runner, tmp_path, example):
-  (tmp_path / 'fabric').symlink_to(Path(BASE_PATH, '../../').resolve())
+  if match := COUNT_TEST_RE.search(example.code):
-  (tmp_path / 'variables.tf').symlink_to(
+    (tmp_path / 'fabric').symlink_to(Path(BASE_PATH, '../../'))
-      Path(BASE_PATH, 'variables.tf').resolve())
+    (tmp_path / 'variables.tf').symlink_to(Path(BASE_PATH, 'variables.tf'))
-  (tmp_path / 'main.tf').write_text(example)
+    (tmp_path / 'main.tf').write_text(example.code)
 
-  match = EXPECTED_RESOURCES_RE.search(example)
+    if match.group(3) is not None:
-  expected_modules = int(match.group(1)) if match is not None else 1
+      requested_files = match.group(3).split(',')
-  expected_resources = int(match.group(2)) if match is not None else 1
+      for f in requested_files:
+        destination = tmp_path / example.files[f].path
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(example.files[f].content)
 
-  num_modules, num_resources = recursive_e2e_plan_runner(
+    expected_modules = int(match.group(1)) if match is not None else 1
-      str(tmp_path), tmpdir=False)
+    expected_resources = int(match.group(2)) if match is not None else 1
-  assert expected_modules == num_modules
+
-  assert expected_resources == num_resources
+    num_modules, num_resources = recursive_e2e_plan_runner(
+        str(tmp_path), tmpdir=False)
+    assert expected_modules == num_modules, 'wrong number of modules'
+    assert expected_resources == num_resources, 'wrong number of resources'
+
+  else:
+    assert False, "can't find tftest directive"

tests/modules/organization/test_plan_org_policies_modules.py

@@ -38,8 +38,8 @@ def test_policy_implementation():
       '+# tfdoc:file:description Folder-level organization policies.\n',
       ' \n',
       ' locals {\n',
-      '   _factory_data_raw = (\n',
+      '   _factory_data_raw = merge([\n',
-      '@@ -69,8 +69,8 @@\n',
+      '@@ -65,8 +65,8 @@\n',
       '   org_policies = {\n',
       '     for k, v in local._org_policies :\n',
       '     k => merge(v, {\n',
@@ -64,8 +64,8 @@ def test_policy_implementation():
       '+# tfdoc:file:description Organization-level organization policies.\n',
       ' \n',
       ' locals {\n',
-      '   _factory_data_raw = (\n',
+      '   _factory_data_raw = merge([\n',
-      '@@ -69,8 +69,8 @@\n',
+      '@@ -65,8 +65,8 @@\n',
       '   org_policies = {\n',
       '     for k, v in local._org_policies :\n',
       '     k => merge(v, {\n',
@@ -76,7 +76,7 @@ def test_policy_implementation():
       ' \n',
       '       is_boolean_policy = v.allow == null && v.deny == null\n',
       '       has_values = (\n',
-      '@@ -143,4 +143,13 @@\n',
+      '@@ -139,4 +139,13 @@\n',
       '       }\n',
       '     }\n',
       '   }\n',