Simplify yaml test spec

tests/collectors.py

@@ -15,19 +15,44 @@
 import pytest
 import yaml
 
-from .fixtures import generic_plan_summary, generic_plan_validator
+from .fixtures import plan_summary, plan_validator
 
 
 class FabricTestFile(pytest.File):
 
   def collect(self):
+    """Read yaml test spec and yield test items for each test definition.
+
+    Test spec should contain a `module` key with the path of the
+    terraform module to test, relative to the root of the repository
+
+    All other top-level keys in the yaml are taken as test names, and
+    should have the following structure:
+
+    test-name:
+      tfvars:
+        - tfvars1.tfvars
+        - tfvars2.tfvars
+      inventory:
+        - inventory1.yaml
+        - inventory2.yaml
+
+    All paths specifications are relative to the location of the test
+    spec. The inventory key is optional, if omitted, the inventory
+    will be taken from the file test-name.yaml
+
+    """
     raw = yaml.safe_load(self.path.open())
-    module = raw['module']
+    module = raw.pop('module')
-    for test_name, spec in raw['tests'].items():
+    for test_name, spec in raw.items():
-      inventory = spec.get('inventory', f'{test_name}.yaml')
+      inventories = spec.get('inventory', [f'{test_name}.yaml'])
       tfvars = spec['tfvars']
-      yield FabricTestItem.from_parent(self, name=test_name, module=module,
+      for i in inventories:
-                                       inventory=inventory, tfvars=tfvars)
+        name = test_name
+        if isinstance(inventories, list) and len(inventories) > 1:
+          name = f'{test_name}[{i}]'
+        yield FabricTestItem.from_parent(self, name=name, module=module,
+                                         inventory=[i], tfvars=tfvars)
 
 
 class FabricTestItem(pytest.Item):
@@ -39,13 +64,14 @@ class FabricTestItem(pytest.Item):
     self.tfvars = tfvars
 
   def runtest(self):
-    s = generic_plan_validator(self.module, self.inventory,
+    s = plan_validator(self.module, self.inventory, self.parent.path.parent,
-                               self.parent.path.parent, self.tfvars)
+                       self.tfvars)
 
   def reportinfo(self):
     return self.path, None, self.name
 
 
 def pytest_collect_file(parent, file_path):
+  'Collect tftest*.yaml files and run plan_validator from them.'
   if file_path.suffix == '.yaml' and file_path.name.startswith('tftest'):
     return FabricTestFile.from_parent(parent, path=file_path)

tests/conftest.py

@@ -23,8 +23,7 @@ import tftest
 import yaml
 
 from .collectors import pytest_collect_file
-from .fixtures import generic_plan_summary_fixture, generic_plan_validator_fixture
+from .fixtures import plan_summary_fixture, plan_validator_fixture
-from .fixtures import generic_plan_summary, generic_plan_validator
 
 BASEDIR = os.path.dirname(os.path.dirname(__file__))
 

tests/fast/stages/s00_bootstrap/tftest.yaml

@@ -2,11 +2,10 @@
 
 module: fast/stages/00-bootstrap
 
-tests:
+simple:
-  simple:
+  tfvars:
-    tfvars:
+    - simple.tfvars
-      - simple.tfvars
+  inventory:
-    inventory:
+    - simple.yaml
-      - simple.yaml
+    - simple_projects.yaml
-      - simple_projects.yaml
+    - simple_sas.yaml
-      - simple_sas.yaml

tests/fixtures.py

@@ -25,7 +25,7 @@ import yaml
 PlanSummary = collections.namedtuple('PlanSummary', 'values counts outputs')
 
 
-def generic_plan_summary(module_path, basedir, tf_var_files=None, **tf_vars):
+def plan_summary(module_path, basedir, tf_var_files=None, **tf_vars):
   """
   Run a Terraform plan on the module located at `module_path`.
 
@@ -117,8 +117,8 @@ def generic_plan_summary(module_path, basedir, tf_var_files=None, **tf_vars):
     return PlanSummary(values, dict(counts), outputs)
 
 
-@pytest.fixture(name='generic_plan_summary')
+@pytest.fixture(name='plan_summary')
-def generic_plan_summary_fixture(request):
+def plan_summary_fixture(request):
   """Return a function to generate a PlanSummary.
 
   In the returned function `basedir` becomes optional and it defaults
@@ -128,17 +128,16 @@ def generic_plan_summary_fixture(request):
   def inner(module_path, basedir=None, tf_var_files=None, **tf_vars):
     if basedir is None:
       basedir = Path(request.fspath).parent
-    return generic_plan_summary(module_path=module_path, basedir=basedir,
+    return plan_summary(module_path=module_path, basedir=basedir,
-                                tf_var_files=tf_var_files, **tf_vars)
+                        tf_var_files=tf_var_files, **tf_vars)
 
   return inner
 
 
-def generic_plan_validator(module_path, inventory_paths, basedir,
+def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
-                           tf_var_files=None, **tf_vars):
+                   **tf_vars):
-  summary = generic_plan_summary(module_path=module_path,
+  summary = plan_summary(module_path=module_path, tf_var_files=tf_var_files,
-                                 tf_var_files=tf_var_files, basedir=basedir,
+                         basedir=basedir, **tf_vars)
-                                 **tf_vars)
 
   # allow single single string for inventory_paths
   if not isinstance(inventory_paths, list):
@@ -201,8 +200,8 @@ def generic_plan_validator(module_path, inventory_paths, basedir,
   return summary
 
 
-@pytest.fixture(name='generic_plan_validator')
+@pytest.fixture(name='plan_validator')
-def generic_plan_validator_fixture(request):
+def plan_validator_fixture(request):
   """Return a function to builds a PlanSummary and compare it to YAML
   inventory.
 
@@ -215,9 +214,8 @@ def generic_plan_validator_fixture(request):
             **tf_vars):
     if basedir is None:
       basedir = Path(request.fspath).parent
-    return generic_plan_validator(module_path=module_path,
+    return plan_validator(module_path=module_path,
-                                  inventory_paths=inventory_paths,
+                          inventory_paths=inventory_paths, basedir=basedir,
-                                  basedir=basedir, tf_var_files=tf_var_paths,
+                          tf_var_files=tf_var_paths, **tf_vars)
-                                  **tf_vars)
 
   return inner

tests/modules/net_vpc/test_routes.py

@@ -22,7 +22,7 @@ _route_parameters = [('gateway', 'global/gateways/default-internet-gateway'),
 
 
 @pytest.mark.parametrize('next_hop_type,next_hop', _route_parameters)
-def test_vpc_routes(generic_plan_summary, next_hop_type, next_hop):
+def test_vpc_routes(plan_summary, next_hop_type, next_hop):
   'Test vpc routes.'
 
   var_routes = '''{
@@ -40,9 +40,8 @@ def test_vpc_routes(generic_plan_summary, next_hop_type, next_hop):
       next_hop      = "global/gateways/default-internet-gateway"
     }
   }''' % (next_hop_type, next_hop)
-  summary = generic_plan_summary('modules/net-vpc',
+  summary = plan_summary('modules/net-vpc', tf_var_files=['common.tfvars'],
-                                 tf_var_files=['common.tfvars'],
+                         routes=var_routes)
-                                 routes=var_routes)
   assert len(summary.values) == 3
   route = summary.values[f'google_compute_route.{next_hop_type}["next-hop"]']
   assert route[f'next_hop_{next_hop_type}'] == next_hop

tests/modules/net_vpc/tftest.yaml

@@ -14,49 +14,46 @@
 
 module: modules/net-vpc
 
-tests:
+simple:
-  simple:
+  tfvars:
-    tfvars:
+    - common.tfvars
-      - common.tfvars
 
-  subnets:
+subnets:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - subnets.tfvars
+    - subnets.tfvars
 
-  peering:
+peering:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - peering.tfvars
+    - peering.tfvars
 
-  shared_vpc:
+shared_vpc:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - shared_vpc.tfvars
+    - shared_vpc.tfvars
 
-  factory:
+factory:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - factory.tfvars
+    - factory.tfvars
-    inventory:
-      - factory.yaml
 
-  psa_simple:
+psa_simple:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - psa_simple.tfvars
+    - psa_simple.tfvars
 
-  psa_routes_export:
+psa_routes_export:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - psa_routes_export.tfvars
+    - psa_routes_export.tfvars
 
-  psa_routes_import:
+psa_routes_import:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - psa_routes_import.tfvars
+    - psa_routes_import.tfvars
 
-  psa_routes_import_export:
+psa_routes_import_export:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - psa_routes_import_export.tfvars
+    - psa_routes_import_export.tfvars

tests/modules/organization/test_plan_org_policies.py

@@ -20,28 +20,26 @@ _params = ['boolean', 'list']
 
 
 @pytest.mark.parametrize('policy_type', _params)
-def test_policy_factory(generic_plan_summary, tfvars_to_yaml, tmp_path,
+def test_policy_factory(plan_summary, tfvars_to_yaml, tmp_path, policy_type):
-                        policy_type):
   dest = tmp_path / 'policies.yaml'
   tfvars_to_yaml(f'org_policies_{policy_type}.tfvars', dest, 'org_policies')
-  tfvars_plan = generic_plan_summary(
+  tfvars_plan = plan_summary(
       'modules/organization',
       tf_var_files=['common.tfvars', f'org_policies_{policy_type}.tfvars'])
-  yaml_plan = generic_plan_summary('modules/organization',
+  yaml_plan = plan_summary('modules/organization',
-                                   tf_var_files=['common.tfvars'],
+                           tf_var_files=['common.tfvars'],
-                                   org_policies_data_path=f'{tmp_path}')
+                           org_policies_data_path=f'{tmp_path}')
   assert tfvars_plan.values == yaml_plan.values
 
 
-def test_custom_constraint_factory(generic_plan_summary, tfvars_to_yaml,
+def test_custom_constraint_factory(plan_summary, tfvars_to_yaml, tmp_path):
-                                   tmp_path):
   dest = tmp_path / 'constraints.yaml'
   tfvars_to_yaml(f'org_policies_custom_constraints.tfvars', dest,
                  'org_policy_custom_constraints')
-  tfvars_plan = generic_plan_summary(
+  tfvars_plan = plan_summary(
       'modules/organization',
       tf_var_files=['common.tfvars', f'org_policies_custom_constraints.tfvars'])
-  yaml_plan = generic_plan_summary(
+  yaml_plan = plan_summary(
       'modules/organization', tf_var_files=['common.tfvars'],
       org_policy_custom_constraints_data_path=f'{tmp_path}')
   assert tfvars_plan.values == yaml_plan.values

tests/modules/organization/tftest.yaml

@@ -14,67 +14,64 @@
 
 module: modules/organization
 
-tests:
+audit_config:
-  audit_config:
+  tfvars:
-    tfvars:
+    - common.tfvars
-      - common.tfvars
+    - audit_config.tfvars
-      - audit_config.tfvars
 
-  iam:
+iam:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - iam.tfvars
+    - iam.tfvars
 
-  iam_additive:
+iam_additive:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - iam_additive.tfvars
+    - iam_additive.tfvars
 
-  logging:
+logging:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - logging.tfvars
+    - logging.tfvars
 
-  logging_exclusions:
+logging_exclusions:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - logging_exclusions.tfvars
+    - logging_exclusions.tfvars
 
-  org_policies_list:
+org_policies_list:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - org_policies_list.tfvars
+    - org_policies_list.tfvars
 
-  org_policies_boolean:
+org_policies_boolean:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - org_policies_boolean.tfvars
+    - org_policies_boolean.tfvars
 
-  org_policies_custom_constraints:
+org_policies_custom_constraints:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - org_policies_custom_constraints.tfvars
+    - org_policies_custom_constraints.tfvars
 
-  tags:
+tags:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - network_tags.tfvars
+    - network_tags.tfvars
-      - resource_tags.tfvars
+    - resource_tags.tfvars
-    inventory:
-      - tags.yaml
 
-  firewall_policies:
+firewall_policies:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - firewall_policies.tfvars
+    - firewall_policies.tfvars
 
-  firewall_policies_factory:
+firewall_policies_factory:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - firewall_policies_factory.tfvars
+    - firewall_policies_factory.tfvars
 
-  firewall_policies_factory_combined:
+firewall_policies_factory_combined:
-    tfvars:
+  tfvars:
-      - common.tfvars
+    - common.tfvars
-      - firewall_policies.tfvars
+    - firewall_policies.tfvars
-      - firewall_policies_factory.tfvars
+    - firewall_policies_factory.tfvars