Run tests in parallel using `pytest-xdist` (#881)

* test terraform cache

* try pytest-xdist

* revert cache changes

* extend to other jobs

* change dist, bump processes to 4

* revert

* mark tests

* run init to prime providers cache

* prime providers cache

* prime providers cache for all jobs

* add local provider to versions

* remove leftover code

.github/workflows/tests.yml

@@ -54,6 +54,15 @@ jobs:
           terraform_version: ${{ env.TF_VERSION }}
           terraform_wrapper: false
 
+      - name: Config Terraform plugin cache
+        run: |
+          echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc
+          mkdir --parents ~/.terraform.d/plugin-cache
+
+      - name: Initialize providers
+        run: |
+          terraform init
+
       # avoid conflicts with user-installed providers on local machines
       - name: Pin provider versions
         run: |
@@ -66,7 +75,7 @@ jobs:
         run: |
           mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
           pip install -r tests/requirements.txt
-          pytest -vv tests/examples
+          pytest -vv --numprocesses auto --dist loadgroup tests/examples
 
   examples:
     runs-on: ubuntu-latest
@@ -89,6 +98,15 @@ jobs:
           terraform_version: ${{ env.TF_VERSION }}
           terraform_wrapper: false
 
+      - name: Config Terraform plugin cache
+        run: |
+          echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc
+          mkdir --parents ~/.terraform.d/plugin-cache
+
+      - name: Initialize providers
+        run: |
+          terraform init
+
       # avoid conflicts with user-installed providers on local machines
       - name: Pin provider versions
         run: |
@@ -101,7 +119,7 @@ jobs:
         run: |
           mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
           pip install -r tests/requirements.txt
-          pytest -vv tests/blueprints
+          pytest -vv --numprocesses auto --dist loadgroup tests/blueprints
 
   modules:
     runs-on: ubuntu-latest
@@ -124,6 +142,15 @@ jobs:
           terraform_version: ${{ env.TF_VERSION }}
           terraform_wrapper: false
 
+      - name: Config Terraform plugin cache
+        run: |
+          echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc
+          mkdir --parents ~/.terraform.d/plugin-cache
+
+      - name: Initialize providers
+        run: |
+          terraform init
+
       # avoid conflicts with user-installed providers on local machines
       - name: Pin provider versions
         run: |
@@ -136,7 +163,7 @@ jobs:
         run: |
           mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
           pip install -r tests/requirements.txt
-          pytest -vv tests/modules
+          pytest -vv --numprocesses auto --dist loadgroup tests/modules
 
   fast:
     runs-on: ubuntu-latest
@@ -159,6 +186,15 @@ jobs:
           terraform_version: ${{ env.TF_VERSION }}
           terraform_wrapper: false
 
+      - name: Config Terraform plugin cache
+        run: |
+          echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc
+          mkdir --parents ~/.terraform.d/plugin-cache
+
+      - name: Initialize providers
+        run: |
+          terraform init
+
       # avoid conflicts with user-installed providers on local machines
       - name: Pin provider versions
         run: |
@@ -171,4 +207,4 @@ jobs:
         run: |
           mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
           pip install -r tests/requirements.txt
-          pytest -vv tests/fast
+          pytest -vv --numprocesses auto --dist loadgroup tests/fast

default-versions.tf

@@ -23,6 +23,10 @@ terraform {
       source  = "hashicorp/google-beta"
       version = ">= 4.36.0" # tftest
     }
+    # used in fast
+    local = {
+      source = "hashicorp/local"
+    }
   }
 }
 

tests/blueprints/conftest.py

@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+def pytest_collection_modifyitems(config, items):
+  for item in items:
+    item.add_marker(
+        pytest.mark.xdist_group(name='/'.join(item.path.parent.parts[-2:])))

tests/examples/conftest.py

@@ -15,6 +15,7 @@
 from pathlib import Path
 
 import marko
+import pytest
 
 FABRIC_ROOT = Path(__file__).parents[2]
 MODULES_PATH = FABRIC_ROOT / 'modules/'
@@ -36,13 +37,14 @@ def pytest_generate_tests(metafunc):
       doc = marko.parse(readme.read_text())
       index = 0
       last_header = None
+      mark = pytest.mark.xdist_group(name=module.name)
       for child in doc.children:
         if isinstance(child, marko.block.FencedCode) and child.lang == 'hcl':
           index += 1
           code = child.children[0].children
           if 'tftest skip' in code:
             continue
-          examples.append(code)
+          examples.append(pytest.param(code, marks=mark))
           path = module.relative_to(FABRIC_ROOT)
           name = f'{path}:{last_header}'
           if index > 1:

tests/fast/conftest.py

@@ -0,0 +1,20 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+def pytest_collection_modifyitems(config, items):
+  for item in items:
+    item.add_marker(pytest.mark.xdist_group(name=item.path.parent.name))

tests/modules/api_gateway/test_plan.py

@@ -12,15 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pytest
 
-
-@pytest.fixture
-def resources(plan_runner):
-  _, resources = plan_runner()
-  return resources
-
-
-def test_resource_count(resources):
+def test_resource_count(plan_runner):
   "Test number of resources created."
+  _, resources = plan_runner()
   assert len(resources) == 5

tests/modules/conftest.py

@@ -0,0 +1,20 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+def pytest_collection_modifyitems(config, items):
+  for item in items:
+    item.add_marker(pytest.mark.xdist_group(name=item.path.parent.name))

tests/requirements.txt

@@ -1,4 +1,5 @@
 pytest>=6.2.5
+pytest-xdist
 PyYAML>=6.0
 tftest>=1.6.3
 marko>=1.2.0