From 3d41d01efcb9da077b287977ee69fe904022ab05 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 24 Mar 2023 13:28:32 +0100 Subject: [PATCH] FAST plugin system (#1266) * plugin folder, gitignore, serverless connector example * add support to fast plugin variables and outputs to tfdoc * rename folder, READMEs * add variable description * show diffs * check documentation, use multiple files * debug check doc * try a different glob * debug tfdoc names * more debug * and even more debug * fix gitignore * fix links * support extra files in tests * fix fixture, switch stage 2 peering to new tests * tfdoc * Allow globs in extra files --------- Co-authored-by: Julio Castillo --- .github/workflows/linting.yml | 2 +- blueprints/serverless/api-gateway/README.md | 1 - fast/.gitignore | 2 + .../README.md | 40 +++++++++ .../local-serverless-connector-outputs.tf | 46 ++++++++++ .../local-serverless-connector-variables.tf | 52 ++++++++++++ .../local-serverless-connector.tf | 84 +++++++++++++++++++ fast/plugins/README.md | 25 ++++++ .../2-networking-a-peering/spoke-dev.tf | 1 + .../2-networking-a-peering/spoke-prod.tf | 1 + fast/stages/2-networking-b-vpn/spoke-dev.tf | 1 + fast/stages/2-networking-b-vpn/spoke-prod.tf | 1 + fast/stages/2-networking-c-nva/spoke-dev.tf | 1 + fast/stages/2-networking-c-nva/spoke-prod.tf | 1 + .../2-networking-d-separate-envs/spoke-dev.tf | 1 + .../spoke-prod.tf | 1 + tests/collectors.py | 18 ++-- .../s2_networking_a_peering/common.tfvars | 1 - .../s2_networking_a_peering/stage.tfvars | 0 .../stage.yaml} | 4 + .../s2_networking_a_peering/test_plan.py | 17 +--- .../tftest.yaml} | 15 ++-- .../stages/s2_networking_b_vpn/common.tfvars | 1 - .../s2_networking_b_vpn/fixture/main.tf | 54 ------------ .../stages/s2_networking_b_vpn/stage.tfvars | 0 .../stage.yaml} | 6 +- .../stages/s2_networking_b_vpn/tftest.yaml | 22 +++++ .../stages/s2_networking_c_nva/common.tfvars | 1 - .../stages/s2_networking_c_nva/stage.tfvars | 0 .../{__init__.py => stage.yaml} | 6 +- .../stages/s2_networking_c_nva/test_plan.py | 21 ----- .../stages/s2_networking_c_nva/tftest.yaml | 22 +++++ .../common.tfvars | 1 - .../stage.tfvars | 0 .../s2_networking_d_separate_envs/stage.yaml | 17 ++++ .../test_plan.py | 21 ----- .../s2_networking_d_separate_envs/tftest.yaml | 22 +++++ tests/fixtures.py | 24 ++++-- tools/check_documentation.py | 5 +- tools/tfdoc.py | 8 +- 40 files changed, 401 insertions(+), 145 deletions(-) create mode 100644 fast/.gitignore create mode 100644 fast/plugins/2-networking-serverless-connector/README.md create mode 100644 fast/plugins/2-networking-serverless-connector/local-serverless-connector-outputs.tf create mode 100644 fast/plugins/2-networking-serverless-connector/local-serverless-connector-variables.tf create mode 100644 fast/plugins/2-networking-serverless-connector/local-serverless-connector.tf create mode 100644 fast/plugins/README.md create mode 100644 tests/fast/stages/s2_networking_a_peering/stage.tfvars rename tests/fast/stages/{s2_networking_b_vpn/__init__.py => s2_networking_a_peering/stage.yaml} (93%) rename tests/fast/stages/{s2_networking_b_vpn/test_plan.py => s2_networking_a_peering/tftest.yaml} (66%) delete mode 100644 tests/fast/stages/s2_networking_b_vpn/fixture/main.tf create mode 100644 tests/fast/stages/s2_networking_b_vpn/stage.tfvars rename tests/fast/stages/{s2_networking_d_separate_envs/__init__.py => s2_networking_b_vpn/stage.yaml} (88%) create mode 100644 tests/fast/stages/s2_networking_b_vpn/tftest.yaml create mode 100644 tests/fast/stages/s2_networking_c_nva/stage.tfvars rename tests/fast/stages/s2_networking_c_nva/{__init__.py => stage.yaml} (88%) delete mode 100644 tests/fast/stages/s2_networking_c_nva/test_plan.py create mode 100644 tests/fast/stages/s2_networking_c_nva/tftest.yaml create mode 100644 tests/fast/stages/s2_networking_d_separate_envs/stage.tfvars create mode 100644 tests/fast/stages/s2_networking_d_separate_envs/stage.yaml delete mode 100644 tests/fast/stages/s2_networking_d_separate_envs/test_plan.py create mode 100644 tests/fast/stages/s2_networking_d_separate_envs/tftest.yaml diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index b8e21a37..4d643dac 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -56,7 +56,7 @@ jobs: - name: Check documentation id: documentation-fabric run: | - python3 tools/check_documentation.py modules fast blueprints + python3 tools/check_documentation.py --show-diffs modules fast blueprints - name: Check documentation links id: documentation-links-fabric diff --git a/blueprints/serverless/api-gateway/README.md b/blueprints/serverless/api-gateway/README.md index 42ae2460..216620ed 100644 --- a/blueprints/serverless/api-gateway/README.md +++ b/blueprints/serverless/api-gateway/README.md @@ -23,7 +23,6 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c Once done testing, you can clean up resources by running `terraform destroy`. - ## Variables diff --git a/fast/.gitignore b/fast/.gitignore new file mode 100644 index 00000000..5f253cb5 --- /dev/null +++ b/fast/.gitignore @@ -0,0 +1,2 @@ +stages/[0-9]*/local-*.tf +stages-multitenant/[0-9]*/local-*.tf diff --git a/fast/plugins/2-networking-serverless-connector/README.md b/fast/plugins/2-networking-serverless-connector/README.md new file mode 100644 index 00000000..f8ba8b2f --- /dev/null +++ b/fast/plugins/2-networking-serverless-connector/README.md @@ -0,0 +1,40 @@ +# VPC Serverless Connector + +This FAST plugin adds centralized [Serverless VPC Access Connectors](https://cloud.google.com/vpc/docs/serverless-vpc-access) to network stages. + +This plugin does not manage + +- IAM bindings for the connectors, which should be added via the stage project-level variables +- firewall rules for the connectors, which should be added via the stage factory + +The plugin only requires a specific configuration if the defaults it uses need to be changed: + +- the connector-specific subnets default to the `10.255.255.0` range +- the machine type, number of instances and thoughput use the API defaults + +To enable the plugin, simply copy or link its files in the networking stage. + + + + +## Files + +| name | description | modules | resources | +|---|---|---|---| +| [local-serverless-connector-outputs.tf](./local-serverless-connector-outputs.tf) | Serverless Connector outputs. | | google_storage_bucket_object ยท local_file | +| [local-serverless-connector-variables.tf](./local-serverless-connector-variables.tf) | Serverless Connector variables. | | | +| [local-serverless-connector.tf](./local-serverless-connector.tf) | Serverless Connector resources. | net-vpc | google_vpc_access_connector | + +## Variables + +| name | description | type | required | default | producer | +|---|---|:---:|:---:|:---:|:---:| +| [serverless_connector_config](local-serverless-connector-variables.tf#L19) | VPC Access Serverless Connectors configuration. | object({…}) | | {…} | | + +## Outputs + +| name | description | sensitive | consumers | +|---|---|:---:|---| +| [plugin_sc_connectors](local-serverless-connector-outputs.tf#L43) | VPC Access Connectors. | | | + + diff --git a/fast/plugins/2-networking-serverless-connector/local-serverless-connector-outputs.tf b/fast/plugins/2-networking-serverless-connector/local-serverless-connector-outputs.tf new file mode 100644 index 00000000..0e1d7b00 --- /dev/null +++ b/fast/plugins/2-networking-serverless-connector/local-serverless-connector-outputs.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Serverless Connector outputs. + +locals { + plugin_sc_tfvars = { + dev = google_vpc_access_connector.dev-primary.0.id + prod = google_vpc_access_connector.prod-primary.0.id + } +} + +# generate tfvars file for subsequent stages + +resource "local_file" "plugin_sc_tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${try(pathexpand(var.outputs_location), "")}/tfvars/2-networking-serverless-connnector.auto.tfvars.json" + content = jsonencode(local.plugin_sc_tfvars) +} + +resource "google_storage_bucket_object" "plugin_sc_tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/2-networking-serverless-connnector.auto.tfvars.json" + content = jsonencode(local.plugin_sc_tfvars) +} + +# outputs + +output "plugin_sc_connectors" { + description = "VPC Access Connectors." + value = local.plugin_sc_tfvars +} diff --git a/fast/plugins/2-networking-serverless-connector/local-serverless-connector-variables.tf b/fast/plugins/2-networking-serverless-connector/local-serverless-connector-variables.tf new file mode 100644 index 00000000..05733c46 --- /dev/null +++ b/fast/plugins/2-networking-serverless-connector/local-serverless-connector-variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Serverless Connector variables. + +variable "serverless_connector_config" { + description = "VPC Access Serverless Connectors configuration." + type = object({ + dev-primary = object({ + ip_cidr_range = optional(string, "10.255.255.128/28") + machine_type = optional(string) + instances = optional(object({ + max = optional(number) + min = optional(number) + }), {}) + throughput = optional(object({ + max = optional(number) + min = optional(number) + }), {}) + }) + prod-primary = object({ + ip_cidr_range = optional(string, "10.255.255.0/28") + machine_type = optional(string) + instances = optional(object({ + max = optional(number) + min = optional(number) + }), {}) + throughput = optional(object({ + max = optional(number) + min = optional(number) + }), {}) + }) + }) + default = { + dev-primary = {} + prod-primary = {} + } + nullable = false +} diff --git a/fast/plugins/2-networking-serverless-connector/local-serverless-connector.tf b/fast/plugins/2-networking-serverless-connector/local-serverless-connector.tf new file mode 100644 index 00000000..d08a28ad --- /dev/null +++ b/fast/plugins/2-networking-serverless-connector/local-serverless-connector.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Serverless Connector resources. + +locals { + plugin_sc_subnets = { + dev = { + for k, v in module.dev-spoke-vpc-serverless.subnets : k => v.name + } + prod = { + for k, v in module.prod-spoke-vpc-serverless.subnets : k => v.name + } + } +} + +module "dev-spoke-vpc-serverless" { + source = "../../../modules/net-vpc" + project_id = module.dev-spoke-project.project_id + name = module.dev-spoke-vpc.name + vpc_create = false + subnets = [{ + name = "access-connector" + description = "VPC Serverless Connector for the primary region." + ip_cidr_range = var.serverless_connector_config.dev-primary.ip_cidr_range + region = var.regions.primary + }] +} + +module "prod-spoke-vpc-serverless" { + source = "../../../modules/net-vpc" + project_id = module.prod-spoke-project.project_id + name = module.prod-spoke-vpc.name + vpc_create = false + subnets = [{ + name = "access-connector" + description = "VPC Serverless Connector for the primary region." + ip_cidr_range = var.serverless_connector_config.prod-primary.ip_cidr_range + region = var.regions.primary + }] +} + +resource "google_vpc_access_connector" "dev-primary" { + count = var.serverless_connector_config.dev-primary == null ? 0 : 1 + project = module.dev-spoke-project.project_id + region = var.regions.primary + name = "access-connector-${local.region_shortnames[var.regions.primary]}" + subnet { + name = local.plugin_sc_subnets.dev["${var.regions.primary}/access-connector"] + } + machine_type = var.serverless_connector_config.dev-primary.machine_type + max_instances = var.serverless_connector_config.dev-primary.instances.max + max_throughput = var.serverless_connector_config.dev-primary.throughput.max + min_instances = var.serverless_connector_config.dev-primary.instances.min + min_throughput = var.serverless_connector_config.dev-primary.throughput.min +} + +resource "google_vpc_access_connector" "prod-primary" { + count = var.serverless_connector_config.prod-primary == null ? 0 : 1 + project = module.prod-spoke-project.project_id + region = var.regions.primary + name = "access-connector-${local.region_shortnames[var.regions.primary]}" + subnet { + name = local.plugin_sc_subnets.prod["${var.regions.primary}/access-connector"] + } + machine_type = var.serverless_connector_config.prod-primary.machine_type + max_instances = var.serverless_connector_config.prod-primary.instances.max + max_throughput = var.serverless_connector_config.prod-primary.throughput.max + min_instances = var.serverless_connector_config.prod-primary.instances.min + min_throughput = var.serverless_connector_config.prod-primary.throughput.min +} diff --git a/fast/plugins/README.md b/fast/plugins/README.md new file mode 100644 index 00000000..d4e7ce78 --- /dev/null +++ b/fast/plugins/README.md @@ -0,0 +1,25 @@ +# FAST plugin system + +This folders details a simple mechanism that can be used to add extra functionality to FAST stages, and a few examples that implement simple plugins that can be used as-is. + +## Available plugins + +### Networking + +- [Serverless VPC Access Connector](./2-networking-serverless-connector/) + +## Anatomy of a plugin + +FAST plugins are much simpler and easier to code than full-blown stages: each plugin is meant to add a single feature using a small set of resources, and interacting directly with stage modules and variables. + +A simple plugin might be composed of a single file with one resource, and grow up to the canonical set of one "main" (resources), one variables, and outputs file. + +Plugin file names start with the `local-` prefix which is purposefully excluded in FAST stages via Git ignore, so that plugins are not accidentally committed to stages during development and staying aligned with our master branch is possible. + +Plugins are structured here as individual folders, organized in top-level folders according to the FAST stage they are designed to work with. + +As an example, the [`2-networking/serverless-connector` plugin](./2-networking-serverless-connector/) implements centralized [Serverless VPC Access Connectors](https://cloud.google.com/vpc/docs/serverless-vpc-access) for our networking stages, and is composed of three files: + +- [`local-serverless-connector.tf`](./2-networking-serverless-connector/local-serverless-connector.tf) managing resources including the subnets needed in each VPC and the connectors themselves +- [`local-serverless-connector-outputs.tf`](./2-networking-serverless-connector/local-serverless-connector-outputs.tf) defining a single `serverless_connectors` output for the plugin, and optional output files +- [`local-serverless-connector-variables.tf`](./2-networking-serverless-connector/local-serverless-connector-variables.tf) defining a single `serverless_connector_config` variable used to configure the plugin diff --git a/fast/stages/2-networking-a-peering/spoke-dev.tf b/fast/stages/2-networking-a-peering/spoke-dev.tf index 15a8e477..b12d260d 100644 --- a/fast/stages/2-networking-a-peering/spoke-dev.tf +++ b/fast/stages/2-networking-a-peering/spoke-dev.tf @@ -30,6 +30,7 @@ module "dev-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-a-peering/spoke-prod.tf b/fast/stages/2-networking-a-peering/spoke-prod.tf index 927c74af..5130d240 100644 --- a/fast/stages/2-networking-a-peering/spoke-prod.tf +++ b/fast/stages/2-networking-a-peering/spoke-prod.tf @@ -30,6 +30,7 @@ module "prod-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-b-vpn/spoke-dev.tf b/fast/stages/2-networking-b-vpn/spoke-dev.tf index 15a8e477..b12d260d 100644 --- a/fast/stages/2-networking-b-vpn/spoke-dev.tf +++ b/fast/stages/2-networking-b-vpn/spoke-dev.tf @@ -30,6 +30,7 @@ module "dev-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-b-vpn/spoke-prod.tf b/fast/stages/2-networking-b-vpn/spoke-prod.tf index 927c74af..5130d240 100644 --- a/fast/stages/2-networking-b-vpn/spoke-prod.tf +++ b/fast/stages/2-networking-b-vpn/spoke-prod.tf @@ -30,6 +30,7 @@ module "prod-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-c-nva/spoke-dev.tf b/fast/stages/2-networking-c-nva/spoke-dev.tf index cc2c1a4f..8e26a732 100644 --- a/fast/stages/2-networking-c-nva/spoke-dev.tf +++ b/fast/stages/2-networking-c-nva/spoke-dev.tf @@ -29,6 +29,7 @@ module "dev-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-c-nva/spoke-prod.tf b/fast/stages/2-networking-c-nva/spoke-prod.tf index f1b79a12..1b2c4e2b 100644 --- a/fast/stages/2-networking-c-nva/spoke-prod.tf +++ b/fast/stages/2-networking-c-nva/spoke-prod.tf @@ -29,6 +29,7 @@ module "prod-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-d-separate-envs/spoke-dev.tf b/fast/stages/2-networking-d-separate-envs/spoke-dev.tf index f51a4e2b..7cad8479 100644 --- a/fast/stages/2-networking-d-separate-envs/spoke-dev.tf +++ b/fast/stages/2-networking-d-separate-envs/spoke-dev.tf @@ -30,6 +30,7 @@ module "dev-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/fast/stages/2-networking-d-separate-envs/spoke-prod.tf b/fast/stages/2-networking-d-separate-envs/spoke-prod.tf index 78944526..e529d5f8 100644 --- a/fast/stages/2-networking-d-separate-envs/spoke-prod.tf +++ b/fast/stages/2-networking-d-separate-envs/spoke-prod.tf @@ -30,6 +30,7 @@ module "prod-spoke-project" { "networkmanagement.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", + "vpcaccess.googleapis.com" ] shared_vpc_host_config = { enabled = true diff --git a/tests/collectors.py b/tests/collectors.py index e69782f0..3df7d3d0 100644 --- a/tests/collectors.py +++ b/tests/collectors.py @@ -37,6 +37,9 @@ class FabricTestFile(pytest.File): have the following structure: test-name: + extra_files: + - bar.tf + - foo.tf tfvars: - tfvars1.tfvars - tfvars2.tfvars @@ -60,27 +63,32 @@ class FabricTestFile(pytest.File): common = raw.pop('common_tfvars', []) for test_name, spec in raw.get('tests', {}).items(): spec = {} if spec is None else spec + extra_files = spec.get('extra_files') inventories = spec.get('inventory', [f'{test_name}.yaml']) - tfvars = common + [f'{test_name}.tfvars'] + spec.get('tfvars', []) + tf_var_files = common + [f'{test_name}.tfvars'] + spec.get('tfvars', []) for i in inventories: name = test_name if isinstance(inventories, list) and len(inventories) > 1: name = f'{test_name}[{i}]' yield FabricTestItem.from_parent(self, name=name, module=module, - inventory=[i], tfvars=tfvars) + inventory=[i], + tf_var_files=tf_var_files, + extra_files=extra_files) class FabricTestItem(pytest.Item): - def __init__(self, name, parent, module, inventory, tfvars): + def __init__(self, name, parent, module, inventory, tf_var_files, + extra_files=None): super().__init__(name, parent) self.module = module self.inventory = inventory - self.tfvars = tfvars + self.tf_var_files = tf_var_files + self.extra_files = extra_files def runtest(self): s = plan_validator(self.module, self.inventory, self.parent.path.parent, - self.tfvars) + self.tf_var_files, self.extra_files) def reportinfo(self): return self.path, None, self.name diff --git a/tests/fast/stages/s2_networking_a_peering/common.tfvars b/tests/fast/stages/s2_networking_a_peering/common.tfvars index e7187df0..94abd3c1 100644 --- a/tests/fast/stages/s2_networking_a_peering/common.tfvars +++ b/tests/fast/stages/s2_networking_a_peering/common.tfvars @@ -1,4 +1,3 @@ -data_dir = "../../../fast/stages/2-networking-a-peering/data/" automation = { outputs_bucket = "test" } diff --git a/tests/fast/stages/s2_networking_a_peering/stage.tfvars b/tests/fast/stages/s2_networking_a_peering/stage.tfvars new file mode 100644 index 00000000..e69de29b diff --git a/tests/fast/stages/s2_networking_b_vpn/__init__.py b/tests/fast/stages/s2_networking_a_peering/stage.yaml similarity index 93% rename from tests/fast/stages/s2_networking_b_vpn/__init__.py rename to tests/fast/stages/s2_networking_a_peering/stage.yaml index 6d6d1266..556d8f14 100644 --- a/tests/fast/stages/s2_networking_b_vpn/__init__.py +++ b/tests/fast/stages/s2_networking_a_peering/stage.yaml @@ -11,3 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +counts: + modules: 31 + resources: 122 \ No newline at end of file diff --git a/tests/fast/stages/s2_networking_a_peering/test_plan.py b/tests/fast/stages/s2_networking_a_peering/test_plan.py index 09590d36..35f3b7b3 100644 --- a/tests/fast/stages/s2_networking_a_peering/test_plan.py +++ b/tests/fast/stages/s2_networking_a_peering/test_plan.py @@ -14,27 +14,14 @@ import hashlib -from pathlib import Path - from deepdiff import DeepDiff - -BASEDIR = Path(__file__).parent -FIXTURE_PEERING = BASEDIR / 'fixture' -FIXTURE_VPN = BASEDIR.parent / 's2_networking_b_vpn/fixture' +from pathlib import Path STAGES = Path(__file__).parents[4] / 'fast/stages' STAGE_PEERING = STAGES / '2-networking-a-peering' STAGE_VPN = STAGES / '2-networking-b-vpn' -def test_counts(plan_summary): - "Test stage." - summary = plan_summary("fast/stages/2-networking-a-peering", - tf_var_files=["common.tfvars"]) - assert summary.counts["modules"] > 0 - assert summary.counts["resources"] > 0 - - def test_vpn_peering_parity(plan_summary): '''Ensure VPN- and peering-based networking stages are identical except for VPN and VPC peering resources''' @@ -65,7 +52,7 @@ def compute_md5(filename): return md5hash.hexdigest() -def test_vpn_peering_checksums(e2e_plan_runner): +def test_vpn_peering_checksums(): '''Compare MD5 sums of common files in the vpn and peering networking stages''' peering_files = { diff --git a/tests/fast/stages/s2_networking_b_vpn/test_plan.py b/tests/fast/stages/s2_networking_a_peering/tftest.yaml similarity index 66% rename from tests/fast/stages/s2_networking_b_vpn/test_plan.py rename to tests/fast/stages/s2_networking_a_peering/tftest.yaml index 8ac1bade..a140e1ff 100644 --- a/tests/fast/stages/s2_networking_b_vpn/test_plan.py +++ b/tests/fast/stages/s2_networking_a_peering/tftest.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +module: fast/stages/2-networking-a-peering +common_tfvars: + - common.tfvars -def test_counts(plan_summary): - "Test stage." - summary = plan_summary("fast/stages/2-networking-b-vpn", - tf_var_files=["common.tfvars"]) - assert summary.counts["modules"] > 0 - assert summary.counts["resources"] > 0 +tests: + stage: + extra_files: + - ../../plugins/2-networking-serverless-connector/*.tf diff --git a/tests/fast/stages/s2_networking_b_vpn/common.tfvars b/tests/fast/stages/s2_networking_b_vpn/common.tfvars index c7d02e6c..51c92a06 100644 --- a/tests/fast/stages/s2_networking_b_vpn/common.tfvars +++ b/tests/fast/stages/s2_networking_b_vpn/common.tfvars @@ -1,4 +1,3 @@ -data_dir = "../../../../../fast/stages/2-networking-b-vpn/data/" automation = { outputs_bucket = "test" } diff --git a/tests/fast/stages/s2_networking_b_vpn/fixture/main.tf b/tests/fast/stages/s2_networking_b_vpn/fixture/main.tf deleted file mode 100644 index 2ddb024f..00000000 --- a/tests/fast/stages/s2_networking_b_vpn/fixture/main.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module "stage" { - source = "../../../../../fast/stages/02-networking-vpn" - data_dir = "../../../../../fast/stages/02-networking-vpn/data/" - automation = { - outputs_bucket = "test" - } - billing_account = { - id = "000000-111111-222222" - organization_id = 123456789012 - } - custom_roles = { - service_project_network_admin = "organizations/123456789012/roles/foo" - } - folder_ids = { - networking = null - networking-dev = null - networking-prod = null - } - region_trigram = { - europe-west1 = "ew1" - europe-west3 = "ew3" - europe-west8 = "ew8" - } - service_accounts = { - data-platform-dev = "string" - data-platform-prod = "string" - gke-dev = "string" - gke-prod = "string" - project-factory-dev = "string" - project-factory-prod = "string" - } - organization = { - domain = "fast.example.com" - id = 123456789012 - customer_id = "C00000000" - } - prefix = "fast2" -} diff --git a/tests/fast/stages/s2_networking_b_vpn/stage.tfvars b/tests/fast/stages/s2_networking_b_vpn/stage.tfvars new file mode 100644 index 00000000..e69de29b diff --git a/tests/fast/stages/s2_networking_d_separate_envs/__init__.py b/tests/fast/stages/s2_networking_b_vpn/stage.yaml similarity index 88% rename from tests/fast/stages/s2_networking_d_separate_envs/__init__.py rename to tests/fast/stages/s2_networking_b_vpn/stage.yaml index 6d6d1266..d63a3e4f 100644 --- a/tests/fast/stages/s2_networking_d_separate_envs/__init__.py +++ b/tests/fast/stages/s2_networking_b_vpn/stage.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +counts: + modules: 33 + resources: 159 diff --git a/tests/fast/stages/s2_networking_b_vpn/tftest.yaml b/tests/fast/stages/s2_networking_b_vpn/tftest.yaml new file mode 100644 index 00000000..56c7bbf2 --- /dev/null +++ b/tests/fast/stages/s2_networking_b_vpn/tftest.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module: fast/stages/2-networking-b-vpn +common_tfvars: + - common.tfvars + +tests: + stage: + extra_files: + - ../../plugins/2-networking-serverless-connector/*.tf diff --git a/tests/fast/stages/s2_networking_c_nva/common.tfvars b/tests/fast/stages/s2_networking_c_nva/common.tfvars index 9980e655..a4b1cdd8 100644 --- a/tests/fast/stages/s2_networking_c_nva/common.tfvars +++ b/tests/fast/stages/s2_networking_c_nva/common.tfvars @@ -1,4 +1,3 @@ -data_dir = "../../../fast/stages/2-networking-c-nva/data/" automation = { outputs_bucket = "test" } diff --git a/tests/fast/stages/s2_networking_c_nva/stage.tfvars b/tests/fast/stages/s2_networking_c_nva/stage.tfvars new file mode 100644 index 00000000..e69de29b diff --git a/tests/fast/stages/s2_networking_c_nva/__init__.py b/tests/fast/stages/s2_networking_c_nva/stage.yaml similarity index 88% rename from tests/fast/stages/s2_networking_c_nva/__init__.py rename to tests/fast/stages/s2_networking_c_nva/stage.yaml index 6d6d1266..f79d4015 100644 --- a/tests/fast/stages/s2_networking_c_nva/__init__.py +++ b/tests/fast/stages/s2_networking_c_nva/stage.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +counts: + modules: 45 + resources: 168 diff --git a/tests/fast/stages/s2_networking_c_nva/test_plan.py b/tests/fast/stages/s2_networking_c_nva/test_plan.py deleted file mode 100644 index 70c37bc3..00000000 --- a/tests/fast/stages/s2_networking_c_nva/test_plan.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def test_counts(plan_summary): - "Test stage." - summary = plan_summary("fast/stages/2-networking-c-nva", - tf_var_files=["common.tfvars"]) - assert summary.counts["modules"] > 0 - assert summary.counts["resources"] > 0 diff --git a/tests/fast/stages/s2_networking_c_nva/tftest.yaml b/tests/fast/stages/s2_networking_c_nva/tftest.yaml new file mode 100644 index 00000000..482694d9 --- /dev/null +++ b/tests/fast/stages/s2_networking_c_nva/tftest.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module: fast/stages/2-networking-c-nva +common_tfvars: + - common.tfvars + +tests: + stage: + extra_files: + - ../../plugins/2-networking-serverless-connector/*.tf diff --git a/tests/fast/stages/s2_networking_d_separate_envs/common.tfvars b/tests/fast/stages/s2_networking_d_separate_envs/common.tfvars index bc9f5ead..7fce32a5 100644 --- a/tests/fast/stages/s2_networking_d_separate_envs/common.tfvars +++ b/tests/fast/stages/s2_networking_d_separate_envs/common.tfvars @@ -1,4 +1,3 @@ -data_dir = "../../../../../fast/stages/2-networking-d-separate-envs/data/" automation = { outputs_bucket = "test" } diff --git a/tests/fast/stages/s2_networking_d_separate_envs/stage.tfvars b/tests/fast/stages/s2_networking_d_separate_envs/stage.tfvars new file mode 100644 index 00000000..e69de29b diff --git a/tests/fast/stages/s2_networking_d_separate_envs/stage.yaml b/tests/fast/stages/s2_networking_d_separate_envs/stage.yaml new file mode 100644 index 00000000..c6824980 --- /dev/null +++ b/tests/fast/stages/s2_networking_d_separate_envs/stage.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +counts: + modules: 28 + resources: 122 diff --git a/tests/fast/stages/s2_networking_d_separate_envs/test_plan.py b/tests/fast/stages/s2_networking_d_separate_envs/test_plan.py deleted file mode 100644 index 51a9257a..00000000 --- a/tests/fast/stages/s2_networking_d_separate_envs/test_plan.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def test_counts(plan_summary): - "Test stage." - summary = plan_summary("fast/stages/2-networking-d-separate-envs", - tf_var_files=["common.tfvars"]) - assert summary.counts["modules"] > 0 - assert summary.counts["resources"] > 0 diff --git a/tests/fast/stages/s2_networking_d_separate_envs/tftest.yaml b/tests/fast/stages/s2_networking_d_separate_envs/tftest.yaml new file mode 100644 index 00000000..d73a8693 --- /dev/null +++ b/tests/fast/stages/s2_networking_d_separate_envs/tftest.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module: fast/stages/2-networking-d-separate-envs +common_tfvars: + - common.tfvars + +tests: + stage: + extra_files: + - ../../plugins/2-networking-serverless-connector/*.tf diff --git a/tests/fixtures.py b/tests/fixtures.py index 11a397de..381e161e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ import collections import contextlib -import itertools +import glob import os import shutil import tempfile @@ -34,7 +34,7 @@ def _prepare_root_module(path): If the TFTEST_COPY environment variable is set, `path` is copied to a temporary directory and a few terraform files (e.g. - terraform.tfvars) are delete to ensure a clean test environment. + terraform.tfvars) are deleted to ensure a clean test environment. Otherwise, `path` is simply returned untouched. """ if os.environ.get('TFTEST_COPY'): @@ -62,7 +62,8 @@ def _prepare_root_module(path): yield path -def plan_summary(module_path, basedir, tf_var_files=None, **tf_vars): +def plan_summary(module_path, basedir, tf_var_files=None, extra_files=None, + **tf_vars): """ Run a Terraform plan on the module located at `module_path`. @@ -97,7 +98,10 @@ def plan_summary(module_path, basedir, tf_var_files=None, **tf_vars): with _prepare_root_module(module_path) as test_path: binary = os.environ.get('TERRAFORM', 'terraform') tf = tftest.TerraformTest(test_path, binary=binary) - tf.setup(upgrade=True) + extra_files = [(module_path / filename).resolve() + for x in extra_files or [] + for filename in glob.glob(x, root_dir=module_path)] + tf.setup(extra_files=extra_files, upgrade=True) tf_var_files = [(basedir / x).resolve() for x in tf_var_files or []] plan = tf.plan(output=True, tf_var_file=tf_var_files, tf_vars=tf_vars) @@ -135,19 +139,21 @@ def plan_summary_fixture(request): to the directory of the calling test """ - def inner(module_path, basedir=None, tf_var_files=None, **tf_vars): + def inner(module_path, basedir=None, tf_var_files=None, extra_files=None, + **tf_vars): if basedir is None: basedir = Path(request.fspath).parent return plan_summary(module_path=module_path, basedir=basedir, - tf_var_files=tf_var_files, **tf_vars) + tf_var_files=tf_var_files, extra_files=extra_files, + **tf_vars) return inner def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None, - **tf_vars): + extra_files=None, **tf_vars): summary = plan_summary(module_path=module_path, tf_var_files=tf_var_files, - basedir=basedir, **tf_vars) + extra_files=extra_files, basedir=basedir, **tf_vars) # allow single single string for inventory_paths if not isinstance(inventory_paths, list): diff --git a/tools/check_documentation.py b/tools/check_documentation.py index 47643493..c9d65313 100755 --- a/tools/check_documentation.py +++ b/tools/check_documentation.py @@ -78,11 +78,10 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False): try: new_doc = tfdoc.create_doc(readme_path.parent, files, show_extra, exclude_files, readme) - # TODO: support variables in multiple files newvars = new_doc.variables newouts = new_doc.outputs - variables = [v.name for v in newvars if v.file == "variables.tf"] - outputs = [o.name for o in newouts if o.file == "outputs.tf"] + variables = [v.name for v in newvars if v.file.endswith('variables.tf')] + outputs = [o.name for o in newouts if o.file.endswith('outputs.tf')] except SystemExit: state = state.SKIP else: diff --git a/tools/tfdoc.py b/tools/tfdoc.py index a69d94ab..662f7aeb 100755 --- a/tools/tfdoc.py +++ b/tools/tfdoc.py @@ -175,7 +175,9 @@ def parse_files(basepath, exclude_files=None): def parse_outputs(basepath, exclude_files=None): 'Return a list of Output named tuples for root module outputs*.tf.' exclude_files = exclude_files or [] - for name in glob.glob(os.path.join(basepath, 'outputs*tf')): + names = glob.glob(os.path.join(basepath, 'outputs*tf')) + names += glob.glob(os.path.join(basepath, 'local-*outputs*tf')) + for name in names: shortname = os.path.basename(name) if shortname in exclude_files: continue @@ -196,7 +198,9 @@ def parse_outputs(basepath, exclude_files=None): def parse_variables(basepath, exclude_files=None): 'Return a list of Variable named tuples for root module variables*.tf.' exclude_files = exclude_files or [] - for name in glob.glob(os.path.join(basepath, 'variables*tf')): + names = glob.glob(os.path.join(basepath, 'variables*tf')) + names += glob.glob(os.path.join(basepath, 'local-*variables*tf')) + for name in names: shortname = os.path.basename(name) if shortname in exclude_files: continue