New folder structure

.ci/.terraformrc

@@ -1 +0,0 @@
-plugin_cache_dir   = "/workspace/.terraform.d/plugin-cache"

.ci/cloudbuild.lint.yaml

@@ -1,61 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-steps:
-  - name: python:3-alpine
-    id: prepare
-    entrypoint: sh
-    args:
-      - -c
-      - |
-        python -m pip install --user --no-warn-script-location -r /workspace/tools/REQUIREMENTS.txt &&
-        wget https://releases.hashicorp.com/terraform/${_TERRAFORM_VERSION}/terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        unzip terraform_${_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/home/.local/bin &&
-        rm terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        chmod 755 /builder/home/.local/bin/terraform &&
-        mkdir -p /workspace/.terraform.d/plugin-cache
-  - name: python:3-alpine
-    id: boilerplate
-    args: ["/workspace/tools/check_boilerplate.py", "/workspace"]
-  - name: python:3-alpine
-    id: terraform-fmt-check
-    entrypoint: sh
-    args:
-      - -c
-      - |
-        terraform fmt -recursive -check /workspace/
-    env:
-      - PATH=/usr/local/bin:/usr/bin:/bin:/builder/home/.local/bin
-      - TF_CLI_CONFIG_FILE=/workspace/.ci/.terraformrc
-  - name: python:3-alpine
-    id: documentation
-    args:
-      [
-        "/workspace/tools/check_documentation.py",
-        "cloud-operations",
-        "data-solutions",
-        "data-solutions/data-platform-foundations",
-        "factories",
-        "factories/firewall-vpc-rules",
-        "foundations",
-        "modules",
-        "networking",
-      ]
-
-substitutions:
-  _TERRAFORM_VERSION: 1.0.4
-
-tags:
-  - ci
-  - lint

.ci/cloudbuild.test.environments.yaml

@@ -1,47 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-steps:
-  - name: python:3-alpine
-    id: prepare
-    entrypoint: sh
-    args:
-      - -c
-      - |
-        python -m pip install --user --no-warn-script-location -r tests/requirements.txt &&
-        wget https://releases.hashicorp.com/terraform/${_TERRAFORM_VERSION}/terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        unzip terraform_${_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/home/.local/bin &&
-        rm terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        chmod 755 /builder/home/.local/bin/terraform &&
-        mkdir -p /workspace/.terraform.d/plugin-cache
-  - name: python:3-alpine
-    id: test-environments
-    entrypoint: pytest
-    args:
-      - -vv
-      - tests/cloud_operations
-      - tests/data_solutions
-      - tests/factories
-      - tests/foundations
-      - tests/networking
-    env:
-      - PATH=/usr/local/bin:/usr/bin:/bin:/builder/home/.local/bin
-      - TF_CLI_CONFIG_FILE=/workspace/.ci/.terraformrc
-
-substitutions:
-  _TERRAFORM_VERSION: 1.0.4
-
-tags:
-  - "ci"
-  - "test"

.ci/cloudbuild.test.examples.yaml

@@ -1,47 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-steps:
-  - name: python:3-alpine
-    id: prepare
-    entrypoint: sh
-    args:
-      - -c
-      - |
-        python -m pip install --user --no-warn-script-location -r tests/requirements.txt &&
-        wget https://releases.hashicorp.com/terraform/${_TERRAFORM_VERSION}/terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        unzip terraform_${_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/home/.local/bin &&
-        rm terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        chmod 755 /builder/home/.local/bin/terraform &&
-        mkdir -p /workspace/.terraform.d/plugin-cache
-  # TODO(ludoo): add a step that detects change files and sets tests to run
-  - name: python:3-alpine
-    id: test-examples
-    entrypoint: pytest
-    args:
-      - -vv
-      - tests/examples
-    env:
-      - PATH=/usr/local/bin:/usr/bin:/bin:/builder/home/.local/bin
-      - TF_CLI_CONFIG_FILE=/workspace/.ci/.terraformrc
-
-options:
-  machineType: "N1_HIGHCPU_8"
-
-substitutions:
-  _TERRAFORM_VERSION: 1.0.4
-
-tags:
-  - "ci"
-  - "test"

.ci/cloudbuild.test.modules.yaml

@@ -1,46 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-steps:
-  - name: python:3-alpine
-    id: prepare
-    entrypoint: sh
-    args:
-      - -c
-      - |
-        python -m pip install --user --no-warn-script-location -r tests/requirements.txt &&
-        wget https://releases.hashicorp.com/terraform/${_TERRAFORM_VERSION}/terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        unzip terraform_${_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/home/.local/bin &&
-        rm terraform_${_TERRAFORM_VERSION}_linux_amd64.zip &&
-        chmod 755 /builder/home/.local/bin/terraform &&
-        mkdir -p /workspace/.terraform.d/plugin-cache
-  - name: python:3-alpine
-    id: test-modules
-    entrypoint: pytest
-    args:
-      - -vv
-      - tests/modules
-    env:
-      - PATH=/usr/local/bin:/usr/bin:/bin:/builder/home/.local/bin
-      - TF_CLI_CONFIG_FILE=/workspace/.ci/.terraformrc
-
-options:
-  machineType: "N1_HIGHCPU_8"
-
-substitutions:
-  _TERRAFORM_VERSION: 1.0.4
-
-tags:
-  - "ci"
-  - "test"

.github/workflows/tests.yml

@@ -54,7 +54,7 @@ jobs:
       - name: Run tests environments
         id: test-environments
         run: |
-          pytest -vv tests/cloud_operations tests/data_solutions tests/factories tests/foundations tests/networking
+          pytest -n 4 --dist loadfile -vv tests/examples
 
   tests-examples:
     runs-on: ubuntu-latest
@@ -83,7 +83,7 @@ jobs:
       - name: Run tests examples
         id: test-examples
         run: |
-          pytest -vv tests/examples
+          pytest -n 4 --dist loadfile -vv tests/modules/examples
 
   tests-modules:
     runs-on: ubuntu-latest
@@ -112,4 +112,4 @@ jobs:
       - name: Run tests modules
         id: test-modules
         run: |
-          pytest -vv tests/modules
+          pytest -n 4 --dist loadgroup -vv --ignore=tests/modules/examples tests/modules

examples/cloud-operations/asset-inventory-feed-remediation/main.tf

@@ -20,7 +20,7 @@ locals {
 }
 
 module "project" {
-  source         = "../../modules/project"
+  source         = "../../../modules/project"
   name           = var.project_id
   project_create = var.project_create
   services = [
@@ -48,7 +48,7 @@ module "project" {
 }
 
 module "vpc" {
-  source     = "../../modules/net-vpc"
+  source     = "../../../modules/net-vpc"
   project_id = module.project.project_id
   name       = var.name
   subnets = [{
@@ -60,7 +60,7 @@ module "vpc" {
 }
 
 module "pubsub" {
-  source        = "../../modules/pubsub"
+  source        = "../../../modules/pubsub"
   project_id    = module.project.project_id
   name          = var.name
   subscriptions = { "${var.name}-default" = null }
@@ -72,14 +72,14 @@ module "pubsub" {
 }
 
 module "service-account" {
-  source     = "../../modules/iam-service-account"
+  source     = "../../../modules/iam-service-account"
   project_id = module.project.project_id
   name       = "${var.name}-cf"
   # iam_project_roles = { (module.project.project_id) = [local.role_id] }
 }
 
 module "cf" {
-  source      = "../../modules/cloud-function"
+  source      = "../../../modules/cloud-function"
   project_id  = module.project.project_id
   name        = var.name
   bucket_name = "${var.name}-${random_pet.random.id}"
@@ -101,7 +101,7 @@ module "cf" {
 }
 
 module "simple-vm-example" {
-  source     = "../../modules/compute-vm"
+  source     = "../../../modules/compute-vm"
   project_id = module.project.project_id
   zone       = "${var.region}-b"
   name       = var.name

examples/cloud-operations/dns-fine-grained-iam/main.tf

@@ -22,7 +22,7 @@ locals {
 }
 
 module "project" {
-  source         = "../../modules/project"
+  source         = "../../../modules/project"
   name           = var.project_id
   project_create = var.project_create
   services = [
@@ -37,7 +37,7 @@ module "project" {
 }
 
 module "vpc" {
-  source     = "../../modules/net-vpc"
+  source     = "../../../modules/net-vpc"
   project_id = module.project.project_id
   name       = var.name
   subnets = [{
@@ -49,13 +49,13 @@ module "vpc" {
 }
 
 module "firewall-a" {
-  source     = "../../modules/net-vpc-firewall"
+  source     = "../../../modules/net-vpc-firewall"
   project_id = module.project.project_id
   network    = module.vpc.name
 }
 
 module "nat-a" {
-  source         = "../../modules/net-cloudnat"
+  source         = "../../../modules/net-cloudnat"
   project_id     = module.project.project_id
   region         = var.region
   name           = var.name
@@ -63,7 +63,7 @@ module "nat-a" {
 }
 
 module "dns-service-zone" {
-  source                      = "../../modules/dns"
+  source                      = "../../../modules/dns"
   project_id                  = module.project.project_id
   type                        = "service-directory"
   name                        = var.name
@@ -73,7 +73,7 @@ module "dns-service-zone" {
 }
 
 module "service-directory" {
-  source     = "../../modules/service-directory"
+  source     = "../../../modules/service-directory"
   project_id = module.project.project_id
   location   = var.region
   name       = var.name
@@ -102,7 +102,7 @@ module "service-directory" {
 }
 
 module "vm-ns-editor" {
-  source     = "../../modules/compute-vm"
+  source     = "../../../modules/compute-vm"
   project_id = module.project.project_id
   zone       = "${var.region}-b"
   name       = "${var.name}-ns"
@@ -118,7 +118,7 @@ module "vm-ns-editor" {
 }
 
 module "vm-svc-editor" {
-  source     = "../../modules/compute-vm"
+  source     = "../../../modules/compute-vm"
   project_id = module.project.project_id
   zone       = "${var.region}-b"
   name       = "${var.name}-svc"

examples/cloud-operations/dns-shared-vpc/main.tf

@@ -22,7 +22,7 @@ locals {
 }
 
 module "project" {
-  source          = "../../modules/project"
+  source          = "../../../modules/project"
   for_each        = toset(var.teams)
   billing_account = var.billing_account_id
   name            = each.value
@@ -36,14 +36,14 @@ module "project" {
 }
 
 module "vpc" {
-  source     = "../../modules/net-vpc"
+  source     = "../../../modules/net-vpc"
   for_each   = local.projects
   project_id = each.value
   name       = "dns-vpc"
 }
 
 module "dns-private" {
-  source          = "../../modules/dns"
+  source          = "../../../modules/dns"
   for_each        = local.projects
   project_id      = each.value
   type            = "private"
@@ -54,7 +54,7 @@ module "dns-private" {
 }
 
 module "dns-peering" {
-  source          = "../../modules/dns"
+  source          = "../../../modules/dns"
   for_each        = local.projects
   project_id      = local.svpc_project_id
   name            = "peering-${each.key}"

examples/cloud-operations/iam-delegated-role-grants/main.tf

@@ -48,7 +48,7 @@ locals {
 }
 
 module "project" {
-  source         = "../../modules/project"
+  source         = "../../../modules/project"
   name           = var.project_id
   project_create = var.project_create
 }

examples/cloud-operations/onprem-sa-key-management/main.tf

@@ -19,14 +19,14 @@ locals {
 }
 
 module "project" {
-  source         = "../../modules/project"
+  source         = "../../../modules/project"
   name           = var.project_id
   project_create = var.project_create
   services       = var.services
 }
 
 module "integration-sa" {
-  source     = "../../modules/iam-service-account"
+  source     = "../../../modules/iam-service-account"
   for_each   = local.service_accounts
   project_id = module.project.project_id
   name       = each.value.name

examples/cloud-operations/packer-image-builder/main.tf

@@ -22,7 +22,7 @@ locals {
 }
 
 module "project" {
-  source          = "../../modules/project"
+  source          = "../../../modules/project"
   name            = var.project_id
   parent          = var.root_node
   billing_account = var.billing_account
@@ -37,7 +37,7 @@ module "project" {
 }
 
 module "service-account-image-builder" {
-  source     = "../../modules/iam-service-account"
+  source     = "../../../modules/iam-service-account"
   project_id = module.project.project_id
   name       = "image-builder"
   iam_project_roles = {
@@ -49,13 +49,13 @@ module "service-account-image-builder" {
 }
 
 module "service-account-image-builder-vm" {
-  source     = "../../modules/iam-service-account"
+  source     = "../../../modules/iam-service-account"
   project_id = module.project.project_id
   name       = "image-builder-vm"
 }
 
 module "vpc" {
-  source     = "../../modules/net-vpc"
+  source     = "../../../modules/net-vpc"
   project_id = module.project.project_id
   name       = "image-builder"
   subnets = [
@@ -69,7 +69,7 @@ module "vpc" {
 }
 
 module "firewall" {
-  source     = "../../modules/net-vpc-firewall"
+  source     = "../../../modules/net-vpc-firewall"
   project_id = module.project.project_id
   network    = module.vpc.name
   custom_rules = {
@@ -88,7 +88,7 @@ module "firewall" {
 }
 
 module "nat" {
-  source                = "../../modules/net-cloudnat"
+  source                = "../../../modules/net-cloudnat"
   project_id            = module.project.project_id
   region                = var.region
   name                  = "default"

examples/cloud-operations/quota-monitoring/main.tf

@@ -23,7 +23,7 @@ locals {
 }
 
 module "project" {
-  source         = "../../modules/project"
+  source         = "../../../modules/project"
   name           = var.project_id
   project_create = var.project_create
   services = [
@@ -40,7 +40,7 @@ module "project" {
 }
 
 module "pubsub" {
-  source     = "../../modules/pubsub"
+  source     = "../../../modules/pubsub"
   project_id = module.project.project_id
   name       = var.name
   subscriptions = {
@@ -51,7 +51,7 @@ module "pubsub" {
 }
 
 module "cf" {
-  source      = "../../modules/cloud-function"
+  source      = "../../../modules/cloud-function"
   project_id  = module.project.project_id
   name        = var.name
   bucket_name = "${var.name}-${random_pet.random.id}"

examples/cloud-operations/scheduled-asset-inventory-export-bq/main.tf

@@ -21,7 +21,7 @@
 ###############################################################################
 
 module "project" {
-  source          = "../../modules/project"
+  source          = "../../../modules/project"
   name            = var.project_id
   parent          = var.root_node
   billing_account = try(var.billing_account, null)
@@ -43,7 +43,7 @@ module "project" {
 }
 
 module "service-account" {
-  source     = "../../modules/iam-service-account"
+  source     = "../../../modules/iam-service-account"
   project_id = module.project.project_id
   name       = "${var.name}-cf"
   iam_project_roles = {
@@ -59,7 +59,7 @@ module "service-account" {
 ###############################################################################
 
 module "pubsub" {
-  source     = "../../modules/pubsub"
+  source     = "../../../modules/pubsub"
   project_id = module.project.project_id
   name       = var.name
   subscriptions = {
@@ -70,7 +70,7 @@ module "pubsub" {
 }
 
 module "pubsub_file" {
-  source     = "../../modules/pubsub"
+  source     = "../../../modules/pubsub"
   project_id = module.project.project_id
   name       = var.name_cffile
   subscriptions = {
@@ -85,7 +85,7 @@ module "pubsub_file" {
 ###############################################################################
 
 module "cf" {
-  source      = "../../modules/cloud-function"
+  source      = "../../../modules/cloud-function"
   project_id  = module.project.project_id
   region      = var.region
   name        = var.name
@@ -109,7 +109,7 @@ module "cf" {
 
 module "cffile" {
   count       = var.cai_gcs_export ? 1 : 0
-  source      = "../../modules/cloud-function"
+  source      = "../../../modules/cloud-function"
   project_id  = module.project.project_id
   region      = var.region
   name        = var.name_cffile
@@ -193,7 +193,7 @@ resource "google_cloud_scheduler_job" "job_file" {
 ###############################################################################
 
 module "bq" {
-  source     = "../../modules/bigquery-dataset"
+  source     = "../../../modules/bigquery-dataset"
   project_id = module.project.project_id
   id         = var.cai_config.bq_dataset
   location   = var.region

examples/data-solutions/cmek-via-centralized-kms/main.tf

@@ -17,7 +17,7 @@
 ###############################################################################
 
 module "project-service" {
-  source          = "../../modules/project"
+  source          = "../../../modules/project"
   name            = var.project_service_name
   parent          = var.root_node
   billing_account = var.billing_account
@@ -30,7 +30,7 @@ module "project-service" {
 }
 
 module "project-kms" {
-  source          = "../../modules/project"
+  source          = "../../../modules/project"
   name            = var.project_kms_name
   parent          = var.root_node
   billing_account = var.billing_account
@@ -46,7 +46,7 @@ module "project-kms" {
 ###############################################################################
 
 module "vpc" {
-  source     = "../../modules/net-vpc"
+  source     = "../../../modules/net-vpc"
   project_id = module.project-service.project_id
   name       = var.vpc_name
   subnets = [
@@ -60,7 +60,7 @@ module "vpc" {
 }
 
 module "vpc-firewall" {
-  source       = "../../modules/net-vpc-firewall"
+  source       = "../../../modules/net-vpc-firewall"
   project_id   = module.project-service.project_id
   network      = module.vpc.name
   admin_ranges = [var.vpc_ip_cidr_range]
@@ -71,7 +71,7 @@ module "vpc-firewall" {
 ###############################################################################
 
 module "kms" {
-  source     = "../../modules/kms"
+  source     = "../../../modules/kms"
   project_id = module.project-kms.project_id
   keyring = {
     name     = "my-keyring",
@@ -97,7 +97,7 @@ module "kms" {
 ###############################################################################
 
 module "vm_example" {
-  source     = "../../modules/compute-vm"
+  source     = "../../../modules/compute-vm"
   project_id = module.project-service.project_id
   zone       = "${var.region}-b"
   name       = "kms-vm"
@@ -135,7 +135,7 @@ module "vm_example" {
 ###############################################################################
 
 module "kms-gcs" {
-  source         = "../../modules/gcs"
+  source         = "../../../modules/gcs"
   project_id     = module.project-service.project_id
   prefix         = "my-bucket-001"
   name           = "kms-gcs"

examples/data-solutions/data-platform-foundations/01-environment/main.tf

@@ -19,7 +19,7 @@
 ###############################################################################
 
 module "project-datamart" {
-  source          = "../../../modules/project"
+  source          = "../../../../modules/project"
   parent          = var.root_node
   billing_account = var.billing_account_id
   prefix          = var.prefix
@@ -45,7 +45,7 @@ module "project-datamart" {
 }
 
 module "project-dwh" {
-  source          = "../../../modules/project"
+  source          = "../../../../modules/project"
   parent          = var.root_node
   billing_account = var.billing_account_id
   prefix          = var.prefix
@@ -70,7 +70,7 @@ module "project-dwh" {
 }
 
 module "project-landing" {
-  source          = "../../../modules/project"
+  source          = "../../../../modules/project"
   parent          = var.root_node
   billing_account = var.billing_account_id
   prefix          = var.prefix
@@ -93,7 +93,7 @@ module "project-landing" {
 }
 
 module "project-services" {
-  source          = "../../../modules/project"
+  source          = "../../../../modules/project"
   parent          = var.root_node
   billing_account = var.billing_account_id
   prefix          = var.prefix
@@ -122,7 +122,7 @@ module "project-services" {
 }
 
 module "project-transformation" {
-  source          = "../../../modules/project"
+  source          = "../../../../modules/project"
   parent          = var.root_node
   billing_account = var.billing_account_id
   prefix          = var.prefix
@@ -154,7 +154,7 @@ module "project-transformation" {
 ###############################################################################
 
 module "sa-services-main" {
-  source     = "../../../modules/iam-service-account"
+  source     = "../../../../modules/iam-service-account"
   project_id = module.project-services.project_id
   name       = var.service_account_names.main
   iam        = var.admins != null ? { "roles/iam.serviceAccountTokenCreator" = var.admins } : {}