From f8f18734f16e10c28f0995e67431c417c8329bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 30 May 2024 19:35:09 +0200 Subject: [PATCH] Add pre-commit hook configuration (#2326) * Pre-commit config Run following linters on commit: Terraform: - terraform fmt - terraform tflint Python specific: - yapf Shell scripts - shellcheck - shfmt YAML files: - yamllint (disabled as of now) - check-yaml Other: - end-of-file-fixer - trailing-whitespace fixer Fabric specific - tools/tfdoc.py - tools/check_boilerplate.py * linting fixes * Fix boilerplate check --- .pre-commit-config.yaml | 82 +++++++++++++++++++ .yamllint | 16 ++++ .../bigquery-analytics/send-requests.sh | 16 ++-- .../workload-identity-federation/setup.sh | 6 +- .../examples/cleanup-policies.yaml | 3 +- .../gcs/examples/iam-authoritative.yaml | 3 +- .../gcs/examples/iam-bindings-additive.yaml | 3 +- tests/modules/gcs/examples/iam-bindings.yaml | 3 +- .../secret_manager/examples/secret-cmek.yaml | 2 - tools/check_boilerplate.py | 50 ++++++----- tools/pre-commit-tfdoc.sh | 34 ++++++++ tools/tflint-fast.py | 7 +- 12 files changed, 182 insertions(+), 43 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint create mode 100755 tools/pre-commit-tfdoc.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..e4b2263e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,82 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +repos: +- repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.86.0 + hooks: + - id: terraform_fmt + - id: terraform_tflint + args: + - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl + files: ^modules/ + exclude: (tests/fixtures/|tools/lockfile/) + # - id: terraform_validate + # args: + # - --hook-config=--retry-once-with-cleanup=true +- repo: local + hooks: + - id: cff-readme + name: Regenerate README.md with tfdoc.py + entry: tools/pre-commit-tfdoc.sh + language: script + #types: [terraform] + files: ^(modules|fast).*(tf|README.md)$ + pass_filenames: true + require_serial: true + - id: licenese + name: Check license presence and other boilerplate checks + language: system + entry: tools/check_boilerplate.py + pass_filenames: true + args: + - --scan-files + - id: tflint-fast + name: Checking FAST code with tflint + entry: tools/tflint-fast.py + language: system + pass_filenames: false + require_serial: true + files: ^fast/.*tf + +# - repo: https://github.com/adrienverge/yamllint +# rev: v1.34.0 +# hooks: +# - id: yamllint +# args: [-c=.yamllint, --no-warnings] +# exclude: (/templates/|modules/cloud-config-container/) + +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: "3.0.0" + hooks: + - id: script-must-have-extension + - id: shellcheck + - id: shfmt + exclude: ".*tpl" + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + args: + - --allow-multiple-documents + exclude: (/templates/|modules/cloud-config-container/) + +- repo: https://github.com/google/yapf/ + rev: v0.40.2 + hooks: + - id: yapf diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..e5f87654 --- /dev/null +++ b/.yamllint @@ -0,0 +1,16 @@ +--- + +yaml-files: +- '*.yaml' +- '*.yml' +- '.yamllint' + +extends: default + +rules: + indentation: + indent-sequences: consistent + line-length: + max: 120 + level: warning + braces: disable diff --git a/blueprints/apigee/bigquery-analytics/send-requests.sh b/blueprints/apigee/bigquery-analytics/send-requests.sh index 325cad78..7b6e7723 100755 --- a/blueprints/apigee/bigquery-analytics/send-requests.sh +++ b/blueprints/apigee/bigquery-analytics/send-requests.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,18 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -#!/bin/bash if [ $# -lt 2 ]; then - echo "Usage: $0 ENVGROUP_HOSTNAME NUM_REQUESTS" - exit 1 -fi + echo "Usage: $0 ENVGROUP_HOSTNAME NUM_REQUESTS" + exit 1 +fi ENVGROUP_HOSTNAME=$1 NUM_REQUESTS=$2 -for i in $(seq 1 $NUM_REQUESTS) -do -curl -v https://$ENVGROUP_HOSTNAME/httpbin/headers +# shellcheck disable=SC2034 +for i in $(seq 1 "$NUM_REQUESTS"); do + curl -v "https://$ENVGROUP_HOSTNAME/httpbin/headers" done diff --git a/blueprints/cloud-operations/workload-identity-federation/setup.sh b/blueprints/cloud-operations/workload-identity-federation/setup.sh index 7fe3a710..6943b648 100644 --- a/blueprints/cloud-operations/workload-identity-federation/setup.sh +++ b/blueprints/cloud-operations/workload-identity-federation/setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,11 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -#!/bin/bash sudo apt -y update sudo apt -y install apt-transport-https ca-certificates gnupg jq echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - -sudo apt-get update && sudo apt-get install google-cloud-sdk \ No newline at end of file +sudo apt-get update && sudo apt-get install google-cloud-sdk diff --git a/tests/modules/artifact_registry/examples/cleanup-policies.yaml b/tests/modules/artifact_registry/examples/cleanup-policies.yaml index f39fb8a1..94617df9 100644 --- a/tests/modules/artifact_registry/examples/cleanup-policies.yaml +++ b/tests/modules/artifact_registry/examples/cleanup-policies.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -values: values: module.registry-docker.google_artifact_registry_repository.registry: cleanup_policies: @@ -38,7 +37,7 @@ values: mode: STANDARD_REPOSITORY project: project-id repository_id: docker-cleanup-policies - + counts: google_artifact_registry_repository: 1 diff --git a/tests/modules/gcs/examples/iam-authoritative.yaml b/tests/modules/gcs/examples/iam-authoritative.yaml index 0be40cfb..84b93b1c 100644 --- a/tests/modules/gcs/examples/iam-authoritative.yaml +++ b/tests/modules/gcs/examples/iam-authoritative.yaml @@ -14,7 +14,6 @@ values: module.bucket.google_storage_bucket.bucket: - autoclass: [] cors: [] custom_placement_config: [] default_event_based_hold: null @@ -46,4 +45,4 @@ counts: google_storage_bucket: 1 google_storage_bucket_iam_binding: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2 diff --git a/tests/modules/gcs/examples/iam-bindings-additive.yaml b/tests/modules/gcs/examples/iam-bindings-additive.yaml index 0c5f2311..34191145 100644 --- a/tests/modules/gcs/examples/iam-bindings-additive.yaml +++ b/tests/modules/gcs/examples/iam-bindings-additive.yaml @@ -14,7 +14,6 @@ values: module.bucket.google_storage_bucket.bucket: - autoclass: [] cors: [] custom_placement_config: [] default_event_based_hold: null @@ -48,4 +47,4 @@ counts: google_storage_bucket: 1 google_storage_bucket_iam_member: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2 diff --git a/tests/modules/gcs/examples/iam-bindings.yaml b/tests/modules/gcs/examples/iam-bindings.yaml index 3111f57b..23d98f72 100644 --- a/tests/modules/gcs/examples/iam-bindings.yaml +++ b/tests/modules/gcs/examples/iam-bindings.yaml @@ -14,7 +14,6 @@ values: module.bucket.google_storage_bucket.bucket: - autoclass: [] cors: [] custom_placement_config: [] default_event_based_hold: null @@ -49,4 +48,4 @@ counts: google_storage_bucket: 1 google_storage_bucket_iam_binding: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2 diff --git a/tests/modules/secret_manager/examples/secret-cmek.yaml b/tests/modules/secret_manager/examples/secret-cmek.yaml index bfeec67f..c793b782 100644 --- a/tests/modules/secret_manager/examples/secret-cmek.yaml +++ b/tests/modules/secret_manager/examples/secret-cmek.yaml @@ -71,5 +71,3 @@ counts: resources: 11 outputs: {} - -outputs: {} \ No newline at end of file diff --git a/tools/check_boilerplate.py b/tools/check_boilerplate.py index a0f3ad87..6cd222c2 100755 --- a/tools/check_boilerplate.py +++ b/tools/check_boilerplate.py @@ -24,11 +24,12 @@ folder paths as arguments, as this tool is designed to be run in CI pipelines triggered by pull requests. ''' -import glob import os import re import sys +import click + _EXCLUDE_DIRS = ('.git', '.terraform') _EXCLUDE_RE = re.compile(r'# skip boilerplate check') _MATCH_FILES = ('Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml') @@ -38,23 +39,34 @@ _MATCH_STRING = (r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+' _MATCH_RE = re.compile(_MATCH_STRING, re.M) -def main(base_dirs): - "Cycle through files in base_dirs and check for the Apache 2.0 boilerplate." +def check_files(root, files, errors, warnings): + for fname in files: + if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES: + fpath = os.path.abspath(os.path.join(root, fname)) + content = open(fpath).read() + if _EXCLUDE_RE.search(content): + continue + try: + if not _MATCH_RE.search(content): + errors.append(fpath) + except (IOError, OSError): + warnings.append(fpath) + + +@click.command() +@click.argument('paths', type=str, nargs=-1) +@click.option('--scan-files', default=False, is_flag=True) +def main(paths, scan_files=False): + "Cycle through files in paths and check for the Apache 2.0 boilerplate." errors, warnings = [], [] - for dir in base_dirs: - for root, dirs, files in os.walk(dir): - dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS] - for fname in files: - if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES: - fpath = os.path.abspath(os.path.join(root, fname)) - content = open(fpath).read() - if _EXCLUDE_RE.search(content): - continue - try: - if not _MATCH_RE.search(content): - errors.append(fpath) - except (IOError, OSError): - warnings.append(fpath) + if scan_files: + check_files("./", paths, errors, warnings) + else: + for dir in paths: + for root, dirs, files in os.walk(dir): + dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS] + check_files(root, files, errors, warnings) + if warnings: print('The following files cannot be accessed:') print('\n'.join(' - {}'.format(s) for s in warnings)) @@ -65,6 +77,4 @@ def main(base_dirs): if __name__ == '__main__': - if len(sys.argv) < 2: - raise SystemExit('No directory to check.') - main(sys.argv[1:]) + main() diff --git a/tools/pre-commit-tfdoc.sh b/tools/pre-commit-tfdoc.sh new file mode 100755 index 00000000..21d18767 --- /dev/null +++ b/tools/pre-commit-tfdoc.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +files=("$@") +declare -A directories + +for file in "${files[@]}"; do + dir=$(dirname "${file}") + if [ -f "${dir}/README.md" ] && [ -f "${dir}/main.tf" ]; then + directories["${dir}"]=1 + fi +done + +for dir in "${!directories[@]}"; do # iterate over keys in directories + echo python "${SCRIPT_DIR}/tfdoc.py" "${dir}" + python "${SCRIPT_DIR}/tfdoc.py" "${dir}" +done diff --git a/tools/tflint-fast.py b/tools/tflint-fast.py index dc02ffdf..e8bb1919 100755 --- a/tools/tflint-fast.py +++ b/tools/tflint-fast.py @@ -42,8 +42,11 @@ def main(junit): args += ['--format=junit'] args += [ '--chdir', - str((BASEDIR / module_path).absolute()), '--var-file', - str((BASEDIR / var_path).absolute()) + str((BASEDIR / module_path).absolute()), + '--var-file', + str((BASEDIR / var_path).absolute()), + '--config', + str((BASEDIR / ".tflint.hcl").absolute()), ] print(' '.join(args)) if junit: