zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reformat all code using yapf

This commit is contained in:
Julio Castillo 2022-02-14 12:13:42 +01:00
parent 72a62d10b6
commit e7dd12fa29
9 changed files with 78 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/linting.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
- name: Install dependencies
run: |
pip install -r tools/REQUIREMENTS.txt
pip install -r tools/requirements.txt
- name: Boilerplate
id: boilerplate
@ -67,3 +67,8 @@ jobs:
id: name-length-fast
run: |
python3 tools/check_names.py --prefix-length=10 fast/stages
- name: Check python formatting
id: yapf
run: |
yapf --style="{based_on_style: google, indent_width: 2, SPLIT_BEFORE_NAMED_ASSIGNS: false}" -p -d tools/*.py

14
tools/check_boilerplate.py
View File

@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Check that boilerplate is present in relevant files.
This tools offers a simple way of ensuring that the required boilerplate header
@ -30,17 +29,12 @@ import os
import re
import sys
_EXCLUDE_DIRS = ('.git', '.terraform')
_EXCLUDE_RE = re.compile(r'# skip boilerplate check')
_MATCH_FILES = (
'Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml'
)
_MATCH_STRING = (
r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+'
r'[#\*]\sLicensed under the Apache License, Version 2.0 '
r'\(the "License"\);\s+'
)
_MATCH_FILES = ('Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml')
_MATCH_STRING = (r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+'
r'[#\*]\sLicensed under the Apache License, Version 2.0 '
r'\(the "License"\);\s+')
_MATCH_RE = re.compile(_MATCH_STRING, re.M)

11
tools/check_documentation.py
View File

@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Recursively check freshness of tfdoc's generated tables in README files.
This tool recursively checks that the embedded variables and outputs tables in
@ -29,10 +28,8 @@ import pathlib
import click
import tfdoc
BASEDIR = pathlib.Path(__file__).resolve().parents[1]
State = enum.Enum('State', 'OK FAIL SKIP')
@ -59,11 +56,9 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False):
state = State.OK
else:
state = State.FAIL
diff = '\n'.join(
[f'----- {mod_name} diff -----\n'] +
list(difflib.ndiff(
result['doc'].split('\n'), new_doc.split('\n')
)))
header = f'----- {mod_name} diff -----\n'
ndiff = difflib.ndiff(result['doc'].split('\n'), new_doc.split('\n'))
diff = '\n'.join([header] + list(ndiff))
yield mod_name, state, diff

7
tools/check_links.py
View File

@ -13,14 +13,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Recursively check link destination validity in Markdown files.
This tool recursively checks that local links in Markdown files point to valid
destinations. Its main use is in CI pipelines triggered by pull requests.
'''
import collections
import pathlib
import urllib.parse
@ -28,7 +26,6 @@ import urllib.parse
import click
import marko
BASEDIR = pathlib.Path(__file__).resolve().parents[1]
DOC = collections.namedtuple('DOC', 'path relpath links')
LINK = collections.namedtuple('LINK', 'dest valid')
@ -57,8 +54,8 @@ def check_docs(dir_name):
yield DOC(readme_path, str(readme_path.relative_to(dir_path)), links)
@ click.command()
@ click.argument('dirs', type=str, nargs=-1)
@click.command()
@click.argument('dirs', type=str, nargs=-1)
def main(dirs):
'Check links in Markdown files contained in dirs.'
errors = 0

18
tools/check_names.py
View File

@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'Parse names from specific Terraform resources and optionally check length.'
import collections
@ -23,20 +22,17 @@ import re
import click
BASEDIR = pathlib.Path(__file__).resolve().parents[1]
LOGGER = logging.getLogger()
MOD_TOKENS = [
('NAME', r'\s*module\s*"([^"]+)"\s*\{\s*'),
('SOURCE', r'\s*source\s*=\s*"([^"]+)"\s*'),
('VALUE', r'\s*name\s*=\s*"([^"]+)"\s*'),
('REST', r'(.*)')
('REST', r'(.*)'),
]
MOD = enum.Enum('MOD', ' '.join(name for name, _ in MOD_TOKENS))
MOD_RE = re.compile('|'.join(f'(?:{pattern})' for _, pattern in MOD_TOKENS))
MOD_LIMITS = {
'project': 30, 'iam-service-account': 30, 'gcs': 63
}
MOD_LIMITS = {'project': 30, 'iam-service-account': 30, 'gcs': 63}
Name = collections.namedtuple('Name', 'source name value length')
@ -91,12 +87,10 @@ def main(dirs, prefix_length=None):
for name in names:
name_length = name.length + prefix_length
flag = '✗' if name_length >= MOD_LIMITS[name.source] else '✓'
print((
f'[{flag}] {name.source.ljust(source_just)} '
f'{name.name.ljust(name_just)} '
f'{name.value.ljust(value_just)} '
f'({name_length})'
))
print(f'[{flag}] {name.source.ljust(source_just)} '
f'{name.name.ljust(name_just)} '
f'{name.value.ljust(value_just)} '
f'({name_length})')
if __name__ == '__main__':

1
tools/REQUIREMENTS.txt → tools/requirements.txt
View File

@ -1,3 +1,4 @@
click
marko
yamale
yapf

32
tools/state_iam.py
View File

@ -22,11 +22,8 @@ import sys
import click
FIELDS = (
'authoritative', 'resource_type', 'resource_id', 'role', 'member_type',
'member_id', 'conditions'
)
FIELDS = ('authoritative', 'resource_type', 'resource_id', 'role',
'member_type', 'member_id', 'conditions')
ORG_IDS = {}
RESOURCE_SORT = {'organization': 0, 'folder': 1, 'project': 2}
RESOURCE_TYPE_RE = re.compile(r'^google_([^_]+)_iam_([^_]+)$')
@ -111,20 +108,17 @@ def output_principals(bindings):
if b.role.startswith('organizations/'):
roles.append(f'{b.role} {additive}{conditions}')
else:
url = (
'https://cloud.google.com/iam/docs/understanding-roles#'
f'{b.role.replace("roles/", "")}'
)
url = ('https://cloud.google.com/iam/docs/understanding-roles#'
f'{b.role.replace("roles/", "")}')
roles.append(f'[{b.role}]({url}) {additive}{conditions}')
print((
f'|<b>{principal[1]}</b><br><small><i>{principal[0]}</i></small>|'
f'{"<br>".join(roles)}|'
))
print(f'|<b>{principal[1]}</b><br><small><i>{principal[0]}</i></small>|'
f'{"<br>".join(roles)}|')
@click.command()
@click.argument('state-file', type=click.File('r'), default=sys.stdin)
@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']), default='raw')
@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']),
default='raw')
@click.option('--prefix', default=None)
def main(state_file, format, prefix=None):
'Output IAM bindings parsed from Terraform state file or standard input.'
@ -133,9 +127,13 @@ def main(state_file, format, prefix=None):
resources = data.get('resources', [])
folders = dict(get_folders(resources))
bindings = get_bindings(resources, prefix=prefix, folders=folders)
bindings = sorted(bindings, key=lambda b: (
RESOURCE_SORT.get(b.resource_type, 99), b.resource_id,
b.member_type, b.member_id))
bindings = sorted(
bindings, key=lambda b: (
RESOURCE_SORT.get(b.resource_type, 99),
b.resource_id,
b.member_type,
b.member_id,
))
if format == 'raw':
for b in bindings:
print(b)

93
tools/tfdoc.py
View File

@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Generate tables for Terraform root module files, outputs and variables.
This tool generates nicely formatted Markdown tables from Terraform source
@ -48,13 +47,10 @@ import urllib.parse
import click
__version__ = '2.1.0'
# TODO(ludomagno): decide if we want to support variables*.tf and outputs*.tf
FILE_DESC_DEFAULTS = {
'main.tf': 'Module-level locals and resources.',
'outputs.tf': 'Module outputs.',
@ -63,11 +59,8 @@ FILE_DESC_DEFAULTS = {
'versions.tf': 'Version pins.',
}
FILE_RE_MODULES = re.compile(
r'(?sm)module\s*"[^"]+"\s*\{[^\}]*?source\s*=\s*"([^"]+)"'
)
FILE_RE_RESOURCES = re.compile(
r'(?sm)resource\s*"([^"]+)"'
)
r'(?sm)module\s*"[^"]+"\s*\{[^\}]*?source\s*=\s*"([^"]+)"')
FILE_RE_RESOURCES = re.compile(r'(?sm)resource\s*"([^"]+)"')
HEREDOC_RE = re.compile(r'(?sm)^<<\-?END(\s*.*?)\s*END$')
MARK_BEGIN = '<!-- BEGIN TFDOC -->'
MARK_END = '<!-- END TFDOC -->'
@ -88,8 +81,7 @@ OUT_RE = re.compile(r'''(?smx)
OUT_TEMPLATE = ('description', 'value', 'sensitive')
TAG_RE = re.compile(r'(?sm)^\s*#\stfdoc:([^:]+:\S+)\s+(.*?)\s*$')
UNESCAPED = string.digits + string.ascii_letters + ' .,;:_-'
VAR_ENUM = enum.Enum(
'V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT')
VAR_ENUM = enum.Enum('V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT')
VAR_RE = re.compile(r'''(?smx)
# variable open
(?:^\s*variable\s*"([^"]+)"\s*\{\s*$) |
@ -107,14 +99,12 @@ VAR_RE = re.compile(r'''(?smx)
VAR_RE_TYPE = re.compile(r'([\(\{\}\)])')
VAR_TEMPLATE = ('default', 'description', 'type', 'nullable')
File = collections.namedtuple('File', 'name description modules resources')
Output = collections.namedtuple('Output',
'name description sensitive consumers line')
Variable = collections.namedtuple(
'Variable', 'name description type default required nullable source line')
# parsing functions
@ -144,7 +134,7 @@ def _parse(body, enum=VAR_ENUM, re=VAR_RE, template=VAR_TEMPLATE):
elif token == enum.ATTR_DATA:
if not item:
continue
context = m.group(m.lastindex-1)
context = m.group(m.lastindex - 1)
item[context].append(data)
elif token == enum.SKIP:
context = token
@ -170,12 +160,11 @@ def parse_files(basepath, exclude_files=None):
except (IOError, OSError) as e:
raise SystemExit(f'Cannot read file {name}: {e}')
tags = _extract_tags(body)
description = tags.get(
'file:description', FILE_DESC_DEFAULTS.get(shortname))
description = tags.get('file:description',
FILE_DESC_DEFAULTS.get(shortname))
modules = set(
os.path.basename(urllib.parse.urlparse(m).path)
for m in FILE_RE_MODULES.findall(body)
)
for m in FILE_RE_MODULES.findall(body))
resources = set(FILE_RE_RESOURCES.findall(body))
yield File(shortname, description, modules, resources)
@ -188,11 +177,11 @@ def parse_outputs(basepath):
except (IOError, OSError) as e:
raise SystemExit(f'No outputs file in {basepath}.')
for item in _parse(body, enum=OUT_ENUM, re=OUT_RE, template=OUT_TEMPLATE):
yield Output(name=item['name'],
description=''.join(item['description']),
sensitive=item['sensitive'] != [],
consumers=item['tags'].get('output:consumers', ''),
line=item['line'])
description = ''.join(item['description'])
sensitive = item['sensitive'] != []
consumers = item['tags'].get('output:consumers', '')
yield Output(name=item['name'], description=description,
sensitive=sensitive, consumers=consumers, line=item['line'])
def parse_variables(basepath):
@ -203,20 +192,18 @@ def parse_variables(basepath):
except (IOError, OSError) as e:
raise SystemExit(f'No variables file in {basepath}.')
for item in _parse(body):
description = ''.join(item['description'])
vtype = '\n'.join(item['type'])
default = HEREDOC_RE.sub(r'\1', '\n'.join(item['default']))
required = not item['default']
vtype = '\n'.join(item['type'])
nullable = item.get('nullable') != ['false']
source = item['tags'].get('variable:source', '')
if not required and default != 'null' and vtype == 'string':
default = f'"{default}"'
yield Variable(name=item['name'],
description=''.join(item['description']),
type=vtype,
default=default,
required=required,
source=item['tags'].get('variable:source', ''),
line=item['line'],
nullable=nullable)
yield Variable(name=item['name'], description=description, type=vtype,
default=default, required=required, source=source,
line=item['line'], nullable=nullable)
# formatting functions
@ -251,25 +238,19 @@ def format_files(items):
num_resources = sum(len(i.resources) for i in items)
yield '| name | description |{}{}'.format(
' modules |' if num_modules else '',
' resources |' if num_resources else ''
)
yield '|---|---|{}{}'.format(
'---|' if num_modules else '',
'---|' if num_resources else ''
)
' resources |' if num_resources else '')
yield '|---|---|{}{}'.format('---|' if num_modules else '',
'---|' if num_resources else '')
for i in items:
modules = resources = ''
if i.modules:
modules = '<code>%s</code>' % '</code> · <code>'.join(
sorted(i.modules))
modules = '<code>%s</code>' % '</code> · <code>'.join(sorted(i.modules))
if i.resources:
resources = '<code>%s</code>' % '</code> · <code>'.join(
sorted(i.resources))
yield '| [{}](./{}) | {} |{}{}'.format(
i.name, i.name, i.description,
f' {modules} |' if num_modules else '',
f' {resources} |' if num_resources else ''
)
i.name, i.name, i.description, f' {modules} |' if num_modules else '',
f' {resources} |' if num_resources else '')
def format_outputs(items, show_extra=True):
@ -277,17 +258,13 @@ def format_outputs(items, show_extra=True):
if not items:
return
items.sort(key=lambda i: i.name)
yield '| name | description | sensitive |' + (
' consumers |' if show_extra else ''
)
yield '|---|---|:---:|' + (
'---|' if show_extra else ''
)
yield '| name | description | sensitive |' + (' consumers |'
if show_extra else '')
yield '|---|---|:---:|' + ('---|' if show_extra else '')
for i in items:
consumers = i.consumers or ''
if consumers:
consumers = '<code>%s</code>' % '</code> · <code>'.join(
consumers.split())
consumers = '<code>%s</code>' % '</code> · <code>'.join(consumers.split())
sensitive = '✓' if i.sensitive else ''
format = f'| [{i.name}](outputs.tf#L{i.line}) | {i.description or ""} | {sensitive} |'
format += f' {consumers} |' if show_extra else ''
@ -301,11 +278,8 @@ def format_variables(items, show_extra=True):
items.sort(key=lambda i: i.name)
items.sort(key=lambda i: i.required, reverse=True)
yield '| name | description | type | required | default |' + (
' producer |' if show_extra else ''
)
yield '|---|---|:---:|:---:|:---:|' + (
':---:|' if show_extra else ''
)
' producer |' if show_extra else '')
yield '|---|---|:---:|:---:|:---:|' + (':---:|' if show_extra else '')
for i in items:
vars = {
'default': f'<code>{_escape(i.default)}</code>' if i.default else '',
@ -326,8 +300,7 @@ def format_variables(items, show_extra=True):
vars[k] = f'<code title="{_escape(title)}">{_escape(value)}</code>'
format = (
f'| [{i.name}](variables.tf#L{i.line}) | {i.description or ""} | {vars["type"]} '
f'| {vars["required"]} | {vars["default"]} |'
)
f'| {vars["required"]} | {vars["default"]} |')
format += f' {vars["source"]} |' if show_extra else ''
yield format
@ -396,7 +369,7 @@ def replace_doc(readme_path, doc, readme=None):
MARK_BEGIN,
doc,
MARK_END,
readme[result['end']:].lstrip()
readme[result['end']:].lstrip(),
]))
except (IOError, OSError) as e:
raise SystemExit(f'Error replacing README {readme_path}: {e}')

15
tools/validate_schema.py
View File

@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Validate YaML document against yamale schemas.
Fast includes YaML driven resource factories, along with their schemas which
are available at `fast/assets/schemas`.
@ -29,12 +28,14 @@ import click
import yamale
@ click.command()
@ click.argument('schema', type=click.Path(exists=True))
@ click.option('--directory', multiple=True, type=click.Path(exists=True, file_okay=False, dir_okay=True))
@ click.option('--file', multiple=True, type=click.Path(exists=True, file_okay=True, dir_okay=False))
@ click.option('--recursive', is_flag=True, default=False)
@ click.option('--quiet', is_flag=True, default=False)
@click.command()
@click.argument('schema', type=click.Path(exists=True))
@click.option('--directory', multiple=True,
type=click.Path(exists=True, file_okay=False, dir_okay=True))
@click.option('--file', multiple=True,
type=click.Path(exists=True, file_okay=True, dir_okay=False))
@click.option('--recursive', is_flag=True, default=False)
@click.option('--quiet', is_flag=True, default=False)
def main(directory=None, file=None, schema=None, recursive=False, quiet=False):
'Program entry point.'