Merge branch 'master' into lcaggioni/fast-data-platform
This commit is contained in:
commit
5eff9ecf43
|
@ -41,7 +41,7 @@ jobs:
|
|||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r tools/REQUIREMENTS.txt
|
||||
pip install -r tools/requirements.txt
|
||||
|
||||
- name: Boilerplate
|
||||
id: boilerplate
|
||||
|
@ -67,3 +67,8 @@ jobs:
|
|||
id: name-length-fast
|
||||
run: |
|
||||
python3 tools/check_names.py --prefix-length=10 fast/stages
|
||||
|
||||
- name: Check python formatting
|
||||
id: yapf
|
||||
run: |
|
||||
yapf --style="{based_on_style: google, indent_width: 2, SPLIT_BEFORE_NAMED_ASSIGNS: false}" -p -d tools/*.py
|
||||
|
|
|
@ -13,7 +13,6 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''Check that boilerplate is present in relevant files.
|
||||
|
||||
This tools offers a simple way of ensuring that the required boilerplate header
|
||||
|
@ -30,17 +29,12 @@ import os
|
|||
import re
|
||||
import sys
|
||||
|
||||
|
||||
_EXCLUDE_DIRS = ('.git', '.terraform')
|
||||
_EXCLUDE_RE = re.compile(r'# skip boilerplate check')
|
||||
_MATCH_FILES = (
|
||||
'Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml'
|
||||
)
|
||||
_MATCH_STRING = (
|
||||
r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+'
|
||||
r'[#\*]\sLicensed under the Apache License, Version 2.0 '
|
||||
r'\(the "License"\);\s+'
|
||||
)
|
||||
_MATCH_FILES = ('Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml')
|
||||
_MATCH_STRING = (r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+'
|
||||
r'[#\*]\sLicensed under the Apache License, Version 2.0 '
|
||||
r'\(the "License"\);\s+')
|
||||
_MATCH_RE = re.compile(_MATCH_STRING, re.M)
|
||||
|
||||
|
||||
|
|
|
@ -13,7 +13,6 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''Recursively check freshness of tfdoc's generated tables in README files.
|
||||
|
||||
This tool recursively checks that the embedded variables and outputs tables in
|
||||
|
@ -29,10 +28,8 @@ import pathlib
|
|||
import click
|
||||
import tfdoc
|
||||
|
||||
|
||||
BASEDIR = pathlib.Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
State = enum.Enum('State', 'OK FAIL SKIP')
|
||||
|
||||
|
||||
|
@ -59,11 +56,9 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False):
|
|||
state = State.OK
|
||||
else:
|
||||
state = State.FAIL
|
||||
diff = '\n'.join(
|
||||
[f'----- {mod_name} diff -----\n'] +
|
||||
list(difflib.ndiff(
|
||||
result['doc'].split('\n'), new_doc.split('\n')
|
||||
)))
|
||||
header = f'----- {mod_name} diff -----\n'
|
||||
ndiff = difflib.ndiff(result['doc'].split('\n'), new_doc.split('\n'))
|
||||
diff = '\n'.join([header] + list(ndiff))
|
||||
yield mod_name, state, diff
|
||||
|
||||
|
||||
|
|
|
@ -13,14 +13,12 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''Recursively check link destination validity in Markdown files.
|
||||
|
||||
This tool recursively checks that local links in Markdown files point to valid
|
||||
destinations. Its main use is in CI pipelines triggered by pull requests.
|
||||
'''
|
||||
|
||||
|
||||
import collections
|
||||
import pathlib
|
||||
import urllib.parse
|
||||
|
@ -28,7 +26,6 @@ import urllib.parse
|
|||
import click
|
||||
import marko
|
||||
|
||||
|
||||
BASEDIR = pathlib.Path(__file__).resolve().parents[1]
|
||||
DOC = collections.namedtuple('DOC', 'path relpath links')
|
||||
LINK = collections.namedtuple('LINK', 'dest valid')
|
||||
|
@ -57,8 +54,8 @@ def check_docs(dir_name):
|
|||
yield DOC(readme_path, str(readme_path.relative_to(dir_path)), links)
|
||||
|
||||
|
||||
@ click.command()
|
||||
@ click.argument('dirs', type=str, nargs=-1)
|
||||
@click.command()
|
||||
@click.argument('dirs', type=str, nargs=-1)
|
||||
def main(dirs):
|
||||
'Check links in Markdown files contained in dirs.'
|
||||
errors = 0
|
||||
|
|
|
@ -12,7 +12,6 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'Parse names from specific Terraform resources and optionally check length.'
|
||||
|
||||
import collections
|
||||
|
@ -23,20 +22,17 @@ import re
|
|||
|
||||
import click
|
||||
|
||||
|
||||
BASEDIR = pathlib.Path(__file__).resolve().parents[1]
|
||||
LOGGER = logging.getLogger()
|
||||
MOD_TOKENS = [
|
||||
('NAME', r'\s*module\s*"([^"]+)"\s*\{\s*'),
|
||||
('SOURCE', r'\s*source\s*=\s*"([^"]+)"\s*'),
|
||||
('VALUE', r'\s*name\s*=\s*"([^"]+)"\s*'),
|
||||
('REST', r'(.*)')
|
||||
('REST', r'(.*)'),
|
||||
]
|
||||
MOD = enum.Enum('MOD', ' '.join(name for name, _ in MOD_TOKENS))
|
||||
MOD_RE = re.compile('|'.join(f'(?:{pattern})' for _, pattern in MOD_TOKENS))
|
||||
MOD_LIMITS = {
|
||||
'project': 30, 'iam-service-account': 30, 'gcs': 63
|
||||
}
|
||||
MOD_LIMITS = {'project': 30, 'iam-service-account': 30, 'gcs': 63}
|
||||
|
||||
Name = collections.namedtuple('Name', 'source name value length')
|
||||
|
||||
|
@ -91,12 +87,10 @@ def main(dirs, prefix_length=None):
|
|||
for name in names:
|
||||
name_length = name.length + prefix_length
|
||||
flag = '✗' if name_length >= MOD_LIMITS[name.source] else '✓'
|
||||
print((
|
||||
f'[{flag}] {name.source.ljust(source_just)} '
|
||||
f'{name.name.ljust(name_just)} '
|
||||
f'{name.value.ljust(value_just)} '
|
||||
f'({name_length})'
|
||||
))
|
||||
print(f'[{flag}] {name.source.ljust(source_just)} '
|
||||
f'{name.name.ljust(name_just)} '
|
||||
f'{name.value.ljust(value_just)} '
|
||||
f'({name_length})')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
click
|
||||
marko
|
||||
yamale
|
||||
yapf
|
|
@ -22,11 +22,8 @@ import sys
|
|||
|
||||
import click
|
||||
|
||||
|
||||
FIELDS = (
|
||||
'authoritative', 'resource_type', 'resource_id', 'role', 'member_type',
|
||||
'member_id', 'conditions'
|
||||
)
|
||||
FIELDS = ('authoritative', 'resource_type', 'resource_id', 'role',
|
||||
'member_type', 'member_id', 'conditions')
|
||||
ORG_IDS = {}
|
||||
RESOURCE_SORT = {'organization': 0, 'folder': 1, 'project': 2}
|
||||
RESOURCE_TYPE_RE = re.compile(r'^google_([^_]+)_iam_([^_]+)$')
|
||||
|
@ -111,20 +108,17 @@ def output_principals(bindings):
|
|||
if b.role.startswith('organizations/'):
|
||||
roles.append(f'{b.role} {additive}{conditions}')
|
||||
else:
|
||||
url = (
|
||||
'https://cloud.google.com/iam/docs/understanding-roles#'
|
||||
f'{b.role.replace("roles/", "")}'
|
||||
)
|
||||
url = ('https://cloud.google.com/iam/docs/understanding-roles#'
|
||||
f'{b.role.replace("roles/", "")}')
|
||||
roles.append(f'[{b.role}]({url}) {additive}{conditions}')
|
||||
print((
|
||||
f'|<b>{principal[1]}</b><br><small><i>{principal[0]}</i></small>|'
|
||||
f'{"<br>".join(roles)}|'
|
||||
))
|
||||
print(f'|<b>{principal[1]}</b><br><small><i>{principal[0]}</i></small>|'
|
||||
f'{"<br>".join(roles)}|')
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('state-file', type=click.File('r'), default=sys.stdin)
|
||||
@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']), default='raw')
|
||||
@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']),
|
||||
default='raw')
|
||||
@click.option('--prefix', default=None)
|
||||
def main(state_file, format, prefix=None):
|
||||
'Output IAM bindings parsed from Terraform state file or standard input.'
|
||||
|
@ -133,9 +127,13 @@ def main(state_file, format, prefix=None):
|
|||
resources = data.get('resources', [])
|
||||
folders = dict(get_folders(resources))
|
||||
bindings = get_bindings(resources, prefix=prefix, folders=folders)
|
||||
bindings = sorted(bindings, key=lambda b: (
|
||||
RESOURCE_SORT.get(b.resource_type, 99), b.resource_id,
|
||||
b.member_type, b.member_id))
|
||||
bindings = sorted(
|
||||
bindings, key=lambda b: (
|
||||
RESOURCE_SORT.get(b.resource_type, 99),
|
||||
b.resource_id,
|
||||
b.member_type,
|
||||
b.member_id,
|
||||
))
|
||||
if format == 'raw':
|
||||
for b in bindings:
|
||||
print(b)
|
||||
|
|
|
@ -13,7 +13,6 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''Generate tables for Terraform root module files, outputs and variables.
|
||||
|
||||
This tool generates nicely formatted Markdown tables from Terraform source
|
||||
|
@ -48,13 +47,10 @@ import urllib.parse
|
|||
|
||||
import click
|
||||
|
||||
|
||||
__version__ = '2.1.0'
|
||||
|
||||
|
||||
# TODO(ludomagno): decide if we want to support variables*.tf and outputs*.tf
|
||||
|
||||
|
||||
FILE_DESC_DEFAULTS = {
|
||||
'main.tf': 'Module-level locals and resources.',
|
||||
'outputs.tf': 'Module outputs.',
|
||||
|
@ -63,11 +59,8 @@ FILE_DESC_DEFAULTS = {
|
|||
'versions.tf': 'Version pins.',
|
||||
}
|
||||
FILE_RE_MODULES = re.compile(
|
||||
r'(?sm)module\s*"[^"]+"\s*\{[^\}]*?source\s*=\s*"([^"]+)"'
|
||||
)
|
||||
FILE_RE_RESOURCES = re.compile(
|
||||
r'(?sm)resource\s*"([^"]+)"'
|
||||
)
|
||||
r'(?sm)module\s*"[^"]+"\s*\{[^\}]*?source\s*=\s*"([^"]+)"')
|
||||
FILE_RE_RESOURCES = re.compile(r'(?sm)resource\s*"([^"]+)"')
|
||||
HEREDOC_RE = re.compile(r'(?sm)^<<\-?END(\s*.*?)\s*END$')
|
||||
MARK_BEGIN = '<!-- BEGIN TFDOC -->'
|
||||
MARK_END = '<!-- END TFDOC -->'
|
||||
|
@ -88,8 +81,7 @@ OUT_RE = re.compile(r'''(?smx)
|
|||
OUT_TEMPLATE = ('description', 'value', 'sensitive')
|
||||
TAG_RE = re.compile(r'(?sm)^\s*#\stfdoc:([^:]+:\S+)\s+(.*?)\s*$')
|
||||
UNESCAPED = string.digits + string.ascii_letters + ' .,;:_-'
|
||||
VAR_ENUM = enum.Enum(
|
||||
'V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT')
|
||||
VAR_ENUM = enum.Enum('V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT')
|
||||
VAR_RE = re.compile(r'''(?smx)
|
||||
# variable open
|
||||
(?:^\s*variable\s*"([^"]+)"\s*\{\s*$) |
|
||||
|
@ -107,14 +99,12 @@ VAR_RE = re.compile(r'''(?smx)
|
|||
VAR_RE_TYPE = re.compile(r'([\(\{\}\)])')
|
||||
VAR_TEMPLATE = ('default', 'description', 'type', 'nullable')
|
||||
|
||||
|
||||
File = collections.namedtuple('File', 'name description modules resources')
|
||||
Output = collections.namedtuple('Output',
|
||||
'name description sensitive consumers line')
|
||||
Variable = collections.namedtuple(
|
||||
'Variable', 'name description type default required nullable source line')
|
||||
|
||||
|
||||
# parsing functions
|
||||
|
||||
|
||||
|
@ -144,7 +134,7 @@ def _parse(body, enum=VAR_ENUM, re=VAR_RE, template=VAR_TEMPLATE):
|
|||
elif token == enum.ATTR_DATA:
|
||||
if not item:
|
||||
continue
|
||||
context = m.group(m.lastindex-1)
|
||||
context = m.group(m.lastindex - 1)
|
||||
item[context].append(data)
|
||||
elif token == enum.SKIP:
|
||||
context = token
|
||||
|
@ -170,12 +160,11 @@ def parse_files(basepath, exclude_files=None):
|
|||
except (IOError, OSError) as e:
|
||||
raise SystemExit(f'Cannot read file {name}: {e}')
|
||||
tags = _extract_tags(body)
|
||||
description = tags.get(
|
||||
'file:description', FILE_DESC_DEFAULTS.get(shortname))
|
||||
description = tags.get('file:description',
|
||||
FILE_DESC_DEFAULTS.get(shortname))
|
||||
modules = set(
|
||||
os.path.basename(urllib.parse.urlparse(m).path)
|
||||
for m in FILE_RE_MODULES.findall(body)
|
||||
)
|
||||
for m in FILE_RE_MODULES.findall(body))
|
||||
resources = set(FILE_RE_RESOURCES.findall(body))
|
||||
yield File(shortname, description, modules, resources)
|
||||
|
||||
|
@ -188,11 +177,11 @@ def parse_outputs(basepath):
|
|||
except (IOError, OSError) as e:
|
||||
raise SystemExit(f'No outputs file in {basepath}.')
|
||||
for item in _parse(body, enum=OUT_ENUM, re=OUT_RE, template=OUT_TEMPLATE):
|
||||
yield Output(name=item['name'],
|
||||
description=''.join(item['description']),
|
||||
sensitive=item['sensitive'] != [],
|
||||
consumers=item['tags'].get('output:consumers', ''),
|
||||
line=item['line'])
|
||||
description = ''.join(item['description'])
|
||||
sensitive = item['sensitive'] != []
|
||||
consumers = item['tags'].get('output:consumers', '')
|
||||
yield Output(name=item['name'], description=description,
|
||||
sensitive=sensitive, consumers=consumers, line=item['line'])
|
||||
|
||||
|
||||
def parse_variables(basepath):
|
||||
|
@ -203,20 +192,18 @@ def parse_variables(basepath):
|
|||
except (IOError, OSError) as e:
|
||||
raise SystemExit(f'No variables file in {basepath}.')
|
||||
for item in _parse(body):
|
||||
description = ''.join(item['description'])
|
||||
vtype = '\n'.join(item['type'])
|
||||
default = HEREDOC_RE.sub(r'\1', '\n'.join(item['default']))
|
||||
required = not item['default']
|
||||
vtype = '\n'.join(item['type'])
|
||||
nullable = item.get('nullable') != ['false']
|
||||
source = item['tags'].get('variable:source', '')
|
||||
if not required and default != 'null' and vtype == 'string':
|
||||
default = f'"{default}"'
|
||||
yield Variable(name=item['name'],
|
||||
description=''.join(item['description']),
|
||||
type=vtype,
|
||||
default=default,
|
||||
required=required,
|
||||
source=item['tags'].get('variable:source', ''),
|
||||
line=item['line'],
|
||||
nullable=nullable)
|
||||
|
||||
yield Variable(name=item['name'], description=description, type=vtype,
|
||||
default=default, required=required, source=source,
|
||||
line=item['line'], nullable=nullable)
|
||||
|
||||
|
||||
# formatting functions
|
||||
|
@ -251,25 +238,19 @@ def format_files(items):
|
|||
num_resources = sum(len(i.resources) for i in items)
|
||||
yield '| name | description |{}{}'.format(
|
||||
' modules |' if num_modules else '',
|
||||
' resources |' if num_resources else ''
|
||||
)
|
||||
yield '|---|---|{}{}'.format(
|
||||
'---|' if num_modules else '',
|
||||
'---|' if num_resources else ''
|
||||
)
|
||||
' resources |' if num_resources else '')
|
||||
yield '|---|---|{}{}'.format('---|' if num_modules else '',
|
||||
'---|' if num_resources else '')
|
||||
for i in items:
|
||||
modules = resources = ''
|
||||
if i.modules:
|
||||
modules = '<code>%s</code>' % '</code> · <code>'.join(
|
||||
sorted(i.modules))
|
||||
modules = '<code>%s</code>' % '</code> · <code>'.join(sorted(i.modules))
|
||||
if i.resources:
|
||||
resources = '<code>%s</code>' % '</code> · <code>'.join(
|
||||
sorted(i.resources))
|
||||
yield '| [{}](./{}) | {} |{}{}'.format(
|
||||
i.name, i.name, i.description,
|
||||
f' {modules} |' if num_modules else '',
|
||||
f' {resources} |' if num_resources else ''
|
||||
)
|
||||
i.name, i.name, i.description, f' {modules} |' if num_modules else '',
|
||||
f' {resources} |' if num_resources else '')
|
||||
|
||||
|
||||
def format_outputs(items, show_extra=True):
|
||||
|
@ -277,17 +258,13 @@ def format_outputs(items, show_extra=True):
|
|||
if not items:
|
||||
return
|
||||
items.sort(key=lambda i: i.name)
|
||||
yield '| name | description | sensitive |' + (
|
||||
' consumers |' if show_extra else ''
|
||||
)
|
||||
yield '|---|---|:---:|' + (
|
||||
'---|' if show_extra else ''
|
||||
)
|
||||
yield '| name | description | sensitive |' + (' consumers |'
|
||||
if show_extra else '')
|
||||
yield '|---|---|:---:|' + ('---|' if show_extra else '')
|
||||
for i in items:
|
||||
consumers = i.consumers or ''
|
||||
if consumers:
|
||||
consumers = '<code>%s</code>' % '</code> · <code>'.join(
|
||||
consumers.split())
|
||||
consumers = '<code>%s</code>' % '</code> · <code>'.join(consumers.split())
|
||||
sensitive = '✓' if i.sensitive else ''
|
||||
format = f'| [{i.name}](outputs.tf#L{i.line}) | {i.description or ""} | {sensitive} |'
|
||||
format += f' {consumers} |' if show_extra else ''
|
||||
|
@ -301,11 +278,8 @@ def format_variables(items, show_extra=True):
|
|||
items.sort(key=lambda i: i.name)
|
||||
items.sort(key=lambda i: i.required, reverse=True)
|
||||
yield '| name | description | type | required | default |' + (
|
||||
' producer |' if show_extra else ''
|
||||
)
|
||||
yield '|---|---|:---:|:---:|:---:|' + (
|
||||
':---:|' if show_extra else ''
|
||||
)
|
||||
' producer |' if show_extra else '')
|
||||
yield '|---|---|:---:|:---:|:---:|' + (':---:|' if show_extra else '')
|
||||
for i in items:
|
||||
vars = {
|
||||
'default': f'<code>{_escape(i.default)}</code>' if i.default else '',
|
||||
|
@ -326,8 +300,7 @@ def format_variables(items, show_extra=True):
|
|||
vars[k] = f'<code title="{_escape(title)}">{_escape(value)}</code>'
|
||||
format = (
|
||||
f'| [{i.name}](variables.tf#L{i.line}) | {i.description or ""} | {vars["type"]} '
|
||||
f'| {vars["required"]} | {vars["default"]} |'
|
||||
)
|
||||
f'| {vars["required"]} | {vars["default"]} |')
|
||||
format += f' {vars["source"]} |' if show_extra else ''
|
||||
yield format
|
||||
|
||||
|
@ -396,7 +369,7 @@ def replace_doc(readme_path, doc, readme=None):
|
|||
MARK_BEGIN,
|
||||
doc,
|
||||
MARK_END,
|
||||
readme[result['end']:].lstrip()
|
||||
readme[result['end']:].lstrip(),
|
||||
]))
|
||||
except (IOError, OSError) as e:
|
||||
raise SystemExit(f'Error replacing README {readme_path}: {e}')
|
||||
|
|
|
@ -13,7 +13,6 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''Validate YaML document against yamale schemas.
|
||||
Fast includes YaML driven resource factories, along with their schemas which
|
||||
are available at `fast/assets/schemas`.
|
||||
|
@ -29,12 +28,14 @@ import click
|
|||
import yamale
|
||||
|
||||
|
||||
@ click.command()
|
||||
@ click.argument('schema', type=click.Path(exists=True))
|
||||
@ click.option('--directory', multiple=True, type=click.Path(exists=True, file_okay=False, dir_okay=True))
|
||||
@ click.option('--file', multiple=True, type=click.Path(exists=True, file_okay=True, dir_okay=False))
|
||||
@ click.option('--recursive', is_flag=True, default=False)
|
||||
@ click.option('--quiet', is_flag=True, default=False)
|
||||
@click.command()
|
||||
@click.argument('schema', type=click.Path(exists=True))
|
||||
@click.option('--directory', multiple=True,
|
||||
type=click.Path(exists=True, file_okay=False, dir_okay=True))
|
||||
@click.option('--file', multiple=True,
|
||||
type=click.Path(exists=True, file_okay=True, dir_okay=False))
|
||||
@click.option('--recursive', is_flag=True, default=False)
|
||||
@click.option('--quiet', is_flag=True, default=False)
|
||||
def main(directory=None, file=None, schema=None, recursive=False, quiet=False):
|
||||
'Program entry point.'
|
||||
|
||||
|
|
Loading…
Reference in New Issue