merge tools changes

tools/REQUIREMENTS.txt

@@ -1 +1,2 @@
 click
+yamale

tools/check_boilerplate.py

@@ -33,22 +33,23 @@ _MATCH_STRING = (
 _MATCH_RE = re.compile(_MATCH_STRING, re.M)
 
 
-def main(dir):
-  "Cycle through files in dir and check for the Apache 2.0 boilerplate."
+def main(base_dirs):
+  "Cycle through files in base_dirs and check for the Apache 2.0 boilerplate."
   errors, warnings = [], []
-  for root, dirs, files in os.walk(dir):
-    dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS]
-    for fname in files:
-      if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES:
-        fpath = os.path.abspath(os.path.join(root, fname))
-        content = open(fpath).read()
-        if _EXCLUDE_RE.search(content):
-          continue
-        try:
-          if not _MATCH_RE.search(content):
-            errors.append(fpath)
-        except (IOError, OSError):
-          warnings.append(fpath)
+  for dir in base_dirs:
+    for root, dirs, files in os.walk(dir):
+      dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS]
+      for fname in files:
+        if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES:
+          fpath = os.path.abspath(os.path.join(root, fname))
+          content = open(fpath).read()
+          if _EXCLUDE_RE.search(content):
+            continue
+          try:
+            if not _MATCH_RE.search(content):
+              errors.append(fpath)
+          except (IOError, OSError):
+            warnings.append(fpath)
   if warnings:
     print('The following files cannot be accessed:')
     print('\n'.join(' - {}'.format(s) for s in warnings))
@@ -59,6 +60,6 @@ def main(dir):
 
 
 if __name__ == '__main__':
-  if len(sys.argv) != 2:
-    raise SystemExit('No directory passed.')
-  main(sys.argv[1])
+  if len(sys.argv) < 2:
+    raise SystemExit('No directory to check.')
+  main(sys.argv[1:])

tools/check_documentation.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import difflib
 import enum
 import pathlib
 
@@ -32,6 +33,7 @@ def _check_dir(dir_name, files=False, show_extra=False):
   for readme_path in dir_path.glob('**/README.md'):
     if '.terraform' in str(readme_path):
       continue
+    diff = None
     readme = readme_path.read_text()
     mod_name = str(readme_path.relative_to(dir_path).parent)
     result = tfdoc.get_doc(readme)
@@ -44,24 +46,36 @@ def _check_dir(dir_name, files=False, show_extra=False):
       except SystemExit:
         state = state.SKIP
       else:
-        state = State.OK if new_doc == result['doc'] else State.FAIL
-    yield mod_name, state
+        if new_doc == result['doc']:
+          state = State.OK
+        else:
+          state = State.FAIL
+          diff = '\n'.join(
+              [f'----- {mod_name} diff -----\n'] +
+              list(difflib.ndiff(
+                  result['doc'].split('\n'), new_doc.split('\n')
+              )))
+    yield mod_name, state, diff
 
 
 @click.command()
 @click.argument('dirs', type=str, nargs=-1)
-@ click.option('--show-extra/--no-show-extra', default=False)
 @ click.option('--files/--no-files', default=False)
-def main(dirs, files=False, show_extra=False):
+@ click.option('--show-diffs/--no-show-diffs', default=False)
+@ click.option('--show-extra/--no-show-extra', default=False)
+def main(dirs, files=False, show_diffs=False, show_extra=False):
   'Cycle through modules and ensure READMEs are up-to-date.'
-  errors = 0
+  errors = []
   state_labels = {State.FAIL: '✗', State.OK: '✓', State.SKIP: '?'}
   for dir_name in dirs:
     print(f'----- {dir_name} -----')
-    for mod_name, state in _check_dir(dir_name, files, show_extra):
-      errors += 1 if state == State.FAIL else 0
+    for mod_name, state, diff in _check_dir(dir_name, files, show_extra):
+      if state == State.FAIL:
+        errors.append(diff)
       print(f'[{state_labels[state]}] {mod_name}')
   if errors:
+    if show_diffs:
+      print('\n'.join(errors))
     raise SystemExit('Errors found.')
 
 

tools/validate_schema.py

@@ -0,0 +1,43 @@
+import glob
+import os
+
+import click
+import yamale
+
+
+@ click.command()
+@ click.argument('schema', type=click.Path(exists=True))
+@ click.option('--directory', multiple=True, type=click.Path(exists=True, file_okay=False, dir_okay=True))
+@ click.option('--file', multiple=True, type=click.Path(exists=True, file_okay=True, dir_okay=False))
+@ click.option('--recursive', is_flag=True, default=False)
+@ click.option('--quiet', is_flag=True, default=False)
+def main(directory=None, file=None, schema=None, recursive=False, quiet=False):
+  'Program entry point.'
+
+  yamale_schema = yamale.make_schema(schema)
+  search = "**/*.yaml" if recursive else "*.yaml"
+  has_errors = []
+
+  files = list(file)
+  for d in directory:
+    files = files + glob.glob(os.path.join(d, search), recursive=recursive)
+
+  for document in files:
+    yamale_data = yamale.make_data(document)
+    try:
+      yamale.validate(yamale_schema, yamale_data)
+      if quiet:
+        pass
+      else:
+        print(f'✅  {document} -> {os.path.basename(schema)}')
+    except ValueError as e:
+      has_errors.append(document)
+      print(e)
+      print(f'❌ {document} -> {os.path.basename(schema)}')
+
+  if len(has_errors) > 0:
+    raise SystemExit(f"❌ Errors found in {len(has_errors)} documents.")
+
+
+if __name__ == '__main__':
+  main()