Add PR validation (#1)
This commit is contained in:
parent
5051c49e31
commit
e08529c885
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,2 @@
|
||||||
|
cbor==1.0.0
|
||||||
|
PyGithub==1.45
|
|
@ -0,0 +1,38 @@
|
||||||
|
from unpack_entities import unpack_entities, InvalidEntitiesDetected
|
||||||
|
import unittest
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
|
|
||||||
|
class TestUnpack(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.test_temp_dir = tempfile.mkdtemp()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
shutil.rmtree(self.test_temp_dir)
|
||||||
|
|
||||||
|
def fixture_dir(self, name):
|
||||||
|
return os.path.join(
|
||||||
|
CURRENT_DIR,
|
||||||
|
'fixtures', '%s_entity_packages' % name
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_entity_package_missing_files(self):
|
||||||
|
"""Tests when the entity package is missing files"""
|
||||||
|
with self.assertRaises(InvalidEntitiesDetected):
|
||||||
|
unpack_entities(self.fixture_dir('bad1'), self.test_temp_dir)
|
||||||
|
|
||||||
|
def test_entity_package_node_not_registered(self):
|
||||||
|
"""Tests when the entity package node is not registered properly"""
|
||||||
|
with self.assertRaises(InvalidEntitiesDetected):
|
||||||
|
unpack_entities(self.fixture_dir('bad2'), self.test_temp_dir)
|
||||||
|
|
||||||
|
def test_succeeds(self):
|
||||||
|
unpack_entities(self.fixture_dir('good'), self.test_temp_dir)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
|
@ -0,0 +1,113 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import tarfile
|
||||||
|
import logging
|
||||||
|
import base64
|
||||||
|
import cbor
|
||||||
|
|
||||||
|
ENTITY_FILENAME_SUFFIX = '-entity.tar.gz'
|
||||||
|
|
||||||
|
EXPECTED_FILES = [
|
||||||
|
'entity/entity.json',
|
||||||
|
'entity/entity_genesis.json',
|
||||||
|
'node/node_genesis.json',
|
||||||
|
]
|
||||||
|
|
||||||
|
logger = logging.getLogger('unpack_entities')
|
||||||
|
|
||||||
|
|
||||||
|
# May want a more granular error in the future
|
||||||
|
class InvalidEntitiesDetected(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def unpack_entities(src_entities_dir_path, dest_entities_dir_path):
|
||||||
|
invalid_entity_packages = []
|
||||||
|
# Unpack all of the entity packages in the form of `*-entity.tar.gz`. Also
|
||||||
|
# unpack the entities in lexicographical order so it is potentially easier
|
||||||
|
# to read logs.
|
||||||
|
for filename in sorted(os.listdir(src_entities_dir_path)):
|
||||||
|
if filename.endswith(ENTITY_FILENAME_SUFFIX):
|
||||||
|
entity_owner = filename[:-len(ENTITY_FILENAME_SUFFIX)]
|
||||||
|
|
||||||
|
unpacked_entity_dir_path = os.path.join(
|
||||||
|
dest_entities_dir_path,
|
||||||
|
entity_owner
|
||||||
|
)
|
||||||
|
# Create the new entity directory
|
||||||
|
logger.info('Unpack package for entity owner "%s"' % entity_owner)
|
||||||
|
os.mkdir(unpacked_entity_dir_path)
|
||||||
|
|
||||||
|
package = tarfile.open(os.path.join(src_entities_dir_path,
|
||||||
|
filename))
|
||||||
|
package.extractall(unpacked_entity_dir_path)
|
||||||
|
|
||||||
|
if not validate_entity_package(unpacked_entity_dir_path):
|
||||||
|
invalid_entity_packages.append(entity_owner)
|
||||||
|
else:
|
||||||
|
logger.info('Entity owned by "%s" is valid' % entity_owner)
|
||||||
|
|
||||||
|
if len(invalid_entity_packages) > 0:
|
||||||
|
for entity_owner in invalid_entity_packages:
|
||||||
|
logger.error('Invalid Entity for %s' % entity_owner)
|
||||||
|
raise InvalidEntitiesDetected()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_entity_package(package_path):
|
||||||
|
is_valid = True
|
||||||
|
# Validate that the expected directory structure exists
|
||||||
|
for expected_file_name in EXPECTED_FILES:
|
||||||
|
expected_file_path = os.path.join(package_path, expected_file_name)
|
||||||
|
|
||||||
|
if not os.path.isfile(expected_file_path):
|
||||||
|
logger.warning('Expected file "%s" missing' % expected_file_path)
|
||||||
|
is_valid = False
|
||||||
|
|
||||||
|
if not is_valid:
|
||||||
|
return is_valid
|
||||||
|
|
||||||
|
# Ensure that the node is properly loaded into the
|
||||||
|
# FIXME we should do this check using something written with oasis-core as a
|
||||||
|
# library. This is quick and dirty.
|
||||||
|
entity_genesis_path = os.path.join(
|
||||||
|
package_path, 'entity/entity_genesis.json')
|
||||||
|
node_genesis_path = os.path.join(package_path, 'node/node_genesis.json')
|
||||||
|
|
||||||
|
with open(entity_genesis_path) as entity_genesis_file:
|
||||||
|
entity_genesis = json.load(entity_genesis_file)
|
||||||
|
|
||||||
|
with open(node_genesis_path) as node_genesis_file:
|
||||||
|
node_genesis = json.load(node_genesis_file)
|
||||||
|
|
||||||
|
entity_descriptor = cbor.loads(base64.b64decode(
|
||||||
|
entity_genesis['untrusted_raw_value']))
|
||||||
|
node_descriptor = cbor.loads(
|
||||||
|
base64.b64decode(node_genesis['untrusted_raw_value']))
|
||||||
|
|
||||||
|
entity_nodes = entity_descriptor['nodes'] or []
|
||||||
|
|
||||||
|
if not node_descriptor['id'] in entity_nodes:
|
||||||
|
logger.warning('Expected node to be added to entity')
|
||||||
|
is_valid = False
|
||||||
|
|
||||||
|
return is_valid
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
|
||||||
|
|
||||||
|
src_entities_dir_path = os.path.abspath(sys.argv[1])
|
||||||
|
dest_entities_dir_path = os.path.abspath(sys.argv[2])
|
||||||
|
|
||||||
|
logger.info('Unpacking to %s' % dest_entities_dir_path)
|
||||||
|
try:
|
||||||
|
unpack_entities(src_entities_dir_path, dest_entities_dir_path)
|
||||||
|
except InvalidEntitiesDetected:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
|
@ -0,0 +1,62 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from github import Github
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidEntityPR(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def validate_entity_pull_request(gh, repo, pr_number):
|
||||||
|
"""Validate if this pull request is a valid entity pull request."""
|
||||||
|
repo = gh.get_repo(repo)
|
||||||
|
pr = repo.get_pull(pr_number)
|
||||||
|
pr_creator = pr.user.login
|
||||||
|
|
||||||
|
is_valid = False
|
||||||
|
|
||||||
|
expected_filename = "entities/%s-entity.tar.gz" % pr_creator
|
||||||
|
|
||||||
|
for changed_file in pr.get_files():
|
||||||
|
if changed_file.filename == expected_filename:
|
||||||
|
is_valid = True
|
||||||
|
|
||||||
|
if not is_valid:
|
||||||
|
raise InvalidEntityPR(
|
||||||
|
'The entity file is expected to be named %s. Please remediate.' % expected_filename
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
token = os.environ.get('GITHUB_TOKEN')
|
||||||
|
github_ref = os.environ.get('GITHUB_REF', '')
|
||||||
|
if not token:
|
||||||
|
print('No github token specified')
|
||||||
|
sys.exit(1)
|
||||||
|
if not github_ref:
|
||||||
|
print('No github_ref specified')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
pr_number = int(github_ref.split('/')[2])
|
||||||
|
except TypeError:
|
||||||
|
print("This might not be a PR or something is wrong")
|
||||||
|
sys.exit(1)
|
||||||
|
gh = Github(token)
|
||||||
|
|
||||||
|
print("Validating PR #%d" % pr_number)
|
||||||
|
|
||||||
|
try:
|
||||||
|
validate_entity_pull_request(
|
||||||
|
gh,
|
||||||
|
'oasislabs/the-quest-entities',
|
||||||
|
pr_number
|
||||||
|
)
|
||||||
|
except InvalidEntityPR as e:
|
||||||
|
print(e)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -0,0 +1,19 @@
|
||||||
|
name: Check validation scripts
|
||||||
|
|
||||||
|
on: [push]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Test that the validation scripts work as expected
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
|
||||||
|
- name: Setup Python 3
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: '3.x'
|
||||||
|
|
||||||
|
- run: pip3 install -r .github/scripts/python/requirements.txt
|
||||||
|
|
||||||
|
- run: python3 .github/scripts/python/test_unpack_entities.py
|
|
@ -0,0 +1,24 @@
|
||||||
|
name: Validate Entity Package PR
|
||||||
|
|
||||||
|
on: [pull_request]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
validate_package_file_name:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@master
|
||||||
|
|
||||||
|
- name: Setup Python 3
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: '3.x'
|
||||||
|
|
||||||
|
- run: pip3 install -r .github/scripts/python/requirements.txt
|
||||||
|
|
||||||
|
- name: Validate the entity package name
|
||||||
|
run: python3 .github/scripts/python/validate_pull_request.py
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||||
|
|
||||||
|
- name: Validate entity packages
|
||||||
|
run: mkdir /tmp/unpack && python3 .github/scripts/python/unpack_entities.py ./entities /tmp/unpack
|
Loading…
Reference in New Issue