diff --git a/.github/scripts/python/fixtures/bad1_entity_packages/bad1-entity.tar.gz b/.github/scripts/python/fixtures/bad1_entity_packages/bad1-entity.tar.gz new file mode 100644 index 0000000..9950de8 Binary files /dev/null and b/.github/scripts/python/fixtures/bad1_entity_packages/bad1-entity.tar.gz differ diff --git a/.github/scripts/python/fixtures/bad2_entity_packages/bad2-entity.tar.gz b/.github/scripts/python/fixtures/bad2_entity_packages/bad2-entity.tar.gz new file mode 100644 index 0000000..cc62204 Binary files /dev/null and b/.github/scripts/python/fixtures/bad2_entity_packages/bad2-entity.tar.gz differ diff --git a/.github/scripts/python/fixtures/good_entity_packages/good-entity.tar.gz b/.github/scripts/python/fixtures/good_entity_packages/good-entity.tar.gz new file mode 100644 index 0000000..164e702 Binary files /dev/null and b/.github/scripts/python/fixtures/good_entity_packages/good-entity.tar.gz differ diff --git a/.github/scripts/python/requirements.txt b/.github/scripts/python/requirements.txt new file mode 100644 index 0000000..c2886b6 --- /dev/null +++ b/.github/scripts/python/requirements.txt @@ -0,0 +1,2 @@ +cbor==1.0.0 +PyGithub==1.45 diff --git a/.github/scripts/python/test_unpack_entities.py b/.github/scripts/python/test_unpack_entities.py new file mode 100644 index 0000000..6b8c5c4 --- /dev/null +++ b/.github/scripts/python/test_unpack_entities.py @@ -0,0 +1,38 @@ +from unpack_entities import unpack_entities, InvalidEntitiesDetected +import unittest +import os +import shutil +import tempfile + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestUnpack(unittest.TestCase): + def setUp(self): + self.test_temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_temp_dir) + + def fixture_dir(self, name): + return os.path.join( + CURRENT_DIR, + 'fixtures', '%s_entity_packages' % name + ) + + def test_entity_package_missing_files(self): + """Tests when the entity package is missing files""" + with self.assertRaises(InvalidEntitiesDetected): + unpack_entities(self.fixture_dir('bad1'), self.test_temp_dir) + + def test_entity_package_node_not_registered(self): + """Tests when the entity package node is not registered properly""" + with self.assertRaises(InvalidEntitiesDetected): + unpack_entities(self.fixture_dir('bad2'), self.test_temp_dir) + + def test_succeeds(self): + unpack_entities(self.fixture_dir('good'), self.test_temp_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/python/unpack_entities.py b/.github/scripts/python/unpack_entities.py new file mode 100755 index 0000000..99aa079 --- /dev/null +++ b/.github/scripts/python/unpack_entities.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import tempfile +import tarfile +import logging +import base64 +import cbor + +ENTITY_FILENAME_SUFFIX = '-entity.tar.gz' + +EXPECTED_FILES = [ + 'entity/entity.json', + 'entity/entity_genesis.json', + 'node/node_genesis.json', +] + +logger = logging.getLogger('unpack_entities') + + +# May want a more granular error in the future +class InvalidEntitiesDetected(Exception): + pass + + +def unpack_entities(src_entities_dir_path, dest_entities_dir_path): + invalid_entity_packages = [] + # Unpack all of the entity packages in the form of `*-entity.tar.gz`. Also + # unpack the entities in lexicographical order so it is potentially easier + # to read logs. + for filename in sorted(os.listdir(src_entities_dir_path)): + if filename.endswith(ENTITY_FILENAME_SUFFIX): + entity_owner = filename[:-len(ENTITY_FILENAME_SUFFIX)] + + unpacked_entity_dir_path = os.path.join( + dest_entities_dir_path, + entity_owner + ) + # Create the new entity directory + logger.info('Unpack package for entity owner "%s"' % entity_owner) + os.mkdir(unpacked_entity_dir_path) + + package = tarfile.open(os.path.join(src_entities_dir_path, + filename)) + package.extractall(unpacked_entity_dir_path) + + if not validate_entity_package(unpacked_entity_dir_path): + invalid_entity_packages.append(entity_owner) + else: + logger.info('Entity owned by "%s" is valid' % entity_owner) + + if len(invalid_entity_packages) > 0: + for entity_owner in invalid_entity_packages: + logger.error('Invalid Entity for %s' % entity_owner) + raise InvalidEntitiesDetected() + + +def validate_entity_package(package_path): + is_valid = True + # Validate that the expected directory structure exists + for expected_file_name in EXPECTED_FILES: + expected_file_path = os.path.join(package_path, expected_file_name) + + if not os.path.isfile(expected_file_path): + logger.warning('Expected file "%s" missing' % expected_file_path) + is_valid = False + + if not is_valid: + return is_valid + + # Ensure that the node is properly loaded into the + # FIXME we should do this check using something written with oasis-core as a + # library. This is quick and dirty. + entity_genesis_path = os.path.join( + package_path, 'entity/entity_genesis.json') + node_genesis_path = os.path.join(package_path, 'node/node_genesis.json') + + with open(entity_genesis_path) as entity_genesis_file: + entity_genesis = json.load(entity_genesis_file) + + with open(node_genesis_path) as node_genesis_file: + node_genesis = json.load(node_genesis_file) + + entity_descriptor = cbor.loads(base64.b64decode( + entity_genesis['untrusted_raw_value'])) + node_descriptor = cbor.loads( + base64.b64decode(node_genesis['untrusted_raw_value'])) + + entity_nodes = entity_descriptor['nodes'] or [] + + if not node_descriptor['id'] in entity_nodes: + logger.warning('Expected node to be added to entity') + is_valid = False + + return is_valid + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + src_entities_dir_path = os.path.abspath(sys.argv[1]) + dest_entities_dir_path = os.path.abspath(sys.argv[2]) + + logger.info('Unpacking to %s' % dest_entities_dir_path) + try: + unpack_entities(src_entities_dir_path, dest_entities_dir_path) + except InvalidEntitiesDetected: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/python/validate_pull_request.py b/.github/scripts/python/validate_pull_request.py new file mode 100644 index 0000000..7dedf1b --- /dev/null +++ b/.github/scripts/python/validate_pull_request.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import os +import sys +from github import Github + + +class InvalidEntityPR(Exception): + pass + + +def validate_entity_pull_request(gh, repo, pr_number): + """Validate if this pull request is a valid entity pull request.""" + repo = gh.get_repo(repo) + pr = repo.get_pull(pr_number) + pr_creator = pr.user.login + + is_valid = False + + expected_filename = "entities/%s-entity.tar.gz" % pr_creator + + for changed_file in pr.get_files(): + if changed_file.filename == expected_filename: + is_valid = True + + if not is_valid: + raise InvalidEntityPR( + 'The entity file is expected to be named %s. Please remediate.' % expected_filename + ) + + +def main(): + token = os.environ.get('GITHUB_TOKEN') + github_ref = os.environ.get('GITHUB_REF', '') + if not token: + print('No github token specified') + sys.exit(1) + if not github_ref: + print('No github_ref specified') + sys.exit(1) + + try: + pr_number = int(github_ref.split('/')[2]) + except TypeError: + print("This might not be a PR or something is wrong") + sys.exit(1) + gh = Github(token) + + print("Validating PR #%d" % pr_number) + + try: + validate_entity_pull_request( + gh, + 'oasislabs/the-quest-entities', + pr_number + ) + except InvalidEntityPR as e: + print(e) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml new file mode 100644 index 0000000..5ca2d60 --- /dev/null +++ b/.github/workflows/test-scripts.yml @@ -0,0 +1,19 @@ +name: Check validation scripts + +on: [push] + +jobs: + build: + name: Test that the validation scripts work as expected + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + + - name: Setup Python 3 + uses: actions/setup-python@v1 + with: + python-version: '3.x' + + - run: pip3 install -r .github/scripts/python/requirements.txt + + - run: python3 .github/scripts/python/test_unpack_entities.py diff --git a/.github/workflows/validate-pull-request-files.yml b/.github/workflows/validate-pull-request-files.yml new file mode 100644 index 0000000..c6f0ef2 --- /dev/null +++ b/.github/workflows/validate-pull-request-files.yml @@ -0,0 +1,24 @@ +name: Validate Entity Package PR + +on: [pull_request] + +jobs: + validate_package_file_name: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + + - name: Setup Python 3 + uses: actions/setup-python@v1 + with: + python-version: '3.x' + + - run: pip3 install -r .github/scripts/python/requirements.txt + + - name: Validate the entity package name + run: python3 .github/scripts/python/validate_pull_request.py + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Validate entity packages + run: mkdir /tmp/unpack && python3 .github/scripts/python/unpack_entities.py ./entities /tmp/unpack