From e08529c8857d363a5a762d834cdc903e44830dd2 Mon Sep 17 00:00:00 2001 From: "Reuven V. Gonzales" Date: Mon, 30 Dec 2019 17:24:25 -0800 Subject: [PATCH] Add PR validation (#1) --- .../bad1_entity_packages/bad1-entity.tar.gz | Bin 0 -> 145 bytes .../bad2_entity_packages/bad2-entity.tar.gz | Bin 0 -> 1247 bytes .../good_entity_packages/good-entity.tar.gz | Bin 0 -> 1292 bytes .github/scripts/python/requirements.txt | 2 + .../scripts/python/test_unpack_entities.py | 38 ++++++ .github/scripts/python/unpack_entities.py | 113 ++++++++++++++++++ .../scripts/python/validate_pull_request.py | 62 ++++++++++ .github/workflows/test-scripts.yml | 19 +++ .../workflows/validate-pull-request-files.yml | 24 ++++ 9 files changed, 258 insertions(+) create mode 100644 .github/scripts/python/fixtures/bad1_entity_packages/bad1-entity.tar.gz create mode 100644 .github/scripts/python/fixtures/bad2_entity_packages/bad2-entity.tar.gz create mode 100644 .github/scripts/python/fixtures/good_entity_packages/good-entity.tar.gz create mode 100644 .github/scripts/python/requirements.txt create mode 100644 .github/scripts/python/test_unpack_entities.py create mode 100755 .github/scripts/python/unpack_entities.py create mode 100644 .github/scripts/python/validate_pull_request.py create mode 100644 .github/workflows/test-scripts.yml create mode 100644 .github/workflows/validate-pull-request-files.yml diff --git a/.github/scripts/python/fixtures/bad1_entity_packages/bad1-entity.tar.gz b/.github/scripts/python/fixtures/bad1_entity_packages/bad1-entity.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9950de8e8887a8e1340a5276ab72fdea66556b7f GIT binary patch literal 145 zcmb2|=3o%b<%(lqetX`L>yUv!>%(=nN9Q=G`O2IZ-r%@U!(?Se&Ekc6-LsVr{m+$~ zeCCm_WYNv(?k~-jtP*S5^C{z--`m@Fb|{>St~KVq)orz&{eEX&WR~G`q2*DjUEd$> r+7y00DC%ea^b_j;UF_~Z`>3KBckJ%^-3%yT-*=XY(u)NcG#D5Fi8VfN literal 0 HcmV?d00001 diff --git a/.github/scripts/python/fixtures/bad2_entity_packages/bad2-entity.tar.gz b/.github/scripts/python/fixtures/bad2_entity_packages/bad2-entity.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..cc6220454d8654ad174327c0bc354510d93977fa GIT binary patch literal 1247 zcmV<51R(n#iwFQTa0*@k1MQaCj+;mnhVv*b-*z+x%+f^4MS~XO^e-_0&Esz zi1O}(yE{?RGwCFwB%{pmNv5DEOmUX~oG`DGx|_cGnIH&45yZ{INRkgDg5i(j6F2CL z9OkI&of`zjNPPC{hWM$wdZMXnw;Z-oy5}%|VcoDSie9+?^c^3^--Phd|Bu5ThpKqV z2LF)39^{j0{?`8}jz9B1f#B$?8|24t<*)qD{r_%~1d}%ttETmQnw@&ClqAn-3TM5% zEe(GYRg1)1^mcA2-pS7QYVmgRdXg7GSWVvK4TP^JE`&vU{}@5{RTAf6aR0O*a$yy| z|F^$!*2c{^JDoEdNP@B*y&z6}+1?d0h@h7?QSh+xxQ%n{W_> z0$Ut+d>ntfFdU(m(do|KU?Btdo^Uw_C=I<_I|d?*d<^DZW~MeqoOgoR)8Oq@NUn0> z)=fFQ^4;Xvcrfwr)3Ez~Mf&8ACed*n(3WeRS)fY8k89c3k(R|y>JJD9TB+rA zPL~-@0I0!%i@fwOz8@+_{ZPwv5b`Z@QFN}8`=JudRYdOh?l3o;Kh5cnb24LGxE^Rs zyfzsi>ug5vhnm-~tZTh9T1y8y6Z@xqdQdj1#4vIEbS^VYyJi8{0-ju%N!*p41n@Oe z=Pa$Mc6PVP(#Dx3W_64jO>@!Zv;&GkU1aaXmEJiLstRNZ8%x4Y(T*rTtr8tbTJ z3*rFEu#DquCvhyPte5IV0Sd3o=e0WZ=f=$ZET&?2F~p1JNSs1S)}0ul_EfL#F8ZZr zn2m9wg*H<8&^{W5j3H@)GcDL*AL2_nI4M*O@G0}@wA)nLf=&&A?g9oDlAyUBMnFe^ zAA6pMck*Rq5L4n*U6vgyV5^fb-=0sXIG<6hlXX5yp!1A`1a2(d%*1^Gc8)#7Jcff8 z`=f79Oz^xvQnr1#igc1pYAA&`Zxl|^J2sW#nr-)f5HQE1=u|omrMZNf%dAg#x44C2 ztN6g;+ciClmq`}{X#{RhMz)e<%mDF!vp*Uc85tQF85tQF85tQF85tQF85#Lq`4hQc JGkO3h005VEVv+y= literal 0 HcmV?d00001 diff --git a/.github/scripts/python/fixtures/good_entity_packages/good-entity.tar.gz b/.github/scripts/python/fixtures/good_entity_packages/good-entity.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..164e7020db101767e683aab67acb390eecb8166b GIT binary patch literal 1292 zcmV+n1@rnJiwFQgc?w}A|OrX#q)4i5kc;o%>)n_1VlkV zX!_k3XQrFByE|<+Z8uH%C3^7T;_+Pm&oMlllXEwH^*uolgd&KWM@y2A7Qtpu_L&SFfe_fM8|2%&@^^mm{J)(fyUCl0b*A;hG(YuRDM`Ll7tVTl zTN=S8suqdA=zyZh&dMP62g@BZ(2{AzsE|Mzit2&<&} z#yar1{wHw!ll~_OiX8j@NAPx1AI{|vgrN~juf1=)tPTf3D6qxR<>UCvh2fa_Ii0$; z0xbrzb;36gKbup8+Q`P-S5+NdHd-Yj(9TGcXV>S2RD%GBvFliu%Exz+YY~WNYlMxj zk1j@JE>G7!re(oYy^79NF1&MH4(`339BV&Ig8MY=zFxgPc+jM=)VT*c+>+`@@udx_ zNbyW@D^|H}oLtx{jmzSO7NO8w?AP1PQEOI4<%(vrmKO6+B@taI_9t{<(e;U`dj7(c z47j_Sy!$z%>?*$Bcl!Iv`H#S^^B+cy^WP8QG5^=M@1*@#KBNB_ic+8CpCD0c)c+sB zzv=%;c+&qbAMoHlU>9wXKi%gPcC@cr;^6g$f9oI?O{1|JNBfd$k}gfzJz!9ZmsDj^RY`7T;kXh7pQ;Gj=ptW&-Olso`?MbaV;zHEIxI`MHb-eVSjK- z2Q#~!kma7We%J}73k;-X<4{ZzehOGN-eK*x(qo1pausn z^3uon-m~TN<6fqNkZ+NTqH~?xgHC2uMD8EFVQn~mUelk}WN2Kt9%wLLn-Iu4lhY3Y zmh~%(Xk5W)Egk4E4o(m0UfHM;gmL_QEJN6?8349`Cs$??cV#Dm*&3b?46UhlKHubN z?cj-79iv*)JajqjfMQS=**|fm-$X)Hfm~r?NpKbIi1O1aF=9h1qu#ctqUYLG50tB4 zqB@5lnse|PkCWPps>7NZl0oRSJQ(8AIuNRc2wF~Hm`VVu!NXQ&WKayBPr_E7(U12c zF_1Zfk%U@uHExd>jqYpO5lMropV?{XL-jJPnoLaXTt406$rJ|(2g(H+2j-kC^}SQ; zS(Rmx6KI^S)5{=ybDkQWpgqkX2#OX%~>F1onsF*kIle~GtXE3^ZIz^ zQMP@!igc1p&Y2X>c%yKN-m$54rrCBM>~`>Y6rD<+WfCi)<}&Y7_jYDs*ec#Lv+bJ3 z<7LwAc4-7|Pe#6yWXu5Z&olFkjEszojEszojEszojEszojEs!@3j7Hgg^W7@C;$NR CV3275 literal 0 HcmV?d00001 diff --git a/.github/scripts/python/requirements.txt b/.github/scripts/python/requirements.txt new file mode 100644 index 0000000..c2886b6 --- /dev/null +++ b/.github/scripts/python/requirements.txt @@ -0,0 +1,2 @@ +cbor==1.0.0 +PyGithub==1.45 diff --git a/.github/scripts/python/test_unpack_entities.py b/.github/scripts/python/test_unpack_entities.py new file mode 100644 index 0000000..6b8c5c4 --- /dev/null +++ b/.github/scripts/python/test_unpack_entities.py @@ -0,0 +1,38 @@ +from unpack_entities import unpack_entities, InvalidEntitiesDetected +import unittest +import os +import shutil +import tempfile + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestUnpack(unittest.TestCase): + def setUp(self): + self.test_temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_temp_dir) + + def fixture_dir(self, name): + return os.path.join( + CURRENT_DIR, + 'fixtures', '%s_entity_packages' % name + ) + + def test_entity_package_missing_files(self): + """Tests when the entity package is missing files""" + with self.assertRaises(InvalidEntitiesDetected): + unpack_entities(self.fixture_dir('bad1'), self.test_temp_dir) + + def test_entity_package_node_not_registered(self): + """Tests when the entity package node is not registered properly""" + with self.assertRaises(InvalidEntitiesDetected): + unpack_entities(self.fixture_dir('bad2'), self.test_temp_dir) + + def test_succeeds(self): + unpack_entities(self.fixture_dir('good'), self.test_temp_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/python/unpack_entities.py b/.github/scripts/python/unpack_entities.py new file mode 100755 index 0000000..99aa079 --- /dev/null +++ b/.github/scripts/python/unpack_entities.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import tempfile +import tarfile +import logging +import base64 +import cbor + +ENTITY_FILENAME_SUFFIX = '-entity.tar.gz' + +EXPECTED_FILES = [ + 'entity/entity.json', + 'entity/entity_genesis.json', + 'node/node_genesis.json', +] + +logger = logging.getLogger('unpack_entities') + + +# May want a more granular error in the future +class InvalidEntitiesDetected(Exception): + pass + + +def unpack_entities(src_entities_dir_path, dest_entities_dir_path): + invalid_entity_packages = [] + # Unpack all of the entity packages in the form of `*-entity.tar.gz`. Also + # unpack the entities in lexicographical order so it is potentially easier + # to read logs. + for filename in sorted(os.listdir(src_entities_dir_path)): + if filename.endswith(ENTITY_FILENAME_SUFFIX): + entity_owner = filename[:-len(ENTITY_FILENAME_SUFFIX)] + + unpacked_entity_dir_path = os.path.join( + dest_entities_dir_path, + entity_owner + ) + # Create the new entity directory + logger.info('Unpack package for entity owner "%s"' % entity_owner) + os.mkdir(unpacked_entity_dir_path) + + package = tarfile.open(os.path.join(src_entities_dir_path, + filename)) + package.extractall(unpacked_entity_dir_path) + + if not validate_entity_package(unpacked_entity_dir_path): + invalid_entity_packages.append(entity_owner) + else: + logger.info('Entity owned by "%s" is valid' % entity_owner) + + if len(invalid_entity_packages) > 0: + for entity_owner in invalid_entity_packages: + logger.error('Invalid Entity for %s' % entity_owner) + raise InvalidEntitiesDetected() + + +def validate_entity_package(package_path): + is_valid = True + # Validate that the expected directory structure exists + for expected_file_name in EXPECTED_FILES: + expected_file_path = os.path.join(package_path, expected_file_name) + + if not os.path.isfile(expected_file_path): + logger.warning('Expected file "%s" missing' % expected_file_path) + is_valid = False + + if not is_valid: + return is_valid + + # Ensure that the node is properly loaded into the + # FIXME we should do this check using something written with oasis-core as a + # library. This is quick and dirty. + entity_genesis_path = os.path.join( + package_path, 'entity/entity_genesis.json') + node_genesis_path = os.path.join(package_path, 'node/node_genesis.json') + + with open(entity_genesis_path) as entity_genesis_file: + entity_genesis = json.load(entity_genesis_file) + + with open(node_genesis_path) as node_genesis_file: + node_genesis = json.load(node_genesis_file) + + entity_descriptor = cbor.loads(base64.b64decode( + entity_genesis['untrusted_raw_value'])) + node_descriptor = cbor.loads( + base64.b64decode(node_genesis['untrusted_raw_value'])) + + entity_nodes = entity_descriptor['nodes'] or [] + + if not node_descriptor['id'] in entity_nodes: + logger.warning('Expected node to be added to entity') + is_valid = False + + return is_valid + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + src_entities_dir_path = os.path.abspath(sys.argv[1]) + dest_entities_dir_path = os.path.abspath(sys.argv[2]) + + logger.info('Unpacking to %s' % dest_entities_dir_path) + try: + unpack_entities(src_entities_dir_path, dest_entities_dir_path) + except InvalidEntitiesDetected: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/python/validate_pull_request.py b/.github/scripts/python/validate_pull_request.py new file mode 100644 index 0000000..7dedf1b --- /dev/null +++ b/.github/scripts/python/validate_pull_request.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import os +import sys +from github import Github + + +class InvalidEntityPR(Exception): + pass + + +def validate_entity_pull_request(gh, repo, pr_number): + """Validate if this pull request is a valid entity pull request.""" + repo = gh.get_repo(repo) + pr = repo.get_pull(pr_number) + pr_creator = pr.user.login + + is_valid = False + + expected_filename = "entities/%s-entity.tar.gz" % pr_creator + + for changed_file in pr.get_files(): + if changed_file.filename == expected_filename: + is_valid = True + + if not is_valid: + raise InvalidEntityPR( + 'The entity file is expected to be named %s. Please remediate.' % expected_filename + ) + + +def main(): + token = os.environ.get('GITHUB_TOKEN') + github_ref = os.environ.get('GITHUB_REF', '') + if not token: + print('No github token specified') + sys.exit(1) + if not github_ref: + print('No github_ref specified') + sys.exit(1) + + try: + pr_number = int(github_ref.split('/')[2]) + except TypeError: + print("This might not be a PR or something is wrong") + sys.exit(1) + gh = Github(token) + + print("Validating PR #%d" % pr_number) + + try: + validate_entity_pull_request( + gh, + 'oasislabs/the-quest-entities', + pr_number + ) + except InvalidEntityPR as e: + print(e) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml new file mode 100644 index 0000000..5ca2d60 --- /dev/null +++ b/.github/workflows/test-scripts.yml @@ -0,0 +1,19 @@ +name: Check validation scripts + +on: [push] + +jobs: + build: + name: Test that the validation scripts work as expected + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + + - name: Setup Python 3 + uses: actions/setup-python@v1 + with: + python-version: '3.x' + + - run: pip3 install -r .github/scripts/python/requirements.txt + + - run: python3 .github/scripts/python/test_unpack_entities.py diff --git a/.github/workflows/validate-pull-request-files.yml b/.github/workflows/validate-pull-request-files.yml new file mode 100644 index 0000000..c6f0ef2 --- /dev/null +++ b/.github/workflows/validate-pull-request-files.yml @@ -0,0 +1,24 @@ +name: Validate Entity Package PR + +on: [pull_request] + +jobs: + validate_package_file_name: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + + - name: Setup Python 3 + uses: actions/setup-python@v1 + with: + python-version: '3.x' + + - run: pip3 install -r .github/scripts/python/requirements.txt + + - name: Validate the entity package name + run: python3 .github/scripts/python/validate_pull_request.py + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Validate entity packages + run: mkdir /tmp/unpack && python3 .github/scripts/python/unpack_entities.py ./entities /tmp/unpack