diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 5c85314b..3fb67083 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -54,6 +54,17 @@ jobs: - name: Generate GitHub GraphQL schema module run: sgqlc-codegen schema github_schema.json github_schema.py + - name: Fetch ZenHub GraphQL schema + run: | + python3 -m sgqlc.introspection \ + --exclude-description \ + -H "Authorization: Bearer ${{ secrets.ZENHUB_TOKEN }}" \ + https://api.zenhub.com/public/graphql \ + zenhub_schema.json + + - name: Generate ZenHub GraphQL schema module + run: sgqlc-codegen schema zenhub_schema.json zenhub_schema.py + - name: Render ECC core DAG run: python3 ./zcash-issue-dag.py env: diff --git a/.gitignore b/.gitignore index bee174b2..6e5a7d88 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,10 @@ public/*.svg github_schema.json github_schema.py -zcash_developer_tools.egg-info +zenhub_schema.json +zenhub_schema.py GITHUB_TOKEN ZENHUB_TOKEN + +zcash_developer_tools.egg-info diff --git a/gen-schema.sh b/gen-schema.sh index 2fb2db81..39fe5f2b 100755 --- a/gen-schema.sh +++ b/gen-schema.sh @@ -9,3 +9,11 @@ uv run python3 -m sgqlc.introspection \ github_schema.json uv run sgqlc-codegen schema github_schema.json github_schema.py + +uv run python3 -m sgqlc.introspection \ + --exclude-description \ + -H "Authorization: bearer $(cat ZENHUB_TOKEN)" \ + https://api.zenhub.com/public/graphql \ + zenhub_schema.json + +uv run sgqlc-codegen schema zenhub_schema.json zenhub_schema.py diff --git a/pyproject.toml b/pyproject.toml index e3c380fc..31b15f50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,6 @@ name = "zcash-developer-tools" version = "0.1.0" dependencies = [ - "drest", "networkx", "pygraphviz", "sgqlc", diff --git a/requirements.txt b/requirements.txt index 2f836ae1..ddc9c0c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -drest networkx pygraphviz sgqlc diff --git a/uv.lock b/uv.lock index 3b40363d..19227dff 100644 --- a/uv.lock +++ b/uv.lock @@ -1,15 +1,6 @@ version = 1 requires-python = ">=3.8" -[[package]] -name = "drest" -version = "0.9.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httplib2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/86c8e61ab7573c2762b39d5b07de3997fbe0dbf56b41b3047165612c1b11/drest-0.9.12.tar.gz", hash = "sha256:5d88a53d19b99844baabebc080452dd0d83030e812a8994719e9723d5356542e", size = 29734 } - [[package]] name = "graphql-core" version = "3.2.5" @@ -22,18 +13,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e3/dc/078bd6b304de790618ebb95e2aedaadb78f4527ac43a9ad8815f006636b6/graphql_core-3.2.5-py3-none-any.whl", hash = "sha256:2f150d5096448aa4f8ab26268567bbfeef823769893b39c1a2e1409590939c8a", size = 203189 }, ] -[[package]] -name = "httplib2" -version = "0.22.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyparsing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854 }, -] - [[package]] name = "networkx" version = "3.1" @@ -49,15 +28,6 @@ version = "1.11" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/19/db/cc09516573e79a35ac73f437bdcf27893939923d1d06b439897ffc7f3217/pygraphviz-1.11.zip", hash = "sha256:a97eb5ced266f45053ebb1f2c6c6d29091690503e3a5c14be7f908b37b06f2d4", size = 120803 } -[[package]] -name = "pyparsing" -version = "3.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/08/13f3bce01b2061f2bbd582c9df82723de943784cf719a35ac886c652043a/pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032", size = 900231 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 }, -] - [[package]] name = "sgqlc" version = "16.4" @@ -90,7 +60,6 @@ name = "zcash-developer-tools" version = "0.1.0" source = { virtual = "." } dependencies = [ - { name = "drest" }, { name = "networkx" }, { name = "pygraphviz" }, { name = "sgqlc" }, @@ -99,7 +68,6 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "drest" }, { name = "networkx" }, { name = "pygraphviz" }, { name = "sgqlc" }, diff --git a/zcash-issue-dag.py b/zcash-issue-dag.py index 5c323b7a..34c4927f 100755 --- a/zcash-issue-dag.py +++ b/zcash-issue-dag.py @@ -4,7 +4,6 @@ # Author: jack@electriccoin.co # Last updated: 2021-05-07 -import drest import networkx as nx from str2bool import str2bool as strtobool @@ -16,6 +15,7 @@ from urllib.parse import urlparse from sgqlc.endpoint.http import HTTPEndpoint from sgqlc.operation import Operation from github_schema import github_schema as schema +from zenhub_schema import zenhub_schema GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN') ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN') @@ -110,6 +110,20 @@ REPO_SETS = { REPOS = REPO_SETS[DAG_VIEW] +WORKSPACE_SETS = { + # ecc-core + '5dc1fd615862290001229f21': CORE_REPOS.keys(), + # ecc-wallet + '5db8aa0244512d0001e0968e': WALLET_REPOS.keys(), + # zf + '5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(), +} + +WORKSPACES = { + workspace_id: [repo_id for repo_id in repos if repo_id in REPOS] + for (workspace_id, repos) in WORKSPACE_SETS.items() +} + SUPPORTED_CATEGORIES = set(['releases', 'targets']) def cats(s): return set([x.strip() for x in s.split(',')]) - set(['']) @@ -245,88 +259,168 @@ def download_issues(endpoint, nodes): return ret +def fetch_workspace_graph(op, workspace_id, repos, cursor): + dependencies = op.workspace(id=workspace_id).issue_dependencies( + # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here, + # not the GitHub repo IDs (which the previous REST API used). + # repository_ids=repos, + first=100, + after=cursor, + ) + dependencies.nodes.id() + dependencies.nodes.blocked_issue.number() + dependencies.nodes.blocked_issue.repository.gh_id() + dependencies.nodes.blocking_issue.number() + dependencies.nodes.blocking_issue.repository.gh_id() + dependencies.page_info.has_next_page() + dependencies.page_info.end_cursor() -class ZHDepsResourceHandler(drest.resource.ResourceHandler): - def get(self, repo_id): - path = '/repositories/%d/dependencies' % repo_id +def get_dependency_graph(endpoint, workspace_id, repos): + edges = [] + cursor = None - try: - response = self.api.make_request('GET', path, {}) - except drest.exc.dRestRequestError as e: - msg = "%s (repo_id: %s)" % (e.msg, repo_id) - raise drest.exc.dRestRequestError(msg, e.response) + while True: + op = Operation(zenhub_schema.Query) + fetch_workspace_graph(op, workspace_id, repos, cursor) - def issue(json): - return (int(json['repo_id']), int(json['issue_number'])) + d = endpoint(op) + data = (op + d) - return nx.DiGraph([ - (issue(edge['blocking']), issue(edge['blocked'])) - for edge in response.data['dependencies'] - ]) + dependencies = data.workspace.issue_dependencies + edges += [ + ( + (node.blocking_issue.repository.gh_id, node.blocking_issue.number), + (node.blocked_issue.repository.gh_id, node.blocked_issue.number), + ) for node in dependencies.nodes + ] -class ZHEpicsResourceHandler(drest.resource.ResourceHandler): - def get(self, repo_id): - path = '/repositories/%d/epics' % repo_id + if dependencies.page_info.has_next_page: + cursor = dependencies.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break - try: - response = self.api.make_request('GET', path, {}) - except drest.exc.dRestRequestError as e: - msg = "%s (repo_id: %s)" % (e.msg, repo_id) - raise drest.exc.dRestRequestError(msg, e.response) + return nx.DiGraph(edges) - return [i['issue_number'] for i in response.data['epic_issues']] +def fetch_epics(op, workspace_id, repos, cursor): + epics = op.workspace(id=workspace_id).epics( + # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here, + # not the GitHub repo IDs (which the previous REST API used). + # repository_ids=repos, + first=100, + after=cursor, + ) + epics.nodes.id() + epics.nodes.issue.number() + epics.nodes.issue.repository.gh_id() + epics.page_info.has_next_page() + epics.page_info.end_cursor() -class ZHEpicsIssuesResourceHandler(drest.resource.ResourceHandler): - def get(self, repo_id, epic_id): - path = '/repositories/%d/epics/%d' % (repo_id, epic_id) +def get_epics(endpoint, workspace_id, repos): + epics = [] + cursor = None - try: - response = self.api.make_request('GET', path, {}) - except drest.exc.dRestRequestError as e: - msg = "%s (repo_id: %s)" % (e.msg, repo_id) - raise drest.exc.dRestRequestError(msg, e.response) + while True: + op = Operation(zenhub_schema.Query) + fetch_epics(op, workspace_id, repos, cursor) - return [(i['repo_id'], i['issue_number']) for i in response.data['issues']] + d = endpoint(op) + data = (op + d) -class ZenHubAPI(drest.api.API): - class Meta: - baseurl = 'https://api.zenhub.com/p1' - extra_headers = { - 'X-Authentication-Token': ZENHUB_TOKEN, - 'Content-Type': 'application/json', - } + epics_page = data.workspace.epics + epics += [ + (node.id, (node.issue.repository.gh_id, node.issue.number)) + for node in epics_page.nodes + ] - def __init__(self, *args, **kw): - super(ZenHubAPI, self).__init__(*args, **kw) - self.add_resource('dependencies', ZHDepsResourceHandler) - self.add_resource('epics', ZHEpicsResourceHandler) - self.add_resource('epics_issues', ZHEpicsIssuesResourceHandler) + if epics_page.page_info.has_next_page: + cursor = epics_page.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break - def auth(self, *args, **kw): - pass + return epics +def fetch_epic_issues(op, workspace_id, epic_id, cursor): + epic = op.workspace(id=workspace_id).epics(ids=[epic_id]) + child_issues = epic.nodes.child_issues( + first=100, + after=cursor, + ) + child_issues.nodes.number() + child_issues.nodes.repository.gh_id() + child_issues.page_info.has_next_page() + child_issues.page_info.end_cursor() + +def get_epic_issues(endpoint, workspace_id, epic_id): + epic_issues = [] + cursor = None + + while True: + op = Operation(zenhub_schema.Query) + fetch_epic_issues(op, workspace_id, epic_id, cursor) + + d = endpoint(op) + data = (op + d) + + epic = data.workspace.epics.nodes[0] + epic_issues += [ + (node.repository.gh_id, node.number) + for node in epic.child_issues.nodes + ] + + if epic.child_issues.page_info.has_next_page: + cursor = epic.child_issues.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break + + return epic_issues def main(): gapi = HTTPEndpoint( 'https://api.github.com/graphql', {'Authorization': 'bearer %s' % GITHUB_TOKEN}, ) - zapi = ZenHubAPI() + zapi = HTTPEndpoint( + 'https://api.zenhub.com/public/graphql', + {'Authorization': 'Bearer %s' % ZENHUB_TOKEN}, + ) - # Build the full dependency graph from ZenHub's per-repo APIs. - dg = nx.compose_all([zapi.dependencies.get(x) for x in REPOS]) + # Build the full dependency graph from ZenHub's per-workspace API. + print('Fetching graph') + dg = nx.compose_all([ + get_dependency_graph(zapi, workspace_id, repos) + for (workspace_id, repos) in WORKSPACES.items() + if len(repos) > 0 + ]) + + print('Rendering DAG') if SHOW_EPICS: epics_issues = [] - for repo_id in REPOS: - for epic_id in zapi.epics.get(repo_id): - epics_issues.append((repo_id, epic_id)) + for (workspace_id, repos) in WORKSPACES.items(): + if len(repos) > 0: + epics_issues += get_epics(zapi, workspace_id, repos) + epics_issues = set(epics_issues) - epics_mapping = download_issues(gapi, epics_issues) + epics_mapping = download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues]) epics_mapping = {k: v for (k, v) in epics_mapping.items() if v.state != 'closed'} issues_by_epic = {} for (i, ((repo_id, epic_id), epic)) in enumerate(epics_mapping.items()): - issues = set(zapi.epics_issues.get(repo_id, epic_id)) + workspace_id = [ + workspace_id + for (workspace_id, repos) in WORKSPACES.items() + if repo_id in repos + ][0] + epic_id = [ + id for (id, gh_ref) in epics_issues + if gh_ref == (repo_id, epic_id) + ][0] + issues = set(get_epic_issues(zapi, workspace_id, epic_id)) issues_by_epic[epic] = issues for i in issues: # zapi.dependencies only returns nodes that have some connection,