switch to ZenHub GraphQL API

Co-authored-by: Jack Grigg <jack@electriccoin.co>

.github/workflows/gh-pages.yml

@@ -54,6 +54,17 @@ jobs:
       - name: Generate GitHub GraphQL schema module
         run: sgqlc-codegen schema github_schema.json github_schema.py
 
+      - name: Fetch ZenHub GraphQL schema
+        run: |
+          python3 -m sgqlc.introspection \
+            --exclude-description \
+            -H "Authorization: Bearer ${{ secrets.ZENHUB_TOKEN }}" \
+            https://api.zenhub.com/public/graphql \
+            zenhub_schema.json
+
+      - name: Generate ZenHub GraphQL schema module
+        run: sgqlc-codegen schema zenhub_schema.json zenhub_schema.py
+
       - name: Render ECC core DAG
         run: python3 ./zcash-issue-dag.py
         env:

.gitignore

@@ -5,7 +5,10 @@ public/*.svg
 github_schema.json
 github_schema.py
 
-zcash_developer_tools.egg-info
+zenhub_schema.json
+zenhub_schema.py
 
 GITHUB_TOKEN
 ZENHUB_TOKEN
+
+zcash_developer_tools.egg-info

gen-schema.sh

@@ -9,3 +9,11 @@ uv run python3 -m sgqlc.introspection \
   github_schema.json
 
 uv run sgqlc-codegen schema github_schema.json github_schema.py
+
+uv run python3 -m sgqlc.introspection \
+  --exclude-description \
+  -H "Authorization: bearer $(cat ZENHUB_TOKEN)" \
+  https://api.zenhub.com/public/graphql \
+  zenhub_schema.json
+
+uv run sgqlc-codegen schema zenhub_schema.json zenhub_schema.py

pyproject.toml

@@ -2,7 +2,6 @@
 name = "zcash-developer-tools"
 version = "0.1.0"
 dependencies = [
-  "drest",
   "networkx",
   "pygraphviz",
   "sgqlc",

requirements.txt

@@ -1,4 +1,3 @@
-drest
 networkx
 pygraphviz
 sgqlc

uv.lock

@@ -1,15 +1,6 @@
 version = 1
 requires-python = ">=3.8"
 
-[[package]]
-name = "drest"
-version = "0.9.12"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "httplib2" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/86c8e61ab7573c2762b39d5b07de3997fbe0dbf56b41b3047165612c1b11/drest-0.9.12.tar.gz", hash = "sha256:5d88a53d19b99844baabebc080452dd0d83030e812a8994719e9723d5356542e", size = 29734 }
-
 [[package]]
 name = "graphql-core"
 version = "3.2.5"
@@ -22,18 +13,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e3/dc/078bd6b304de790618ebb95e2aedaadb78f4527ac43a9ad8815f006636b6/graphql_core-3.2.5-py3-none-any.whl", hash = "sha256:2f150d5096448aa4f8ab26268567bbfeef823769893b39c1a2e1409590939c8a", size = 203189 },
 ]
 
-[[package]]
-name = "httplib2"
-version = "0.22.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "pyparsing" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854 },
-]
-
 [[package]]
 name = "networkx"
 version = "3.1"
@@ -49,15 +28,6 @@ version = "1.11"
 source = { registry = "https://pypi.org/simple" }
 sdist = { url = "https://files.pythonhosted.org/packages/19/db/cc09516573e79a35ac73f437bdcf27893939923d1d06b439897ffc7f3217/pygraphviz-1.11.zip", hash = "sha256:a97eb5ced266f45053ebb1f2c6c6d29091690503e3a5c14be7f908b37b06f2d4", size = 120803 }
 
-[[package]]
-name = "pyparsing"
-version = "3.1.4"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/83/08/13f3bce01b2061f2bbd582c9df82723de943784cf719a35ac886c652043a/pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032", size = 900231 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 },
-]
-
 [[package]]
 name = "sgqlc"
 version = "16.4"
@@ -90,7 +60,6 @@ name = "zcash-developer-tools"
 version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
-    { name = "drest" },
     { name = "networkx" },
     { name = "pygraphviz" },
     { name = "sgqlc" },
@@ -99,7 +68,6 @@ dependencies = [
 
 [package.metadata]
 requires-dist = [
-    { name = "drest" },
     { name = "networkx" },
     { name = "pygraphviz" },
     { name = "sgqlc" },

zcash-issue-dag.py

@@ -4,7 +4,6 @@
 # Author: jack@electriccoin.co
 # Last updated: 2021-05-07
 
-import drest
 import networkx as nx
 
 from str2bool import str2bool as strtobool
@@ -16,6 +15,7 @@ from urllib.parse import urlparse
 from sgqlc.endpoint.http import HTTPEndpoint
 from sgqlc.operation import Operation
 from github_schema import github_schema as schema
+from zenhub_schema import zenhub_schema
 
 GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
 ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')
@@ -110,6 +110,20 @@ REPO_SETS = {
 
 REPOS = REPO_SETS[DAG_VIEW]
 
+WORKSPACE_SETS = {
+    # ecc-core
+    '5dc1fd615862290001229f21': CORE_REPOS.keys(),
+    # ecc-wallet
+    '5db8aa0244512d0001e0968e': WALLET_REPOS.keys(),
+    # zf
+    '5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(),
+}
+
+WORKSPACES = {
+    workspace_id: [repo_id for repo_id in repos if repo_id in REPOS]
+    for (workspace_id, repos) in WORKSPACE_SETS.items()
+}
+
 SUPPORTED_CATEGORIES = set(['releases', 'targets'])
 def cats(s):
     return set([x.strip() for x in s.split(',')]) - set([''])
@@ -245,88 +259,168 @@ def download_issues(endpoint, nodes):
 
     return ret
 
+def fetch_workspace_graph(op, workspace_id, repos, cursor):
+    dependencies = op.workspace(id=workspace_id).issue_dependencies(
+        # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
+        # not the GitHub repo IDs (which the previous REST API used).
+        # repository_ids=repos,
+        first=100,
+        after=cursor,
+    )
+    dependencies.nodes.id()
+    dependencies.nodes.blocked_issue.number()
+    dependencies.nodes.blocked_issue.repository.gh_id()
+    dependencies.nodes.blocking_issue.number()
+    dependencies.nodes.blocking_issue.repository.gh_id()
+    dependencies.page_info.has_next_page()
+    dependencies.page_info.end_cursor()
 
-class ZHDepsResourceHandler(drest.resource.ResourceHandler):
+def get_dependency_graph(endpoint, workspace_id, repos):
-    def get(self, repo_id):
+    edges = []
-        path = '/repositories/%d/dependencies' % repo_id
+    cursor = None
 
-        try:
+    while True:
-            response = self.api.make_request('GET', path, {})
+        op = Operation(zenhub_schema.Query)
-        except drest.exc.dRestRequestError as e:
+        fetch_workspace_graph(op, workspace_id, repos, cursor)
-            msg = "%s (repo_id: %s)" % (e.msg, repo_id)
-            raise drest.exc.dRestRequestError(msg, e.response)
 
-        def issue(json):
+        d = endpoint(op)
-            return (int(json['repo_id']), int(json['issue_number']))
+        data = (op + d)
 
-        return nx.DiGraph([
+        dependencies = data.workspace.issue_dependencies
-            (issue(edge['blocking']), issue(edge['blocked']))
+        edges += [
-            for edge in response.data['dependencies']
+            (
-        ])
+                (node.blocking_issue.repository.gh_id, node.blocking_issue.number),
+                (node.blocked_issue.repository.gh_id, node.blocked_issue.number),
+            ) for node in dependencies.nodes
+        ]
 
-class ZHEpicsResourceHandler(drest.resource.ResourceHandler):
+        if dependencies.page_info.has_next_page:
-    def get(self, repo_id):
+            cursor = dependencies.page_info.end_cursor
-        path = '/repositories/%d/epics' % repo_id
+            print('.', end='', flush=True)
+        else:
+            print()
+            break
 
-        try:
+    return nx.DiGraph(edges)
-            response = self.api.make_request('GET', path, {})
-        except drest.exc.dRestRequestError as e:
-            msg = "%s (repo_id: %s)" % (e.msg, repo_id)
-            raise drest.exc.dRestRequestError(msg, e.response)
 
-        return [i['issue_number'] for i in response.data['epic_issues']]
+def fetch_epics(op, workspace_id, repos, cursor):
+    epics = op.workspace(id=workspace_id).epics(
+        # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
+        # not the GitHub repo IDs (which the previous REST API used).
+        # repository_ids=repos,
+        first=100,
+        after=cursor,
+    )
+    epics.nodes.id()
+    epics.nodes.issue.number()
+    epics.nodes.issue.repository.gh_id()
+    epics.page_info.has_next_page()
+    epics.page_info.end_cursor()
 
-class ZHEpicsIssuesResourceHandler(drest.resource.ResourceHandler):
+def get_epics(endpoint, workspace_id, repos):
-    def get(self, repo_id, epic_id):
+    epics = []
-        path = '/repositories/%d/epics/%d' % (repo_id, epic_id)
+    cursor = None
 
-        try:
+    while True:
-            response = self.api.make_request('GET', path, {})
+        op = Operation(zenhub_schema.Query)
-        except drest.exc.dRestRequestError as e:
+        fetch_epics(op, workspace_id, repos, cursor)
-            msg = "%s (repo_id: %s)" % (e.msg, repo_id)
-            raise drest.exc.dRestRequestError(msg, e.response)
 
-        return [(i['repo_id'], i['issue_number']) for i in response.data['issues']]
+        d = endpoint(op)
+        data = (op + d)
 
-class ZenHubAPI(drest.api.API):
+        epics_page = data.workspace.epics
-    class Meta:
+        epics += [
-        baseurl = 'https://api.zenhub.com/p1'
+            (node.id, (node.issue.repository.gh_id, node.issue.number))
-        extra_headers = {
+            for node in epics_page.nodes
-            'X-Authentication-Token': ZENHUB_TOKEN,
+        ]
-            'Content-Type': 'application/json',
-        }
 
-    def __init__(self, *args, **kw):
+        if epics_page.page_info.has_next_page:
-        super(ZenHubAPI, self).__init__(*args, **kw)
+            cursor = epics_page.page_info.end_cursor
-        self.add_resource('dependencies', ZHDepsResourceHandler)
+            print('.', end='', flush=True)
-        self.add_resource('epics', ZHEpicsResourceHandler)
+        else:
-        self.add_resource('epics_issues', ZHEpicsIssuesResourceHandler)
+            print()
+            break
 
-    def auth(self, *args, **kw):
+    return epics
-        pass
 
+def fetch_epic_issues(op, workspace_id, epic_id, cursor):
+    epic = op.workspace(id=workspace_id).epics(ids=[epic_id])
+    child_issues = epic.nodes.child_issues(
+        first=100,
+        after=cursor,
+    )
+    child_issues.nodes.number()
+    child_issues.nodes.repository.gh_id()
+    child_issues.page_info.has_next_page()
+    child_issues.page_info.end_cursor()
+
+def get_epic_issues(endpoint, workspace_id, epic_id):
+    epic_issues = []
+    cursor = None
+
+    while True:
+        op = Operation(zenhub_schema.Query)
+        fetch_epic_issues(op, workspace_id, epic_id, cursor)
+
+        d = endpoint(op)
+        data = (op + d)
+
+        epic = data.workspace.epics.nodes[0]
+        epic_issues += [
+            (node.repository.gh_id, node.number)
+            for node in epic.child_issues.nodes
+        ]
+
+        if epic.child_issues.page_info.has_next_page:
+            cursor = epic.child_issues.page_info.end_cursor
+            print('.', end='', flush=True)
+        else:
+            print()
+            break
+
+    return epic_issues
 
 def main():
     gapi = HTTPEndpoint(
         'https://api.github.com/graphql',
         {'Authorization': 'bearer %s' % GITHUB_TOKEN},
     )
-    zapi = ZenHubAPI()
+    zapi = HTTPEndpoint(
+        'https://api.zenhub.com/public/graphql',
+        {'Authorization': 'Bearer %s' % ZENHUB_TOKEN},
+    )
 
-    # Build the full dependency graph from ZenHub's per-repo APIs.
+    # Build the full dependency graph from ZenHub's per-workspace API.
-    dg = nx.compose_all([zapi.dependencies.get(x) for x in REPOS])
+    print('Fetching graph')
+    dg = nx.compose_all([
+        get_dependency_graph(zapi, workspace_id, repos)
+        for (workspace_id, repos) in WORKSPACES.items()
+        if len(repos) > 0
+    ])
+
+    print('Rendering DAG')
 
     if SHOW_EPICS:
         epics_issues = []
-        for repo_id in REPOS:
+        for (workspace_id, repos) in WORKSPACES.items():
-            for epic_id in zapi.epics.get(repo_id):
+            if len(repos) > 0:
-                epics_issues.append((repo_id, epic_id))
+                epics_issues += get_epics(zapi, workspace_id, repos)
+        epics_issues = set(epics_issues)
 
-        epics_mapping = download_issues(gapi, epics_issues)
+        epics_mapping = download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues])
         epics_mapping = {k: v for (k, v) in epics_mapping.items() if v.state != 'closed'}
         issues_by_epic = {}
         for (i, ((repo_id, epic_id), epic)) in enumerate(epics_mapping.items()):
-            issues = set(zapi.epics_issues.get(repo_id, epic_id))
+            workspace_id = [
+                workspace_id
+                for (workspace_id, repos) in WORKSPACES.items()
+                if repo_id in repos
+            ][0]
+            epic_id = [
+                id for (id, gh_ref) in epics_issues
+                if gh_ref == (repo_id, epic_id)
+            ][0]
+            issues = set(get_epic_issues(zapi, workspace_id, epic_id))
             issues_by_epic[epic] = issues
             for i in issues:
                 # zapi.dependencies only returns nodes that have some connection,