zec
/
developers
mirror of https://github.com/zcash/developers.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

switch to ZenHub GraphQL API 

Co-authored-by: Jack Grigg <jack@electriccoin.co>
This commit is contained in:
Conrado Gouvea 2024-06-06 20:29:10 -03:00 committed by Jack Grigg
parent 09cdcd6d7f
commit 86d74f0bff
7 changed files with 172 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/gh-pages.yml vendored
View File

@ -54,6 +54,17 @@ jobs:
- name: Generate GitHub GraphQL schema module
run: sgqlc-codegen schema github_schema.json github_schema.py
- name: Fetch ZenHub GraphQL schema
run: |
python3 -m sgqlc.introspection \
--exclude-description \
-H "Authorization: Bearer ${{ secrets.ZENHUB_TOKEN }}" \
https://api.zenhub.com/public/graphql \
zenhub_schema.json
- name: Generate ZenHub GraphQL schema module
run: sgqlc-codegen schema zenhub_schema.json zenhub_schema.py
- name: Render ECC core DAG
run: python3 ./zcash-issue-dag.py
env:

5
.gitignore vendored
View File

@ -5,7 +5,10 @@ public/*.svg
github_schema.json
github_schema.py
zcash_developer_tools.egg-info
zenhub_schema.json
zenhub_schema.py
GITHUB_TOKEN
ZENHUB_TOKEN
zcash_developer_tools.egg-info

8
gen-schema.sh
View File

@ -9,3 +9,11 @@ uv run python3 -m sgqlc.introspection \
github_schema.json
uv run sgqlc-codegen schema github_schema.json github_schema.py
uv run python3 -m sgqlc.introspection \
--exclude-description \
-H "Authorization: bearer $(cat ZENHUB_TOKEN)" \
https://api.zenhub.com/public/graphql \
zenhub_schema.json
uv run sgqlc-codegen schema zenhub_schema.json zenhub_schema.py

1
pyproject.toml
View File

@ -2,7 +2,6 @@
name = "zcash-developer-tools"
version = "0.1.0"
dependencies = [
"drest",
"networkx",
"pygraphviz",
"sgqlc",

1
requirements.txt
View File

@ -1,4 +1,3 @@
drest
networkx
pygraphviz
sgqlc

32
uv.lock
View File

@ -1,15 +1,6 @@
version = 1
requires-python = ">=3.8"
[[package]]
name = "drest"
version = "0.9.12"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "httplib2" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/86c8e61ab7573c2762b39d5b07de3997fbe0dbf56b41b3047165612c1b11/drest-0.9.12.tar.gz", hash = "sha256:5d88a53d19b99844baabebc080452dd0d83030e812a8994719e9723d5356542e", size = 29734 }
[[package]]
name = "graphql-core"
version = "3.2.5"
@ -22,18 +13,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e3/dc/078bd6b304de790618ebb95e2aedaadb78f4527ac43a9ad8815f006636b6/graphql_core-3.2.5-py3-none-any.whl", hash = "sha256:2f150d5096448aa4f8ab26268567bbfeef823769893b39c1a2e1409590939c8a", size = 203189 },
]
[[package]]
name = "httplib2"
version = "0.22.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyparsing" },
]
sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854 },
]
[[package]]
name = "networkx"
version = "3.1"
@ -49,15 +28,6 @@ version = "1.11"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/19/db/cc09516573e79a35ac73f437bdcf27893939923d1d06b439897ffc7f3217/pygraphviz-1.11.zip", hash = "sha256:a97eb5ced266f45053ebb1f2c6c6d29091690503e3a5c14be7f908b37b06f2d4", size = 120803 }
[[package]]
name = "pyparsing"
version = "3.1.4"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/83/08/13f3bce01b2061f2bbd582c9df82723de943784cf719a35ac886c652043a/pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032", size = 900231 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 },
]
[[package]]
name = "sgqlc"
version = "16.4"
@ -90,7 +60,6 @@ name = "zcash-developer-tools"
version = "0.1.0"
source = { virtual = "." }
dependencies = [
{ name = "drest" },
{ name = "networkx" },
{ name = "pygraphviz" },
{ name = "sgqlc" },
@ -99,7 +68,6 @@ dependencies = [
[package.metadata]
requires-dist = [
{ name = "drest" },
{ name = "networkx" },
{ name = "pygraphviz" },
{ name = "sgqlc" },

204
zcash-issue-dag.py
View File

@ -4,7 +4,6 @@
# Author: jack@electriccoin.co
# Last updated: 2021-05-07
import drest
import networkx as nx
from str2bool import str2bool as strtobool
@ -16,6 +15,7 @@ from urllib.parse import urlparse
from sgqlc.endpoint.http import HTTPEndpoint
from sgqlc.operation import Operation
from github_schema import github_schema as schema
from zenhub_schema import zenhub_schema
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')
@ -110,6 +110,20 @@ REPO_SETS = {
REPOS = REPO_SETS[DAG_VIEW]
WORKSPACE_SETS = {
# ecc-core
'5dc1fd615862290001229f21': CORE_REPOS.keys(),
# ecc-wallet
'5db8aa0244512d0001e0968e': WALLET_REPOS.keys(),
# zf
'5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(),
}
WORKSPACES = {
workspace_id: [repo_id for repo_id in repos if repo_id in REPOS]
for (workspace_id, repos) in WORKSPACE_SETS.items()
}
SUPPORTED_CATEGORIES = set(['releases', 'targets'])
def cats(s):
return set([x.strip() for x in s.split(',')]) - set([''])
@ -245,88 +259,168 @@ def download_issues(endpoint, nodes):
return ret
def fetch_workspace_graph(op, workspace_id, repos, cursor):
dependencies = op.workspace(id=workspace_id).issue_dependencies(
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
# not the GitHub repo IDs (which the previous REST API used).
# repository_ids=repos,
first=100,
after=cursor,
)
dependencies.nodes.id()
dependencies.nodes.blocked_issue.number()
dependencies.nodes.blocked_issue.repository.gh_id()
dependencies.nodes.blocking_issue.number()
dependencies.nodes.blocking_issue.repository.gh_id()
dependencies.page_info.has_next_page()
dependencies.page_info.end_cursor()
class ZHDepsResourceHandler(drest.resource.ResourceHandler):
def get(self, repo_id):
path = '/repositories/%d/dependencies' % repo_id
def get_dependency_graph(endpoint, workspace_id, repos):
edges = []
cursor = None
try:
response = self.api.make_request('GET', path, {})
except drest.exc.dRestRequestError as e:
msg = "%s (repo_id: %s)" % (e.msg, repo_id)
raise drest.exc.dRestRequestError(msg, e.response)
while True:
op = Operation(zenhub_schema.Query)
fetch_workspace_graph(op, workspace_id, repos, cursor)
def issue(json):
return (int(json['repo_id']), int(json['issue_number']))
d = endpoint(op)
data = (op + d)
return nx.DiGraph([
(issue(edge['blocking']), issue(edge['blocked']))
for edge in response.data['dependencies']
])
dependencies = data.workspace.issue_dependencies
edges += [
(
(node.blocking_issue.repository.gh_id, node.blocking_issue.number),
(node.blocked_issue.repository.gh_id, node.blocked_issue.number),
) for node in dependencies.nodes
]
class ZHEpicsResourceHandler(drest.resource.ResourceHandler):
def get(self, repo_id):
path = '/repositories/%d/epics' % repo_id
if dependencies.page_info.has_next_page:
cursor = dependencies.page_info.end_cursor
print('.', end='', flush=True)
else:
print()
break
try:
response = self.api.make_request('GET', path, {})
except drest.exc.dRestRequestError as e:
msg = "%s (repo_id: %s)" % (e.msg, repo_id)
raise drest.exc.dRestRequestError(msg, e.response)
return nx.DiGraph(edges)
return [i['issue_number'] for i in response.data['epic_issues']]
def fetch_epics(op, workspace_id, repos, cursor):
epics = op.workspace(id=workspace_id).epics(
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
# not the GitHub repo IDs (which the previous REST API used).
# repository_ids=repos,
first=100,
after=cursor,
)
epics.nodes.id()
epics.nodes.issue.number()
epics.nodes.issue.repository.gh_id()
epics.page_info.has_next_page()
epics.page_info.end_cursor()
class ZHEpicsIssuesResourceHandler(drest.resource.ResourceHandler):
def get(self, repo_id, epic_id):
path = '/repositories/%d/epics/%d' % (repo_id, epic_id)
def get_epics(endpoint, workspace_id, repos):
epics = []
cursor = None
try:
response = self.api.make_request('GET', path, {})
except drest.exc.dRestRequestError as e:
msg = "%s (repo_id: %s)" % (e.msg, repo_id)
raise drest.exc.dRestRequestError(msg, e.response)
while True:
op = Operation(zenhub_schema.Query)
fetch_epics(op, workspace_id, repos, cursor)
return [(i['repo_id'], i['issue_number']) for i in response.data['issues']]
d = endpoint(op)
data = (op + d)
class ZenHubAPI(drest.api.API):
class Meta:
baseurl = 'https://api.zenhub.com/p1'
extra_headers = {
'X-Authentication-Token': ZENHUB_TOKEN,
'Content-Type': 'application/json',
}
epics_page = data.workspace.epics
epics += [
(node.id, (node.issue.repository.gh_id, node.issue.number))
for node in epics_page.nodes
]
def __init__(self, *args, **kw):
super(ZenHubAPI, self).__init__(*args, **kw)
self.add_resource('dependencies', ZHDepsResourceHandler)
self.add_resource('epics', ZHEpicsResourceHandler)
self.add_resource('epics_issues', ZHEpicsIssuesResourceHandler)
if epics_page.page_info.has_next_page:
cursor = epics_page.page_info.end_cursor
print('.', end='', flush=True)
else:
print()
break
def auth(self, *args, **kw):
pass
return epics
def fetch_epic_issues(op, workspace_id, epic_id, cursor):
epic = op.workspace(id=workspace_id).epics(ids=[epic_id])
child_issues = epic.nodes.child_issues(
first=100,
after=cursor,
)
child_issues.nodes.number()
child_issues.nodes.repository.gh_id()
child_issues.page_info.has_next_page()
child_issues.page_info.end_cursor()
def get_epic_issues(endpoint, workspace_id, epic_id):
epic_issues = []
cursor = None
while True:
op = Operation(zenhub_schema.Query)
fetch_epic_issues(op, workspace_id, epic_id, cursor)
d = endpoint(op)
data = (op + d)
epic = data.workspace.epics.nodes[0]
epic_issues += [
(node.repository.gh_id, node.number)
for node in epic.child_issues.nodes
]
if epic.child_issues.page_info.has_next_page:
cursor = epic.child_issues.page_info.end_cursor
print('.', end='', flush=True)
else:
print()
break
return epic_issues
def main():
gapi = HTTPEndpoint(
'https://api.github.com/graphql',
{'Authorization': 'bearer %s' % GITHUB_TOKEN},
)
zapi = ZenHubAPI()
zapi = HTTPEndpoint(
'https://api.zenhub.com/public/graphql',
{'Authorization': 'Bearer %s' % ZENHUB_TOKEN},
)
# Build the full dependency graph from ZenHub's per-repo APIs.
dg = nx.compose_all([zapi.dependencies.get(x) for x in REPOS])
# Build the full dependency graph from ZenHub's per-workspace API.
print('Fetching graph')
dg = nx.compose_all([
get_dependency_graph(zapi, workspace_id, repos)
for (workspace_id, repos) in WORKSPACES.items()
if len(repos) > 0
])
print('Rendering DAG')
if SHOW_EPICS:
epics_issues = []
for repo_id in REPOS:
for epic_id in zapi.epics.get(repo_id):
epics_issues.append((repo_id, epic_id))
for (workspace_id, repos) in WORKSPACES.items():
if len(repos) > 0:
epics_issues += get_epics(zapi, workspace_id, repos)
epics_issues = set(epics_issues)
epics_mapping = download_issues(gapi, epics_issues)
epics_mapping = download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues])
epics_mapping = {k: v for (k, v) in epics_mapping.items() if v.state != 'closed'}
issues_by_epic = {}
for (i, ((repo_id, epic_id), epic)) in enumerate(epics_mapping.items()):
issues = set(zapi.epics_issues.get(repo_id, epic_id))
workspace_id = [
workspace_id
for (workspace_id, repos) in WORKSPACES.items()
if repo_id in repos
][0]
epic_id = [
id for (id, gh_ref) in epics_issues
if gh_ref == (repo_id, epic_id)
][0]
issues = set(get_epic_issues(zapi, workspace_id, epic_id))
issues_by_epic[epic] = issues
for i in issues:
# zapi.dependencies only returns nodes that have some connection,