mirror of https://github.com/zcash/developers.git
Merge pull request #58 from zcash/zenhub-graphql
Switch to ZenHub GraphQL API
This commit is contained in:
Kris Nuttycombe
GitHub
commit
e3b9d60b59
|
|
@ -54,6 +54,17 @@ jobs:
|
|||
- name: Generate GitHub GraphQL schema module
|
||||
run: sgqlc-codegen schema github_schema.json github_schema.py
|
||||
|
||||
- name: Fetch ZenHub GraphQL schema
|
||||
run: |
|
||||
python3 -m sgqlc.introspection \
|
||||
--exclude-description \
|
||||
-H "Authorization: Bearer ${{ secrets.ZENHUB_TOKEN }}" \
|
||||
https://api.zenhub.com/public/graphql \
|
||||
zenhub_schema.json
|
||||
|
||||
- name: Generate ZenHub GraphQL schema module
|
||||
run: sgqlc-codegen schema zenhub_schema.json zenhub_schema.py
|
||||
|
||||
- name: Render ECC core DAG
|
||||
run: python3 ./zcash-issue-dag.py
|
||||
env:
|
||||
|
|
@ -137,6 +148,12 @@ jobs:
|
|||
DAG_VIEW: transparent-deprecation
|
||||
TERMINATE_AT: "zcash/zcash#4203"
|
||||
|
||||
- name: Render Zashi pipeline
|
||||
run: python3 ./zashi-pipeline.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ZENHUB_TOKEN: ${{ secrets.ZENHUB_TOKEN }}
|
||||
|
||||
- name: Copy the index page
|
||||
run: cp ./index.html ./public
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@ public/*.svg
|
|||
github_schema.json
|
||||
github_schema.py
|
||||
|
||||
zcash_developer_tools.egg-info
|
||||
zenhub_schema.json
|
||||
zenhub_schema.py
|
||||
|
||||
GITHUB_TOKEN
|
||||
ZENHUB_TOKEN
|
||||
|
||||
zcash_developer_tools.egg-info
|
||||
|
|
|
|||
|
|
@ -9,3 +9,11 @@ uv run python3 -m sgqlc.introspection \
|
|||
github_schema.json
|
||||
|
||||
uv run sgqlc-codegen schema github_schema.json github_schema.py
|
||||
|
||||
uv run python3 -m sgqlc.introspection \
|
||||
--exclude-description \
|
||||
-H "Authorization: Bearer $(cat ZENHUB_TOKEN)" \
|
||||
https://api.zenhub.com/public/graphql \
|
||||
zenhub_schema.json
|
||||
|
||||
uv run sgqlc-codegen schema zenhub_schema.json zenhub_schema.py
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
from sgqlc.endpoint.http import HTTPEndpoint
|
||||
from sgqlc.operation import Operation
|
||||
|
||||
from github_schema import github_schema as schema
|
||||
|
||||
# To get the id of a repo, see <https://stackoverflow.com/a/47223479/393146>.
|
||||
|
||||
HALO2_REPOS = {
|
||||
290019239: ('zcash', 'halo2'),
|
||||
344239327: ('zcash', 'pasta_curves'),
|
||||
}
|
||||
|
||||
CORE_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
48303644: ('zcash', 'incrementalmerkletree'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
133857578: ('zcash-hackworks', 'zcash-test-vectors'),
|
||||
111058300: ('zcash', 'sapling-crypto'),
|
||||
**HALO2_REPOS,
|
||||
305835578: ('zcash', 'orchard'),
|
||||
}
|
||||
|
||||
TFL_REPOS = {
|
||||
642135348: ('Electric-Coin-Company', 'tfl-book'),
|
||||
725179873: ('Electric-Coin-Company', 'zebra-tfl'),
|
||||
695805989: ('zcash', 'simtfl'),
|
||||
}
|
||||
|
||||
ANDROID_REPOS = {
|
||||
390808594: ('Electric-Coin-Company', 'zashi-android'),
|
||||
151763639: ('Electric-Coin-Company', 'zcash-android-wallet-sdk'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
IOS_REPOS = {
|
||||
387551125: ('Electric-Coin-Company', 'zashi-ios'),
|
||||
185480114: ('Electric-Coin-Company', 'zcash-swift-wallet-sdk'),
|
||||
270825987: ('Electric-Coin-Company', 'MnemonicSwift'),
|
||||
439137887: ('Electric-Coin-Company', 'zcash-light-client-ffi'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
WALLET_REPOS = {
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
**ANDROID_REPOS,
|
||||
**IOS_REPOS,
|
||||
}
|
||||
|
||||
ECC_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**TFL_REPOS,
|
||||
**WALLET_REPOS,
|
||||
65419597: ('Electric-Coin-Company', 'infrastructure'),
|
||||
}
|
||||
|
||||
ZF_REPOS = {
|
||||
205255683: ('ZcashFoundation', 'zebra'),
|
||||
225479018: ('ZcashFoundation', 'redjubjub'),
|
||||
235651437: ('ZcashFoundation', 'ed25519-zebra'),
|
||||
279422254: ('ZcashFoundation', 'zcash_script'),
|
||||
}
|
||||
|
||||
ZF_FROST_REPOS = {
|
||||
437862440: ('ZcashFoundation', 'frost'),
|
||||
}
|
||||
|
||||
ZCASHD_DEPRECATION_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
863610221: ('zcash', 'wallet'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
}
|
||||
|
||||
POOL_DEPRECATION_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**WALLET_REPOS,
|
||||
}
|
||||
|
||||
REPO_SETS = {
|
||||
'core': CORE_REPOS,
|
||||
'halo2': HALO2_REPOS,
|
||||
'tfl': TFL_REPOS,
|
||||
'wallet': WALLET_REPOS,
|
||||
'wallet-ios': IOS_REPOS,
|
||||
'wallet-android': ANDROID_REPOS,
|
||||
'ecc': ECC_REPOS,
|
||||
'zf': ZF_REPOS,
|
||||
'zf-frost': ZF_FROST_REPOS,
|
||||
'zf-devops': {**ZF_REPOS, **ZF_FROST_REPOS},
|
||||
'zcashd-deprecation': ZCASHD_DEPRECATION_REPOS,
|
||||
'sprout-deprecation': POOL_DEPRECATION_REPOS,
|
||||
'transparent-deprecation': POOL_DEPRECATION_REPOS,
|
||||
}
|
||||
|
||||
|
||||
def api(token):
|
||||
return HTTPEndpoint(
|
||||
'https://api.github.com/graphql',
|
||||
{'Authorization': 'bearer %s' % token},
|
||||
)
|
||||
|
||||
|
||||
class GitHubIssue:
|
||||
def __init__(self, repo_id, issue_number, data, REPOS):
|
||||
self.repo_id = repo_id
|
||||
self.issue_number = issue_number
|
||||
self.milestone = None
|
||||
self._REPOS = REPOS
|
||||
|
||||
if data is not None:
|
||||
labels = [label['name'] for label in data['labels']['nodes']]
|
||||
self.title = data['title']
|
||||
self.labels = labels
|
||||
self.is_release = 'C-release' in labels
|
||||
self.is_target = 'C-target' in labels
|
||||
self.is_pr = 'merged' in data
|
||||
self.is_committed = 'S-committed' in labels
|
||||
self.is_in_progress = 'S-in-progress' in labels
|
||||
self.waiting_on_review = 'S-waiting-on-review' in labels
|
||||
self.url = data['url']
|
||||
self.state = 'closed' if data['state'] in ['CLOSED', 'MERGED'] else 'open'
|
||||
if 'milestone' in data and data['milestone']:
|
||||
self.milestone = data['milestone']['title']
|
||||
else:
|
||||
# If we can't fetch issue data, assume we don't care.
|
||||
self.title = ''
|
||||
self.labels = []
|
||||
self.url = None
|
||||
self.is_release = False
|
||||
self.is_target = False
|
||||
self.is_pr = False
|
||||
self.is_committed = False
|
||||
self.is_in_progress = False
|
||||
self.waiting_on_review = False
|
||||
self.state = 'closed'
|
||||
|
||||
def __repr__(self):
|
||||
if self.repo_id in self._REPOS:
|
||||
repo = self._REPOS[self.repo_id]
|
||||
# Shorten the representation of long repo names.
|
||||
if repo[0] == 'Electric-Coin-Company':
|
||||
repo = ('ECC', repo[1])
|
||||
repo = '/'.join(repo)
|
||||
return '%s#%d' % (repo, self.issue_number)
|
||||
else:
|
||||
return 'Unknown'
|
||||
|
||||
def __eq__(self, other):
|
||||
return (self.repo_id, self.issue_number) == (other.repo_id, other.issue_number)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.repo_id, self.issue_number))
|
||||
|
||||
def any_cat(self, categories):
|
||||
release_cat = self.is_release if 'releases' in categories else False
|
||||
targets_cat = self.is_target if 'targets' in categories else False
|
||||
return release_cat or targets_cat
|
||||
|
||||
|
||||
def fetch_issues(op, issues, REPOS):
|
||||
repos = set([repo for (repo, _) in issues])
|
||||
repos = {repo: [issue for (r, issue) in issues if r == repo] for repo in repos}
|
||||
|
||||
for repo, issues in repos.items():
|
||||
conn = op.repository(
|
||||
owner=REPOS[repo][0],
|
||||
name=REPOS[repo][1],
|
||||
__alias__='repo%d' % repo,
|
||||
)
|
||||
|
||||
for issue in issues:
|
||||
res = conn.issue_or_pull_request(number=issue, __alias__='issue%d' % issue)
|
||||
for typ in [schema.Issue, schema.PullRequest]:
|
||||
node = res.__as__(typ)
|
||||
node.labels(first=50).nodes().name()
|
||||
node.state()
|
||||
node.milestone().title()
|
||||
node.title()
|
||||
node.url()
|
||||
if typ == schema.PullRequest:
|
||||
node.merged()
|
||||
|
||||
|
||||
def download_issues(endpoint, nodes, REPOS):
|
||||
issues = [(repo, issue) for (repo, issue) in nodes if repo in REPOS]
|
||||
|
||||
ret = {}
|
||||
|
||||
# Ensure that any graph nodes from ZenHub that are not in the repos we care about have
|
||||
# default entries, to simplify subsequent graph manipulation code.
|
||||
for repo, issue in [(repo, issue) for (repo, issue) in nodes if repo not in REPOS]:
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, None, REPOS)
|
||||
|
||||
def chunks(lst, n):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i : i + n]
|
||||
|
||||
for issues in chunks(issues, 50):
|
||||
op = Operation(schema.Query)
|
||||
fetch_issues(op, issues, REPOS)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
for repo, issue in issues:
|
||||
repo_data = data['repo%d' % repo]
|
||||
issue_key = 'issue%d' % issue
|
||||
# If GITHUB_TOKEN doesn't have permission to read from a particular private
|
||||
# repository in REPOS, GitHub returns an empty repo_data section.
|
||||
issue_data = repo_data[issue_key] if issue_key in repo_data else None
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data, REPOS)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def fetch_issues_with_labels(op, labels, repos):
|
||||
for (repo_id, (repo, issue_cursor, pr_cursor)) in repos:
|
||||
conn = op.repository(
|
||||
owner=repo[0],
|
||||
name=repo[1],
|
||||
__alias__='repo%d' % repo_id,
|
||||
)
|
||||
|
||||
if issue_cursor != -1:
|
||||
issues = conn.issues(
|
||||
labels=labels,
|
||||
first=50,
|
||||
after=issue_cursor,
|
||||
)
|
||||
issues.nodes.number()
|
||||
issues.nodes.labels(first=50).nodes().name()
|
||||
issues.nodes.state()
|
||||
issues.nodes.milestone().title()
|
||||
issues.nodes.title()
|
||||
issues.nodes.url()
|
||||
issues.page_info.has_next_page()
|
||||
issues.page_info.end_cursor()
|
||||
|
||||
if pr_cursor != -1:
|
||||
prs = conn.pull_requests(
|
||||
labels=labels,
|
||||
first=50,
|
||||
after=pr_cursor,
|
||||
)
|
||||
prs.nodes.number()
|
||||
prs.nodes.labels(first=50).nodes().name()
|
||||
prs.nodes.state()
|
||||
prs.nodes.milestone().title()
|
||||
prs.nodes.title()
|
||||
prs.nodes.url()
|
||||
prs.nodes.merged()
|
||||
prs.page_info.has_next_page()
|
||||
prs.page_info.end_cursor()
|
||||
|
||||
|
||||
def download_issues_with_labels(endpoint, labels, REPOS):
|
||||
ret = {}
|
||||
repos = {repo_id: (repo, None, None) for (repo_id, repo) in REPOS.items()}
|
||||
|
||||
while True:
|
||||
op = Operation(schema.Query)
|
||||
fetch_issues_with_labels(op, labels, repos.items())
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
done = []
|
||||
for (repo_id, (repo, _, _)) in repos.items():
|
||||
repo_data = data['repo%d' % repo_id]
|
||||
|
||||
if hasattr(repo_data, 'issues'):
|
||||
for issue in repo_data.issues.nodes:
|
||||
ret[(repo_id, issue.number)] = GitHubIssue(repo_id, issue.number, issue, REPOS)
|
||||
if repo_data.issues.page_info.has_next_page:
|
||||
issue_cursor = repo_data.issues.page_info.end_cursor
|
||||
else:
|
||||
issue_cursor = -1
|
||||
else:
|
||||
issue_cursor = -1
|
||||
|
||||
if hasattr(repo_data, 'pull_requests'):
|
||||
for pr in repo_data.pull_requests.nodes:
|
||||
ret[(repo_id, pr.number)] = GitHubIssue(repo_id, pr.number, pr, REPOS)
|
||||
if repo_data.pull_requests.page_info.has_next_page:
|
||||
pr_cursor = repo_data.pull_requests.page_info.end_cursor
|
||||
else:
|
||||
pr_cursor = -1
|
||||
else:
|
||||
pr_cursor = -1
|
||||
|
||||
if issue_cursor == -1 and pr_cursor == -1:
|
||||
done.append(repo_id)
|
||||
else:
|
||||
repos[repo_id] = (repo, issue_cursor, pr_cursor)
|
||||
|
||||
for repo_id in done:
|
||||
del repos[repo_id]
|
||||
|
||||
if len(repos) > 0:
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return ret
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
import networkx as nx
|
||||
from sgqlc.endpoint.http import HTTPEndpoint
|
||||
from sgqlc.operation import Operation
|
||||
|
||||
from helpers.github import CORE_REPOS, WALLET_REPOS, ZF_REPOS, ZF_FROST_REPOS
|
||||
from zenhub_schema import zenhub_schema
|
||||
|
||||
WORKSPACE_SETS = {
|
||||
# ecc-core
|
||||
'5dc1fd615862290001229f21': CORE_REPOS.keys(),
|
||||
# ecc-wallet
|
||||
'5db8aa0244512d0001e0968e': WALLET_REPOS.keys(),
|
||||
# zf
|
||||
'5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(),
|
||||
# zf-frost
|
||||
'607d75e0169bd50011d5410f': ZF_FROST_REPOS.keys(),
|
||||
}
|
||||
|
||||
|
||||
def api(token):
|
||||
return HTTPEndpoint(
|
||||
'https://api.zenhub.com/public/graphql',
|
||||
{'Authorization': 'Bearer %s' % token},
|
||||
)
|
||||
|
||||
|
||||
def fetch_workspace_graph(op, workspace_id, repos, cursor):
|
||||
dependencies = op.workspace(id=workspace_id).issue_dependencies(
|
||||
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
|
||||
# not the GitHub repo IDs (which the previous REST API used).
|
||||
# repository_ids=repos,
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
dependencies.nodes.id()
|
||||
dependencies.nodes.blocked_issue.number()
|
||||
dependencies.nodes.blocked_issue.repository.gh_id()
|
||||
dependencies.nodes.blocking_issue.number()
|
||||
dependencies.nodes.blocking_issue.repository.gh_id()
|
||||
dependencies.page_info.has_next_page()
|
||||
dependencies.page_info.end_cursor()
|
||||
|
||||
|
||||
def get_dependency_graph(endpoint, workspace_id, repos):
|
||||
edges = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_workspace_graph(op, workspace_id, repos, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
if hasattr(data.workspace, 'issue_dependencies'):
|
||||
dependencies = data.workspace.issue_dependencies
|
||||
edges += [
|
||||
(
|
||||
(node.blocking_issue.repository.gh_id, node.blocking_issue.number),
|
||||
(node.blocked_issue.repository.gh_id, node.blocked_issue.number),
|
||||
)
|
||||
for node in dependencies.nodes
|
||||
]
|
||||
|
||||
if dependencies.page_info.has_next_page:
|
||||
cursor = dependencies.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return nx.DiGraph(edges)
|
||||
|
||||
|
||||
def fetch_epics(op, workspace_id, repos, cursor):
|
||||
epics = op.workspace(id=workspace_id).epics(
|
||||
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
|
||||
# not the GitHub repo IDs (which the previous REST API used).
|
||||
# repository_ids=repos,
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
epics.nodes.id()
|
||||
epics.nodes.issue.number()
|
||||
epics.nodes.issue.repository.gh_id()
|
||||
epics.page_info.has_next_page()
|
||||
epics.page_info.end_cursor()
|
||||
|
||||
|
||||
def get_epics(endpoint, workspace_id, repos):
|
||||
epics = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_epics(op, workspace_id, repos, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
epics_page = data.workspace.epics
|
||||
epics += [
|
||||
(node.id, (node.issue.repository.gh_id, node.issue.number))
|
||||
for node in epics_page.nodes
|
||||
]
|
||||
|
||||
if epics_page.page_info.has_next_page:
|
||||
cursor = epics_page.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return epics
|
||||
|
||||
|
||||
def fetch_epic_issues(op, workspace_id, epic_id, cursor):
|
||||
epic = op.workspace(id=workspace_id).epics(ids=[epic_id])
|
||||
child_issues = epic.nodes.child_issues(
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
child_issues.nodes.number()
|
||||
child_issues.nodes.repository.gh_id()
|
||||
child_issues.page_info.has_next_page()
|
||||
child_issues.page_info.end_cursor()
|
||||
|
||||
|
||||
def get_epic_issues(endpoint, workspace_id, epic_id):
|
||||
epic_issues = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_epic_issues(op, workspace_id, epic_id, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
epic = data.workspace.epics.nodes[0]
|
||||
epic_issues += [
|
||||
(node.repository.gh_id, node.number) for node in epic.child_issues.nodes
|
||||
]
|
||||
|
||||
if epic.child_issues.page_info.has_next_page:
|
||||
cursor = epic.child_issues.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return epic_issues
|
||||
|
|
@ -17,5 +17,6 @@
|
|||
<p><a href="zcash-zcashd-deprecation-dag"><code>zcashd</code> deprecation DAG</a></p>
|
||||
<p><a href="zcash-sprout-deprecation-dag">Sprout pool deprecation DAG</a></p>
|
||||
<p><a href="zcash-transparent-deprecation-dag">Transparent pool deprecation DAG</a></p>
|
||||
<p><a href="zashi-pipeline">Zashi pipeline</a></p>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
name = "zcash-developer-tools"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"drest",
|
||||
"networkx",
|
||||
"pygraphviz",
|
||||
"sgqlc",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
drest
|
||||
networkx
|
||||
pygraphviz
|
||||
sgqlc
|
||||
|
|
|
|||
32
uv.lock
32
uv.lock
|
|
@ -1,15 +1,6 @@
|
|||
version = 1
|
||||
requires-python = ">=3.8"
|
||||
|
||||
[[package]]
|
||||
name = "drest"
|
||||
version = "0.9.12"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "httplib2" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/86c8e61ab7573c2762b39d5b07de3997fbe0dbf56b41b3047165612c1b11/drest-0.9.12.tar.gz", hash = "sha256:5d88a53d19b99844baabebc080452dd0d83030e812a8994719e9723d5356542e", size = 29734 }
|
||||
|
||||
[[package]]
|
||||
name = "graphql-core"
|
||||
version = "3.2.5"
|
||||
|
|
@ -22,18 +13,6 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/e3/dc/078bd6b304de790618ebb95e2aedaadb78f4527ac43a9ad8815f006636b6/graphql_core-3.2.5-py3-none-any.whl", hash = "sha256:2f150d5096448aa4f8ab26268567bbfeef823769893b39c1a2e1409590939c8a", size = 203189 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httplib2"
|
||||
version = "0.22.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pyparsing" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "networkx"
|
||||
version = "3.1"
|
||||
|
|
@ -49,15 +28,6 @@ version = "1.11"
|
|||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/19/db/cc09516573e79a35ac73f437bdcf27893939923d1d06b439897ffc7f3217/pygraphviz-1.11.zip", hash = "sha256:a97eb5ced266f45053ebb1f2c6c6d29091690503e3a5c14be7f908b37b06f2d4", size = 120803 }
|
||||
|
||||
[[package]]
|
||||
name = "pyparsing"
|
||||
version = "3.1.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/83/08/13f3bce01b2061f2bbd582c9df82723de943784cf719a35ac886c652043a/pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032", size = 900231 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", size = 104100 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sgqlc"
|
||||
version = "16.4"
|
||||
|
|
@ -90,7 +60,6 @@ name = "zcash-developer-tools"
|
|||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "drest" },
|
||||
{ name = "networkx" },
|
||||
{ name = "pygraphviz" },
|
||||
{ name = "sgqlc" },
|
||||
|
|
@ -99,7 +68,6 @@ dependencies = [
|
|||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "drest" },
|
||||
{ name = "networkx" },
|
||||
{ name = "pygraphviz" },
|
||||
{ name = "sgqlc" },
|
||||
|
|
|
|||
|
|
@ -0,0 +1,250 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import networkx as nx
|
||||
from str2bool import str2bool as strtobool
|
||||
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
from textwrap import wrap
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from helpers import github, zenhub
|
||||
|
||||
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
|
||||
ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')
|
||||
|
||||
# IDs of repos we look for releases in.
|
||||
RUST = 85334928
|
||||
ANDROID_SDK = 151763639
|
||||
SWIFT_SDK = 185480114
|
||||
ZASHI_ANDROID = 390808594
|
||||
ZASHI_IOS = 387551125
|
||||
|
||||
REPOS = {
|
||||
**github.CORE_REPOS,
|
||||
**github.WALLET_REPOS,
|
||||
}
|
||||
|
||||
RELEASE_MATRIX = {
|
||||
RUST: [ANDROID_SDK, SWIFT_SDK],
|
||||
ANDROID_SDK: [ZASHI_ANDROID],
|
||||
SWIFT_SDK: [ZASHI_IOS],
|
||||
ZASHI_ANDROID: [],
|
||||
ZASHI_IOS: []
|
||||
}
|
||||
|
||||
class TrackedIssue:
|
||||
def __init__(self, issue):
|
||||
self.issue = issue
|
||||
|
||||
def build_release_matrix_from(dg, issue, repo_id):
|
||||
acc = []
|
||||
for child in dg.neighbors(issue):
|
||||
if child.repo_id == repo_id and 'C-release' in child.labels:
|
||||
# Fetch the rows that each child's downstreams need rendered.
|
||||
child_deps = [
|
||||
build_release_matrix_from(dg, child, dep_repo)
|
||||
for dep_repo in RELEASE_MATRIX.get(repo_id)
|
||||
]
|
||||
|
||||
# Merge the rows from each downstream repo together.
|
||||
child_releases = [
|
||||
{k: v for d in prod for k, v in d.items()}
|
||||
for prod in itertools.product(*child_deps)
|
||||
]
|
||||
|
||||
if len(child_releases) > 0:
|
||||
for rec in child_releases:
|
||||
rec[repo_id] = child
|
||||
else:
|
||||
child_releases = [{repo_id: child}]
|
||||
|
||||
acc.extend(child_releases)
|
||||
else:
|
||||
acc.extend(build_release_matrix_from(dg, child, repo_id))
|
||||
|
||||
return acc
|
||||
|
||||
def main():
|
||||
gapi = github.api(GITHUB_TOKEN)
|
||||
zapi = zenhub.api(ZENHUB_TOKEN)
|
||||
|
||||
print('Fetching tracked issues')
|
||||
tracked_issues = github.download_issues_with_labels(gapi, ['C-tracked-bug', 'C-tracked-feature'], REPOS)
|
||||
|
||||
# The repos we care about are now:
|
||||
# - Any repo containing a tracked issue.
|
||||
# - The wallet repos where releases occur.
|
||||
tracked_repos = set([repo_id for (repo_id, _) in tracked_issues])
|
||||
repos = {
|
||||
**github.WALLET_REPOS,
|
||||
}
|
||||
for repo_id in tracked_repos:
|
||||
repos[repo_id] = REPOS[repo_id]
|
||||
workspaces = {
|
||||
workspace_id: [repo_id for repo_id in repos if repo_id in repos]
|
||||
for (workspace_id, repos) in zenhub.WORKSPACE_SETS.items()
|
||||
}
|
||||
|
||||
# Build the full dependency graph from ZenHub's per-workspace API.
|
||||
print('Fetching graph')
|
||||
dg = nx.compose_all([
|
||||
zenhub.get_dependency_graph(zapi, workspace_id, repos)
|
||||
for (workspace_id, repos) in workspaces.items()
|
||||
if len(repos) > 0
|
||||
])
|
||||
|
||||
print('Rendering deployment pipeline')
|
||||
|
||||
# Ensure that the tracked issues all exist in the graph. This is a no-op for
|
||||
# issues that are already present.
|
||||
start_at = set([issue for issue in tracked_issues])
|
||||
for i in start_at:
|
||||
dg.add_node(i)
|
||||
|
||||
# Replace the graph with the subgraph that only includes the tracked
|
||||
# issues and their descendants.
|
||||
descendants = [nx.descendants(dg, n) for n in start_at]
|
||||
dg = nx.subgraph(dg, start_at.union(*descendants))
|
||||
|
||||
# Fetch the issues within the graph.
|
||||
mapping = github.download_issues(gapi, dg.nodes, repos)
|
||||
|
||||
# Relabel the graph
|
||||
dg = nx.relabel_nodes(dg, mapping)
|
||||
|
||||
# Filter out unknown issues
|
||||
unknown = [n for n in dg if n.repo_id not in repos]
|
||||
if len(unknown) > 0:
|
||||
dg.remove_nodes_from(unknown)
|
||||
|
||||
# Apply property annotations
|
||||
for (source, sink) in dg.edges:
|
||||
attrs = dg.edges[source, sink]
|
||||
attrs['is_open'] = 0 if source.state == 'closed' else 1
|
||||
|
||||
# Render the HTML version!
|
||||
html_header = '''<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Zashi Pipeline</title>
|
||||
|
||||
<style>
|
||||
body {
|
||||
color: #1f2328;
|
||||
font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
a {
|
||||
color: #0969da;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
width: max-content;
|
||||
max-width: 100%;
|
||||
overflow: auto;
|
||||
}
|
||||
table tr {
|
||||
border-top: 1px solid #d1d9e0b3;
|
||||
}
|
||||
table th, table td {
|
||||
border: 1px solid #d1d9e0b3;
|
||||
padding: 6px 13px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
body {
|
||||
background-color: #121212;
|
||||
color: #f0f6fc;
|
||||
}
|
||||
a {
|
||||
color: #4493f8;
|
||||
}
|
||||
table tr {
|
||||
border-top: 1px solid #3d444db3;
|
||||
}
|
||||
table th, table td {
|
||||
border: 1px solid #3d444db3;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Zashi Pipeline</h1>
|
||||
<p>🐞 = bug, 💡 = feature, ✅ = implemented / released, 🛑 = unfinished, 📥 = unassigned / DAG needs updating</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Issue</th>
|
||||
<th>Rust crate</th>
|
||||
<th>Android SDK</th>
|
||||
<th>Swift SDK</th>
|
||||
<th>Zashi Android</th>
|
||||
<th>Zashi iOS</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
'''
|
||||
html_footer = '''
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
with open('public/zashi-pipeline.html', 'w') as f:
|
||||
f.write(html_header)
|
||||
|
||||
for issue in tracked_issues.values():
|
||||
rows = build_release_matrix_from(dg, issue, RUST);
|
||||
for i, row in enumerate(rows):
|
||||
f.write('<tr>')
|
||||
|
||||
if i == 0:
|
||||
rowspan = ''
|
||||
if len(rows) > 1:
|
||||
rowspan = ' rowspan="{}"'.format(len(rows))
|
||||
|
||||
f.write('<td{}>{}</td>'.format(
|
||||
rowspan,
|
||||
'🐞' if 'C-tracked-bug' in issue.labels else '💡',
|
||||
))
|
||||
f.write('<td{}>{} <a href="{}">{}</a></td>'.format(
|
||||
rowspan,
|
||||
'✅' if issue.state == 'closed' else '🛑',
|
||||
issue.url,
|
||||
issue.title,
|
||||
))
|
||||
|
||||
for repo_id in [RUST, ANDROID_SDK, SWIFT_SDK, ZASHI_ANDROID, ZASHI_IOS]:
|
||||
child = row.get(repo_id)
|
||||
if child is None:
|
||||
# Release not found in this repo
|
||||
f.write('<td>📥</td>')
|
||||
else:
|
||||
# Extract version number from title
|
||||
if repo_id == RUST:
|
||||
version = re.search(r'zcash_[^ ]+ \d+(\.\d+)+', child.title).group()
|
||||
else:
|
||||
version = re.search(r'\d+(\.\d+)+', child.title).group()
|
||||
|
||||
f.write('<td>{} <a href="{}">{}</a></td>'.format(
|
||||
'✅' if child.state == 'closed' else '🛑',
|
||||
child.url,
|
||||
version,
|
||||
))
|
||||
f.write('</tr>')
|
||||
|
||||
f.write(html_footer)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if GITHUB_TOKEN and ZENHUB_TOKEN:
|
||||
main()
|
||||
else:
|
||||
print('Please set the GITHUB_TOKEN and ZENHUB_TOKEN environment variables.')
|
||||
|
|
@ -4,112 +4,27 @@
|
|||
# Author: jack@electriccoin.co
|
||||
# Last updated: 2021-05-07
|
||||
|
||||
import drest
|
||||
import networkx as nx
|
||||
|
||||
from str2bool import str2bool as strtobool
|
||||
import mimetypes
|
||||
import os
|
||||
from textwrap import wrap
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from sgqlc.endpoint.http import HTTPEndpoint
|
||||
from sgqlc.operation import Operation
|
||||
from github_schema import github_schema as schema
|
||||
from helpers import github, zenhub
|
||||
|
||||
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
|
||||
ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')
|
||||
|
||||
DAG_VIEW = os.environ.get('DAG_VIEW', 'core')
|
||||
|
||||
# To get the id of a repo, see <https://stackoverflow.com/a/47223479/393146>.
|
||||
REPOS = github.REPO_SETS[DAG_VIEW]
|
||||
|
||||
HALO2_REPOS = {
|
||||
290019239: ('zcash', 'halo2'),
|
||||
344239327: ('zcash', 'pasta_curves'),
|
||||
WORKSPACES = {
|
||||
workspace_id: [repo_id for repo_id in repos if repo_id in REPOS]
|
||||
for (workspace_id, repos) in zenhub.WORKSPACE_SETS.items()
|
||||
}
|
||||
|
||||
CORE_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
48303644: ('zcash', 'incrementalmerkletree'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
133857578: ('zcash-hackworks', 'zcash-test-vectors'),
|
||||
111058300: ('zcash', 'sapling-crypto'),
|
||||
**HALO2_REPOS,
|
||||
305835578: ('zcash', 'orchard'),
|
||||
}
|
||||
|
||||
TFL_REPOS = {
|
||||
642135348: ('Electric-Coin-Company', 'tfl-book'),
|
||||
725179873: ('Electric-Coin-Company', 'zebra-tfl'),
|
||||
695805989: ('zcash', 'simtfl'),
|
||||
}
|
||||
|
||||
ANDROID_REPOS = {
|
||||
390808594: ('Electric-Coin-Company', 'zashi-android'),
|
||||
151763639: ('Electric-Coin-Company', 'zcash-android-wallet-sdk'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
IOS_REPOS = {
|
||||
387551125: ('Electric-Coin-Company', 'zashi-ios'),
|
||||
185480114: ('Electric-Coin-Company', 'zcash-swift-wallet-sdk'),
|
||||
270825987: ('Electric-Coin-Company', 'MnemonicSwift'),
|
||||
439137887: ('Electric-Coin-Company', 'zcash-light-client-ffi'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
WALLET_REPOS = {
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
**ANDROID_REPOS,
|
||||
**IOS_REPOS,
|
||||
}
|
||||
|
||||
ECC_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**TFL_REPOS,
|
||||
**WALLET_REPOS,
|
||||
65419597: ('Electric-Coin-Company', 'infrastructure'),
|
||||
}
|
||||
|
||||
ZF_REPOS = {
|
||||
205255683: ('ZcashFoundation', 'zebra'),
|
||||
225479018: ('ZcashFoundation', 'redjubjub'),
|
||||
235651437: ('ZcashFoundation', 'ed25519-zebra'),
|
||||
279422254: ('ZcashFoundation', 'zcash_script'),
|
||||
}
|
||||
|
||||
ZCASHD_DEPRECATION_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
863610221: ('zcash', 'wallet'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
}
|
||||
|
||||
POOL_DEPRECATION_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**WALLET_REPOS,
|
||||
}
|
||||
|
||||
REPO_SETS = {
|
||||
'core': CORE_REPOS,
|
||||
'halo2': HALO2_REPOS,
|
||||
'tfl': TFL_REPOS,
|
||||
'wallet': WALLET_REPOS,
|
||||
'wallet-ios': IOS_REPOS,
|
||||
'wallet-android': ANDROID_REPOS,
|
||||
'ecc': ECC_REPOS,
|
||||
'zf': ZF_REPOS,
|
||||
'zcashd-deprecation': ZCASHD_DEPRECATION_REPOS,
|
||||
'sprout-deprecation': POOL_DEPRECATION_REPOS,
|
||||
'transparent-deprecation': POOL_DEPRECATION_REPOS,
|
||||
}
|
||||
|
||||
REPOS = REPO_SETS[DAG_VIEW]
|
||||
|
||||
SUPPORTED_CATEGORIES = set(['releases', 'targets'])
|
||||
def cats(s):
|
||||
return set([x.strip() for x in s.split(',')]) - set([''])
|
||||
|
|
@ -138,195 +53,41 @@ SHOW_MILESTONES = strtobool(os.environ.get('SHOW_MILESTONES', 'false'))
|
|||
SHOW_EPICS = strtobool(os.environ.get('SHOW_EPICS', 'false'))
|
||||
|
||||
|
||||
class GitHubIssue:
|
||||
def __init__(self, repo_id, issue_number, data):
|
||||
self.repo_id = repo_id
|
||||
self.issue_number = issue_number
|
||||
self.milestone = None
|
||||
|
||||
if data is not None:
|
||||
labels = [label['name'] for label in data['labels']['nodes']]
|
||||
self.title = data['title']
|
||||
self.is_release = 'C-release' in labels
|
||||
self.is_target = 'C-target' in labels
|
||||
self.is_pr = 'merged' in data
|
||||
self.is_committed = 'S-committed' in labels
|
||||
self.is_in_progress = 'S-in-progress' in labels
|
||||
self.waiting_on_review = 'S-waiting-on-review' in labels
|
||||
self.url = data['url']
|
||||
self.state = 'closed' if data['state'] in ['CLOSED', 'MERGED'] else 'open'
|
||||
if 'milestone' in data and data['milestone']:
|
||||
self.milestone = data['milestone']['title']
|
||||
else:
|
||||
# If we can't fetch issue data, assume we don't care.
|
||||
self.title = ''
|
||||
self.url = None
|
||||
self.is_release = False
|
||||
self.is_target = False
|
||||
self.is_pr = False
|
||||
self.is_committed = False
|
||||
self.is_in_progress = False
|
||||
self.waiting_on_review = False
|
||||
self.state = 'closed'
|
||||
|
||||
def __repr__(self):
|
||||
if self.repo_id in REPOS:
|
||||
repo = REPOS[self.repo_id]
|
||||
# Shorten the representation of long repo names.
|
||||
if repo[0] == 'Electric-Coin-Company':
|
||||
repo = ('ECC', repo[1])
|
||||
repo = '/'.join(repo)
|
||||
return '%s#%d' % (repo, self.issue_number)
|
||||
else:
|
||||
return 'Unknown'
|
||||
|
||||
def __eq__(self, other):
|
||||
return (self.repo_id, self.issue_number) == (other.repo_id, other.issue_number)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.repo_id, self.issue_number))
|
||||
|
||||
def any_cat(self, categories):
|
||||
release_cat = self.is_release if 'releases' in categories else False
|
||||
targets_cat = self.is_target if 'targets' in categories else False
|
||||
return release_cat or targets_cat
|
||||
|
||||
def fetch_issues(op, issues):
|
||||
repos = set([repo for (repo, _) in issues])
|
||||
repos = {repo: [issue for (r, issue) in issues if r == repo] for repo in repos}
|
||||
|
||||
for (repo, issues) in repos.items():
|
||||
conn = op.repository(
|
||||
owner=REPOS[repo][0],
|
||||
name=REPOS[repo][1],
|
||||
__alias__='repo%d' % repo,
|
||||
)
|
||||
|
||||
for issue in issues:
|
||||
res = conn.issue_or_pull_request(number=issue, __alias__='issue%d' % issue)
|
||||
for typ in [schema.Issue, schema.PullRequest]:
|
||||
node = res.__as__(typ)
|
||||
node.labels(first=50).nodes().name()
|
||||
node.state()
|
||||
node.milestone().title()
|
||||
node.title()
|
||||
node.url()
|
||||
if typ == schema.PullRequest:
|
||||
node.merged()
|
||||
|
||||
def download_issues(endpoint, nodes):
|
||||
issues = [(repo, issue) for (repo, issue) in nodes if repo in REPOS]
|
||||
|
||||
ret = {}
|
||||
|
||||
# Ensure that any graph nodes from ZenHub that are not in the repos we care about have
|
||||
# default entries, to simplify subsequent graph manipulation code.
|
||||
for (repo, issue) in [(repo, issue) for (repo, issue) in nodes if repo not in REPOS]:
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, None)
|
||||
|
||||
def chunks(lst, n):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
|
||||
for issues in chunks(issues, 50):
|
||||
op = Operation(schema.Query)
|
||||
fetch_issues(op, issues)
|
||||
|
||||
d = endpoint(op)
|
||||
data = (op + d)
|
||||
|
||||
for (repo, issue) in issues:
|
||||
repo_data = data['repo%d' % repo]
|
||||
issue_key = 'issue%d' % issue
|
||||
# If GITHUB_TOKEN doesn't have permission to read from a particular private
|
||||
# repository in REPOS, GitHub returns an empty repo_data section.
|
||||
issue_data = repo_data[issue_key] if issue_key in repo_data else None
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
class ZHDepsResourceHandler(drest.resource.ResourceHandler):
|
||||
def get(self, repo_id):
|
||||
path = '/repositories/%d/dependencies' % repo_id
|
||||
|
||||
try:
|
||||
response = self.api.make_request('GET', path, {})
|
||||
except drest.exc.dRestRequestError as e:
|
||||
msg = "%s (repo_id: %s)" % (e.msg, repo_id)
|
||||
raise drest.exc.dRestRequestError(msg, e.response)
|
||||
|
||||
def issue(json):
|
||||
return (int(json['repo_id']), int(json['issue_number']))
|
||||
|
||||
return nx.DiGraph([
|
||||
(issue(edge['blocking']), issue(edge['blocked']))
|
||||
for edge in response.data['dependencies']
|
||||
])
|
||||
|
||||
class ZHEpicsResourceHandler(drest.resource.ResourceHandler):
|
||||
def get(self, repo_id):
|
||||
path = '/repositories/%d/epics' % repo_id
|
||||
|
||||
try:
|
||||
response = self.api.make_request('GET', path, {})
|
||||
except drest.exc.dRestRequestError as e:
|
||||
msg = "%s (repo_id: %s)" % (e.msg, repo_id)
|
||||
raise drest.exc.dRestRequestError(msg, e.response)
|
||||
|
||||
return [i['issue_number'] for i in response.data['epic_issues']]
|
||||
|
||||
class ZHEpicsIssuesResourceHandler(drest.resource.ResourceHandler):
|
||||
def get(self, repo_id, epic_id):
|
||||
path = '/repositories/%d/epics/%d' % (repo_id, epic_id)
|
||||
|
||||
try:
|
||||
response = self.api.make_request('GET', path, {})
|
||||
except drest.exc.dRestRequestError as e:
|
||||
msg = "%s (repo_id: %s)" % (e.msg, repo_id)
|
||||
raise drest.exc.dRestRequestError(msg, e.response)
|
||||
|
||||
return [(i['repo_id'], i['issue_number']) for i in response.data['issues']]
|
||||
|
||||
class ZenHubAPI(drest.api.API):
|
||||
class Meta:
|
||||
baseurl = 'https://api.zenhub.com/p1'
|
||||
extra_headers = {
|
||||
'X-Authentication-Token': ZENHUB_TOKEN,
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(ZenHubAPI, self).__init__(*args, **kw)
|
||||
self.add_resource('dependencies', ZHDepsResourceHandler)
|
||||
self.add_resource('epics', ZHEpicsResourceHandler)
|
||||
self.add_resource('epics_issues', ZHEpicsIssuesResourceHandler)
|
||||
|
||||
def auth(self, *args, **kw):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
gapi = HTTPEndpoint(
|
||||
'https://api.github.com/graphql',
|
||||
{'Authorization': 'bearer %s' % GITHUB_TOKEN},
|
||||
)
|
||||
zapi = ZenHubAPI()
|
||||
gapi = github.api(GITHUB_TOKEN)
|
||||
zapi = zenhub.api(ZENHUB_TOKEN)
|
||||
|
||||
# Build the full dependency graph from ZenHub's per-repo APIs.
|
||||
dg = nx.compose_all([zapi.dependencies.get(x) for x in REPOS])
|
||||
# Build the full dependency graph from ZenHub's per-workspace API.
|
||||
print('Fetching graph')
|
||||
dg = nx.compose_all([
|
||||
zenhub.get_dependency_graph(zapi, workspace_id, repos)
|
||||
for (workspace_id, repos) in WORKSPACES.items()
|
||||
if len(repos) > 0
|
||||
])
|
||||
|
||||
print('Rendering DAG')
|
||||
|
||||
if SHOW_EPICS:
|
||||
epics_issues = []
|
||||
for repo_id in REPOS:
|
||||
for epic_id in zapi.epics.get(repo_id):
|
||||
epics_issues.append((repo_id, epic_id))
|
||||
for (workspace_id, repos) in WORKSPACES.items():
|
||||
if len(repos) > 0:
|
||||
epics_issues += zenhub.get_epics(zapi, workspace_id, repos)
|
||||
epics_issues = set(epics_issues)
|
||||
|
||||
epics_mapping = download_issues(gapi, epics_issues)
|
||||
epics_mapping = github.download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues], REPOS)
|
||||
epics_mapping = {k: v for (k, v) in epics_mapping.items() if v.state != 'closed'}
|
||||
issues_by_epic = {}
|
||||
for (i, ((repo_id, epic_id), epic)) in enumerate(epics_mapping.items()):
|
||||
issues = set(zapi.epics_issues.get(repo_id, epic_id))
|
||||
workspace_id = [
|
||||
workspace_id
|
||||
for (workspace_id, repos) in WORKSPACES.items()
|
||||
if repo_id in repos
|
||||
][0]
|
||||
epic_id = [
|
||||
id for (id, gh_ref) in epics_issues
|
||||
if gh_ref == (repo_id, epic_id)
|
||||
][0]
|
||||
issues = set(zenhub.get_epic_issues(zapi, workspace_id, epic_id))
|
||||
issues_by_epic[epic] = issues
|
||||
for i in issues:
|
||||
# zapi.dependencies only returns nodes that have some connection,
|
||||
|
|
@ -346,7 +107,7 @@ def main():
|
|||
dg = nx.subgraph(dg, terminate_at.union(*ancestors))
|
||||
|
||||
# Fetch the issues within the graph.
|
||||
mapping = download_issues(gapi, dg.nodes)
|
||||
mapping = github.download_issues(gapi, dg.nodes, REPOS)
|
||||
|
||||
# Relabel the graph
|
||||
dg = nx.relabel_nodes(dg, mapping)
|
||||
|
|
|
|||
Loading…
Reference in New Issue