mirror of https://github.com/zcash/developers.git
Merge pull request #67 from zcash/change-deployment-pipeline
Add a Zashi pipeline renderer
This commit is contained in:
commit
ec032ce14a
|
@ -148,6 +148,12 @@ jobs:
|
|||
DAG_VIEW: transparent-deprecation
|
||||
TERMINATE_AT: "zcash/zcash#4203"
|
||||
|
||||
- name: Render Zashi pipeline
|
||||
run: python3 ./zashi-pipeline.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ZENHUB_TOKEN: ${{ secrets.ZENHUB_TOKEN }}
|
||||
|
||||
- name: Copy the index page
|
||||
run: cp ./index.html ./public
|
||||
|
||||
|
|
|
@ -0,0 +1,308 @@
|
|||
from sgqlc.endpoint.http import HTTPEndpoint
|
||||
from sgqlc.operation import Operation
|
||||
|
||||
from github_schema import github_schema as schema
|
||||
|
||||
# To get the id of a repo, see <https://stackoverflow.com/a/47223479/393146>.
|
||||
|
||||
HALO2_REPOS = {
|
||||
290019239: ('zcash', 'halo2'),
|
||||
344239327: ('zcash', 'pasta_curves'),
|
||||
}
|
||||
|
||||
CORE_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
48303644: ('zcash', 'incrementalmerkletree'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
133857578: ('zcash-hackworks', 'zcash-test-vectors'),
|
||||
111058300: ('zcash', 'sapling-crypto'),
|
||||
**HALO2_REPOS,
|
||||
305835578: ('zcash', 'orchard'),
|
||||
}
|
||||
|
||||
TFL_REPOS = {
|
||||
642135348: ('Electric-Coin-Company', 'tfl-book'),
|
||||
725179873: ('Electric-Coin-Company', 'zebra-tfl'),
|
||||
695805989: ('zcash', 'simtfl'),
|
||||
}
|
||||
|
||||
ANDROID_REPOS = {
|
||||
390808594: ('Electric-Coin-Company', 'zashi-android'),
|
||||
151763639: ('Electric-Coin-Company', 'zcash-android-wallet-sdk'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
IOS_REPOS = {
|
||||
387551125: ('Electric-Coin-Company', 'zashi-ios'),
|
||||
185480114: ('Electric-Coin-Company', 'zcash-swift-wallet-sdk'),
|
||||
270825987: ('Electric-Coin-Company', 'MnemonicSwift'),
|
||||
439137887: ('Electric-Coin-Company', 'zcash-light-client-ffi'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
WALLET_REPOS = {
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
**ANDROID_REPOS,
|
||||
**IOS_REPOS,
|
||||
}
|
||||
|
||||
ECC_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**TFL_REPOS,
|
||||
**WALLET_REPOS,
|
||||
65419597: ('Electric-Coin-Company', 'infrastructure'),
|
||||
}
|
||||
|
||||
ZF_REPOS = {
|
||||
205255683: ('ZcashFoundation', 'zebra'),
|
||||
225479018: ('ZcashFoundation', 'redjubjub'),
|
||||
235651437: ('ZcashFoundation', 'ed25519-zebra'),
|
||||
279422254: ('ZcashFoundation', 'zcash_script'),
|
||||
}
|
||||
|
||||
ZF_FROST_REPOS = {
|
||||
437862440: ('ZcashFoundation', 'frost'),
|
||||
}
|
||||
|
||||
ZCASHD_DEPRECATION_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
863610221: ('zcash', 'wallet'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
}
|
||||
|
||||
POOL_DEPRECATION_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**WALLET_REPOS,
|
||||
}
|
||||
|
||||
REPO_SETS = {
|
||||
'core': CORE_REPOS,
|
||||
'halo2': HALO2_REPOS,
|
||||
'tfl': TFL_REPOS,
|
||||
'wallet': WALLET_REPOS,
|
||||
'wallet-ios': IOS_REPOS,
|
||||
'wallet-android': ANDROID_REPOS,
|
||||
'ecc': ECC_REPOS,
|
||||
'zf': ZF_REPOS,
|
||||
'zf-frost': ZF_FROST_REPOS,
|
||||
'zf-devops': {**ZF_REPOS, **ZF_FROST_REPOS},
|
||||
'zcashd-deprecation': ZCASHD_DEPRECATION_REPOS,
|
||||
'sprout-deprecation': POOL_DEPRECATION_REPOS,
|
||||
'transparent-deprecation': POOL_DEPRECATION_REPOS,
|
||||
}
|
||||
|
||||
|
||||
def api(token):
|
||||
return HTTPEndpoint(
|
||||
'https://api.github.com/graphql',
|
||||
{'Authorization': 'bearer %s' % token},
|
||||
)
|
||||
|
||||
|
||||
class GitHubIssue:
|
||||
def __init__(self, repo_id, issue_number, data, REPOS):
|
||||
self.repo_id = repo_id
|
||||
self.issue_number = issue_number
|
||||
self.milestone = None
|
||||
self._REPOS = REPOS
|
||||
|
||||
if data is not None:
|
||||
labels = [label['name'] for label in data['labels']['nodes']]
|
||||
self.title = data['title']
|
||||
self.labels = labels
|
||||
self.is_release = 'C-release' in labels
|
||||
self.is_target = 'C-target' in labels
|
||||
self.is_pr = 'merged' in data
|
||||
self.is_committed = 'S-committed' in labels
|
||||
self.is_in_progress = 'S-in-progress' in labels
|
||||
self.waiting_on_review = 'S-waiting-on-review' in labels
|
||||
self.url = data['url']
|
||||
self.state = 'closed' if data['state'] in ['CLOSED', 'MERGED'] else 'open'
|
||||
if 'milestone' in data and data['milestone']:
|
||||
self.milestone = data['milestone']['title']
|
||||
else:
|
||||
# If we can't fetch issue data, assume we don't care.
|
||||
self.title = ''
|
||||
self.labels = []
|
||||
self.url = None
|
||||
self.is_release = False
|
||||
self.is_target = False
|
||||
self.is_pr = False
|
||||
self.is_committed = False
|
||||
self.is_in_progress = False
|
||||
self.waiting_on_review = False
|
||||
self.state = 'closed'
|
||||
|
||||
def __repr__(self):
|
||||
if self.repo_id in self._REPOS:
|
||||
repo = self._REPOS[self.repo_id]
|
||||
# Shorten the representation of long repo names.
|
||||
if repo[0] == 'Electric-Coin-Company':
|
||||
repo = ('ECC', repo[1])
|
||||
repo = '/'.join(repo)
|
||||
return '%s#%d' % (repo, self.issue_number)
|
||||
else:
|
||||
return 'Unknown'
|
||||
|
||||
def __eq__(self, other):
|
||||
return (self.repo_id, self.issue_number) == (other.repo_id, other.issue_number)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.repo_id, self.issue_number))
|
||||
|
||||
def any_cat(self, categories):
|
||||
release_cat = self.is_release if 'releases' in categories else False
|
||||
targets_cat = self.is_target if 'targets' in categories else False
|
||||
return release_cat or targets_cat
|
||||
|
||||
|
||||
def fetch_issues(op, issues, REPOS):
|
||||
repos = set([repo for (repo, _) in issues])
|
||||
repos = {repo: [issue for (r, issue) in issues if r == repo] for repo in repos}
|
||||
|
||||
for repo, issues in repos.items():
|
||||
conn = op.repository(
|
||||
owner=REPOS[repo][0],
|
||||
name=REPOS[repo][1],
|
||||
__alias__='repo%d' % repo,
|
||||
)
|
||||
|
||||
for issue in issues:
|
||||
res = conn.issue_or_pull_request(number=issue, __alias__='issue%d' % issue)
|
||||
for typ in [schema.Issue, schema.PullRequest]:
|
||||
node = res.__as__(typ)
|
||||
node.labels(first=50).nodes().name()
|
||||
node.state()
|
||||
node.milestone().title()
|
||||
node.title()
|
||||
node.url()
|
||||
if typ == schema.PullRequest:
|
||||
node.merged()
|
||||
|
||||
|
||||
def download_issues(endpoint, nodes, REPOS):
|
||||
issues = [(repo, issue) for (repo, issue) in nodes if repo in REPOS]
|
||||
|
||||
ret = {}
|
||||
|
||||
# Ensure that any graph nodes from ZenHub that are not in the repos we care about have
|
||||
# default entries, to simplify subsequent graph manipulation code.
|
||||
for repo, issue in [(repo, issue) for (repo, issue) in nodes if repo not in REPOS]:
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, None, REPOS)
|
||||
|
||||
def chunks(lst, n):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i : i + n]
|
||||
|
||||
for issues in chunks(issues, 50):
|
||||
op = Operation(schema.Query)
|
||||
fetch_issues(op, issues, REPOS)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
for repo, issue in issues:
|
||||
repo_data = data['repo%d' % repo]
|
||||
issue_key = 'issue%d' % issue
|
||||
# If GITHUB_TOKEN doesn't have permission to read from a particular private
|
||||
# repository in REPOS, GitHub returns an empty repo_data section.
|
||||
issue_data = repo_data[issue_key] if issue_key in repo_data else None
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data, REPOS)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def fetch_issues_with_labels(op, labels, repos):
|
||||
for (repo_id, (repo, issue_cursor, pr_cursor)) in repos:
|
||||
conn = op.repository(
|
||||
owner=repo[0],
|
||||
name=repo[1],
|
||||
__alias__='repo%d' % repo_id,
|
||||
)
|
||||
|
||||
if issue_cursor != -1:
|
||||
issues = conn.issues(
|
||||
labels=labels,
|
||||
first=50,
|
||||
after=issue_cursor,
|
||||
)
|
||||
issues.nodes.number()
|
||||
issues.nodes.labels(first=50).nodes().name()
|
||||
issues.nodes.state()
|
||||
issues.nodes.milestone().title()
|
||||
issues.nodes.title()
|
||||
issues.nodes.url()
|
||||
issues.page_info.has_next_page()
|
||||
issues.page_info.end_cursor()
|
||||
|
||||
if pr_cursor != -1:
|
||||
prs = conn.pull_requests(
|
||||
labels=labels,
|
||||
first=50,
|
||||
after=pr_cursor,
|
||||
)
|
||||
prs.nodes.number()
|
||||
prs.nodes.labels(first=50).nodes().name()
|
||||
prs.nodes.state()
|
||||
prs.nodes.milestone().title()
|
||||
prs.nodes.title()
|
||||
prs.nodes.url()
|
||||
prs.nodes.merged()
|
||||
prs.page_info.has_next_page()
|
||||
prs.page_info.end_cursor()
|
||||
|
||||
|
||||
def download_issues_with_labels(endpoint, labels, REPOS):
|
||||
ret = {}
|
||||
repos = {repo_id: (repo, None, None) for (repo_id, repo) in REPOS.items()}
|
||||
|
||||
while True:
|
||||
op = Operation(schema.Query)
|
||||
fetch_issues_with_labels(op, labels, repos.items())
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
done = []
|
||||
for (repo_id, (repo, _, _)) in repos.items():
|
||||
repo_data = data['repo%d' % repo_id]
|
||||
|
||||
if hasattr(repo_data, 'issues'):
|
||||
for issue in repo_data.issues.nodes:
|
||||
ret[(repo_id, issue.number)] = GitHubIssue(repo_id, issue.number, issue, REPOS)
|
||||
if repo_data.issues.page_info.has_next_page:
|
||||
issue_cursor = repo_data.issues.page_info.end_cursor
|
||||
else:
|
||||
issue_cursor = -1
|
||||
else:
|
||||
issue_cursor = -1
|
||||
|
||||
if hasattr(repo_data, 'pull_requests'):
|
||||
for pr in repo_data.pull_requests.nodes:
|
||||
ret[(repo_id, pr.number)] = GitHubIssue(repo_id, pr.number, pr, REPOS)
|
||||
if repo_data.pull_requests.page_info.has_next_page:
|
||||
pr_cursor = repo_data.pull_requests.page_info.end_cursor
|
||||
else:
|
||||
pr_cursor = -1
|
||||
else:
|
||||
pr_cursor = -1
|
||||
|
||||
if issue_cursor == -1 and pr_cursor == -1:
|
||||
done.append(repo_id)
|
||||
else:
|
||||
repos[repo_id] = (repo, issue_cursor, pr_cursor)
|
||||
|
||||
for repo_id in done:
|
||||
del repos[repo_id]
|
||||
|
||||
if len(repos) > 0:
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return ret
|
|
@ -0,0 +1,155 @@
|
|||
import networkx as nx
|
||||
from sgqlc.endpoint.http import HTTPEndpoint
|
||||
from sgqlc.operation import Operation
|
||||
|
||||
from helpers.github import CORE_REPOS, WALLET_REPOS, ZF_REPOS, ZF_FROST_REPOS
|
||||
from zenhub_schema import zenhub_schema
|
||||
|
||||
WORKSPACE_SETS = {
|
||||
# ecc-core
|
||||
'5dc1fd615862290001229f21': CORE_REPOS.keys(),
|
||||
# ecc-wallet
|
||||
'5db8aa0244512d0001e0968e': WALLET_REPOS.keys(),
|
||||
# zf
|
||||
'5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(),
|
||||
# zf-frost
|
||||
'607d75e0169bd50011d5410f': ZF_FROST_REPOS.keys(),
|
||||
}
|
||||
|
||||
|
||||
def api(token):
|
||||
return HTTPEndpoint(
|
||||
'https://api.zenhub.com/public/graphql',
|
||||
{'Authorization': 'Bearer %s' % token},
|
||||
)
|
||||
|
||||
|
||||
def fetch_workspace_graph(op, workspace_id, repos, cursor):
|
||||
dependencies = op.workspace(id=workspace_id).issue_dependencies(
|
||||
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
|
||||
# not the GitHub repo IDs (which the previous REST API used).
|
||||
# repository_ids=repos,
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
dependencies.nodes.id()
|
||||
dependencies.nodes.blocked_issue.number()
|
||||
dependencies.nodes.blocked_issue.repository.gh_id()
|
||||
dependencies.nodes.blocking_issue.number()
|
||||
dependencies.nodes.blocking_issue.repository.gh_id()
|
||||
dependencies.page_info.has_next_page()
|
||||
dependencies.page_info.end_cursor()
|
||||
|
||||
|
||||
def get_dependency_graph(endpoint, workspace_id, repos):
|
||||
edges = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_workspace_graph(op, workspace_id, repos, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
if hasattr(data.workspace, 'issue_dependencies'):
|
||||
dependencies = data.workspace.issue_dependencies
|
||||
edges += [
|
||||
(
|
||||
(node.blocking_issue.repository.gh_id, node.blocking_issue.number),
|
||||
(node.blocked_issue.repository.gh_id, node.blocked_issue.number),
|
||||
)
|
||||
for node in dependencies.nodes
|
||||
]
|
||||
|
||||
if dependencies.page_info.has_next_page:
|
||||
cursor = dependencies.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return nx.DiGraph(edges)
|
||||
|
||||
|
||||
def fetch_epics(op, workspace_id, repos, cursor):
|
||||
epics = op.workspace(id=workspace_id).epics(
|
||||
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
|
||||
# not the GitHub repo IDs (which the previous REST API used).
|
||||
# repository_ids=repos,
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
epics.nodes.id()
|
||||
epics.nodes.issue.number()
|
||||
epics.nodes.issue.repository.gh_id()
|
||||
epics.page_info.has_next_page()
|
||||
epics.page_info.end_cursor()
|
||||
|
||||
|
||||
def get_epics(endpoint, workspace_id, repos):
|
||||
epics = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_epics(op, workspace_id, repos, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
epics_page = data.workspace.epics
|
||||
epics += [
|
||||
(node.id, (node.issue.repository.gh_id, node.issue.number))
|
||||
for node in epics_page.nodes
|
||||
]
|
||||
|
||||
if epics_page.page_info.has_next_page:
|
||||
cursor = epics_page.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return epics
|
||||
|
||||
|
||||
def fetch_epic_issues(op, workspace_id, epic_id, cursor):
|
||||
epic = op.workspace(id=workspace_id).epics(ids=[epic_id])
|
||||
child_issues = epic.nodes.child_issues(
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
child_issues.nodes.number()
|
||||
child_issues.nodes.repository.gh_id()
|
||||
child_issues.page_info.has_next_page()
|
||||
child_issues.page_info.end_cursor()
|
||||
|
||||
|
||||
def get_epic_issues(endpoint, workspace_id, epic_id):
|
||||
epic_issues = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_epic_issues(op, workspace_id, epic_id, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = op + d
|
||||
|
||||
epic = data.workspace.epics.nodes[0]
|
||||
epic_issues += [
|
||||
(node.repository.gh_id, node.number) for node in epic.child_issues.nodes
|
||||
]
|
||||
|
||||
if epic.child_issues.page_info.has_next_page:
|
||||
cursor = epic.child_issues.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return epic_issues
|
|
@ -17,5 +17,6 @@
|
|||
<p><a href="zcash-zcashd-deprecation-dag"><code>zcashd</code> deprecation DAG</a></p>
|
||||
<p><a href="zcash-sprout-deprecation-dag">Sprout pool deprecation DAG</a></p>
|
||||
<p><a href="zcash-transparent-deprecation-dag">Transparent pool deprecation DAG</a></p>
|
||||
<p><a href="zashi-pipeline">Zashi pipeline</a></p>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
@ -0,0 +1,250 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import networkx as nx
|
||||
from str2bool import str2bool as strtobool
|
||||
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
from textwrap import wrap
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from helpers import github, zenhub
|
||||
|
||||
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
|
||||
ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')
|
||||
|
||||
# IDs of repos we look for releases in.
|
||||
RUST = 85334928
|
||||
ANDROID_SDK = 151763639
|
||||
SWIFT_SDK = 185480114
|
||||
ZASHI_ANDROID = 390808594
|
||||
ZASHI_IOS = 387551125
|
||||
|
||||
REPOS = {
|
||||
**github.CORE_REPOS,
|
||||
**github.WALLET_REPOS,
|
||||
}
|
||||
|
||||
RELEASE_MATRIX = {
|
||||
RUST: [ANDROID_SDK, SWIFT_SDK],
|
||||
ANDROID_SDK: [ZASHI_ANDROID],
|
||||
SWIFT_SDK: [ZASHI_IOS],
|
||||
ZASHI_ANDROID: [],
|
||||
ZASHI_IOS: []
|
||||
}
|
||||
|
||||
class TrackedIssue:
|
||||
def __init__(self, issue):
|
||||
self.issue = issue
|
||||
|
||||
def build_release_matrix_from(dg, issue, repo_id):
|
||||
acc = []
|
||||
for child in dg.neighbors(issue):
|
||||
if child.repo_id == repo_id and 'C-release' in child.labels:
|
||||
# Fetch the rows that each child's downstreams need rendered.
|
||||
child_deps = [
|
||||
build_release_matrix_from(dg, child, dep_repo)
|
||||
for dep_repo in RELEASE_MATRIX.get(repo_id)
|
||||
]
|
||||
|
||||
# Merge the rows from each downstream repo together.
|
||||
child_releases = [
|
||||
{k: v for d in prod for k, v in d.items()}
|
||||
for prod in itertools.product(*child_deps)
|
||||
]
|
||||
|
||||
if len(child_releases) > 0:
|
||||
for rec in child_releases:
|
||||
rec[repo_id] = child
|
||||
else:
|
||||
child_releases = [{repo_id: child}]
|
||||
|
||||
acc.extend(child_releases)
|
||||
else:
|
||||
acc.extend(build_release_matrix_from(dg, child, repo_id))
|
||||
|
||||
return acc
|
||||
|
||||
def main():
|
||||
gapi = github.api(GITHUB_TOKEN)
|
||||
zapi = zenhub.api(ZENHUB_TOKEN)
|
||||
|
||||
print('Fetching tracked issues')
|
||||
tracked_issues = github.download_issues_with_labels(gapi, ['C-tracked-bug', 'C-tracked-feature'], REPOS)
|
||||
|
||||
# The repos we care about are now:
|
||||
# - Any repo containing a tracked issue.
|
||||
# - The wallet repos where releases occur.
|
||||
tracked_repos = set([repo_id for (repo_id, _) in tracked_issues])
|
||||
repos = {
|
||||
**github.WALLET_REPOS,
|
||||
}
|
||||
for repo_id in tracked_repos:
|
||||
repos[repo_id] = REPOS[repo_id]
|
||||
workspaces = {
|
||||
workspace_id: [repo_id for repo_id in repos if repo_id in repos]
|
||||
for (workspace_id, repos) in zenhub.WORKSPACE_SETS.items()
|
||||
}
|
||||
|
||||
# Build the full dependency graph from ZenHub's per-workspace API.
|
||||
print('Fetching graph')
|
||||
dg = nx.compose_all([
|
||||
zenhub.get_dependency_graph(zapi, workspace_id, repos)
|
||||
for (workspace_id, repos) in workspaces.items()
|
||||
if len(repos) > 0
|
||||
])
|
||||
|
||||
print('Rendering deployment pipeline')
|
||||
|
||||
# Ensure that the tracked issues all exist in the graph. This is a no-op for
|
||||
# issues that are already present.
|
||||
start_at = set([issue for issue in tracked_issues])
|
||||
for i in start_at:
|
||||
dg.add_node(i)
|
||||
|
||||
# Replace the graph with the subgraph that only includes the tracked
|
||||
# issues and their descendants.
|
||||
descendants = [nx.descendants(dg, n) for n in start_at]
|
||||
dg = nx.subgraph(dg, start_at.union(*descendants))
|
||||
|
||||
# Fetch the issues within the graph.
|
||||
mapping = github.download_issues(gapi, dg.nodes, repos)
|
||||
|
||||
# Relabel the graph
|
||||
dg = nx.relabel_nodes(dg, mapping)
|
||||
|
||||
# Filter out unknown issues
|
||||
unknown = [n for n in dg if n.repo_id not in repos]
|
||||
if len(unknown) > 0:
|
||||
dg.remove_nodes_from(unknown)
|
||||
|
||||
# Apply property annotations
|
||||
for (source, sink) in dg.edges:
|
||||
attrs = dg.edges[source, sink]
|
||||
attrs['is_open'] = 0 if source.state == 'closed' else 1
|
||||
|
||||
# Render the HTML version!
|
||||
html_header = '''<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Zashi Pipeline</title>
|
||||
|
||||
<style>
|
||||
body {
|
||||
color: #1f2328;
|
||||
font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
a {
|
||||
color: #0969da;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
width: max-content;
|
||||
max-width: 100%;
|
||||
overflow: auto;
|
||||
}
|
||||
table tr {
|
||||
border-top: 1px solid #d1d9e0b3;
|
||||
}
|
||||
table th, table td {
|
||||
border: 1px solid #d1d9e0b3;
|
||||
padding: 6px 13px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
body {
|
||||
background-color: #121212;
|
||||
color: #f0f6fc;
|
||||
}
|
||||
a {
|
||||
color: #4493f8;
|
||||
}
|
||||
table tr {
|
||||
border-top: 1px solid #3d444db3;
|
||||
}
|
||||
table th, table td {
|
||||
border: 1px solid #3d444db3;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Zashi Pipeline</h1>
|
||||
<p>🐞 = bug, 💡 = feature, ✅ = implemented / released, 🛑 = unfinished, 📥 = unassigned / DAG needs updating</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Issue</th>
|
||||
<th>Rust crate</th>
|
||||
<th>Android SDK</th>
|
||||
<th>Swift SDK</th>
|
||||
<th>Zashi Android</th>
|
||||
<th>Zashi iOS</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
'''
|
||||
html_footer = '''
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
with open('public/zashi-pipeline.html', 'w') as f:
|
||||
f.write(html_header)
|
||||
|
||||
for issue in tracked_issues.values():
|
||||
rows = build_release_matrix_from(dg, issue, RUST);
|
||||
for i, row in enumerate(rows):
|
||||
f.write('<tr>')
|
||||
|
||||
if i == 0:
|
||||
rowspan = ''
|
||||
if len(rows) > 1:
|
||||
rowspan = ' rowspan="{}"'.format(len(rows))
|
||||
|
||||
f.write('<td{}>{}</td>'.format(
|
||||
rowspan,
|
||||
'🐞' if 'C-tracked-bug' in issue.labels else '💡',
|
||||
))
|
||||
f.write('<td{}>{} <a href="{}">{}</a></td>'.format(
|
||||
rowspan,
|
||||
'✅' if issue.state == 'closed' else '🛑',
|
||||
issue.url,
|
||||
issue.title,
|
||||
))
|
||||
|
||||
for repo_id in [RUST, ANDROID_SDK, SWIFT_SDK, ZASHI_ANDROID, ZASHI_IOS]:
|
||||
child = row.get(repo_id)
|
||||
if child is None:
|
||||
# Release not found in this repo
|
||||
f.write('<td>📥</td>')
|
||||
else:
|
||||
# Extract version number from title
|
||||
if repo_id == RUST:
|
||||
version = re.search(r'zcash_[^ ]+ \d+(\.\d+)+', child.title).group()
|
||||
else:
|
||||
version = re.search(r'\d+(\.\d+)+', child.title).group()
|
||||
|
||||
f.write('<td>{} <a href="{}">{}</a></td>'.format(
|
||||
'✅' if child.state == 'closed' else '🛑',
|
||||
child.url,
|
||||
version,
|
||||
))
|
||||
f.write('</tr>')
|
||||
|
||||
f.write(html_footer)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if GITHUB_TOKEN and ZENHUB_TOKEN:
|
||||
main()
|
||||
else:
|
||||
print('Please set the GITHUB_TOKEN and ZENHUB_TOKEN environment variables.')
|
|
@ -7,129 +7,22 @@
|
|||
import networkx as nx
|
||||
|
||||
from str2bool import str2bool as strtobool
|
||||
import mimetypes
|
||||
import os
|
||||
from textwrap import wrap
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from sgqlc.endpoint.http import HTTPEndpoint
|
||||
from sgqlc.operation import Operation
|
||||
from github_schema import github_schema as schema
|
||||
from zenhub_schema import zenhub_schema
|
||||
from helpers import github, zenhub
|
||||
|
||||
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
|
||||
ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN')
|
||||
|
||||
DAG_VIEW = os.environ.get('DAG_VIEW', 'core')
|
||||
|
||||
# To get the id of a repo, see <https://stackoverflow.com/a/47223479/393146>.
|
||||
|
||||
HALO2_REPOS = {
|
||||
290019239: ('zcash', 'halo2'),
|
||||
344239327: ('zcash', 'pasta_curves'),
|
||||
}
|
||||
|
||||
CORE_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
48303644: ('zcash', 'incrementalmerkletree'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
133857578: ('zcash-hackworks', 'zcash-test-vectors'),
|
||||
111058300: ('zcash', 'sapling-crypto'),
|
||||
**HALO2_REPOS,
|
||||
305835578: ('zcash', 'orchard'),
|
||||
}
|
||||
|
||||
TFL_REPOS = {
|
||||
642135348: ('Electric-Coin-Company', 'tfl-book'),
|
||||
725179873: ('Electric-Coin-Company', 'zebra-tfl'),
|
||||
695805989: ('zcash', 'simtfl'),
|
||||
}
|
||||
|
||||
ANDROID_REPOS = {
|
||||
390808594: ('Electric-Coin-Company', 'zashi-android'),
|
||||
151763639: ('Electric-Coin-Company', 'zcash-android-wallet-sdk'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
IOS_REPOS = {
|
||||
387551125: ('Electric-Coin-Company', 'zashi-ios'),
|
||||
185480114: ('Electric-Coin-Company', 'zcash-swift-wallet-sdk'),
|
||||
270825987: ('Electric-Coin-Company', 'MnemonicSwift'),
|
||||
439137887: ('Electric-Coin-Company', 'zcash-light-client-ffi'),
|
||||
719178328: ('Electric-Coin-Company', 'zashi'),
|
||||
}
|
||||
|
||||
WALLET_REPOS = {
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
**ANDROID_REPOS,
|
||||
**IOS_REPOS,
|
||||
}
|
||||
|
||||
ECC_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**TFL_REPOS,
|
||||
**WALLET_REPOS,
|
||||
65419597: ('Electric-Coin-Company', 'infrastructure'),
|
||||
}
|
||||
|
||||
ZF_REPOS = {
|
||||
205255683: ('ZcashFoundation', 'zebra'),
|
||||
225479018: ('ZcashFoundation', 'redjubjub'),
|
||||
235651437: ('ZcashFoundation', 'ed25519-zebra'),
|
||||
279422254: ('ZcashFoundation', 'zcash_script'),
|
||||
}
|
||||
|
||||
ZF_FROST_REPOS = {
|
||||
437862440: ('ZcashFoundation', 'frost'),
|
||||
}
|
||||
|
||||
ZCASHD_DEPRECATION_REPOS = {
|
||||
26987049: ('zcash', 'zcash'),
|
||||
47279130: ('zcash', 'zips'),
|
||||
85334928: ('zcash', 'librustzcash'),
|
||||
863610221: ('zcash', 'wallet'),
|
||||
159714694: ('zcash', 'lightwalletd'),
|
||||
}
|
||||
|
||||
POOL_DEPRECATION_REPOS = {
|
||||
**CORE_REPOS,
|
||||
**WALLET_REPOS,
|
||||
}
|
||||
|
||||
REPO_SETS = {
|
||||
'core': CORE_REPOS,
|
||||
'halo2': HALO2_REPOS,
|
||||
'tfl': TFL_REPOS,
|
||||
'wallet': WALLET_REPOS,
|
||||
'wallet-ios': IOS_REPOS,
|
||||
'wallet-android': ANDROID_REPOS,
|
||||
'ecc': ECC_REPOS,
|
||||
'zf': ZF_REPOS,
|
||||
'zf-frost': ZF_FROST_REPOS,
|
||||
'zf-devops': {**ZF_REPOS, **ZF_FROST_REPOS},
|
||||
'zcashd-deprecation': ZCASHD_DEPRECATION_REPOS,
|
||||
'sprout-deprecation': POOL_DEPRECATION_REPOS,
|
||||
'transparent-deprecation': POOL_DEPRECATION_REPOS,
|
||||
}
|
||||
|
||||
REPOS = REPO_SETS[DAG_VIEW]
|
||||
|
||||
WORKSPACE_SETS = {
|
||||
# ecc-core
|
||||
'5dc1fd615862290001229f21': CORE_REPOS.keys(),
|
||||
# ecc-wallet
|
||||
'5db8aa0244512d0001e0968e': WALLET_REPOS.keys(),
|
||||
# zf
|
||||
'5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(),
|
||||
# zf-frost
|
||||
'607d75e0169bd50011d5410f': ZF_FROST_REPOS.keys(),
|
||||
}
|
||||
REPOS = github.REPO_SETS[DAG_VIEW]
|
||||
|
||||
WORKSPACES = {
|
||||
workspace_id: [repo_id for repo_id in repos if repo_id in REPOS]
|
||||
for (workspace_id, repos) in WORKSPACE_SETS.items()
|
||||
for (workspace_id, repos) in zenhub.WORKSPACE_SETS.items()
|
||||
}
|
||||
|
||||
SUPPORTED_CATEGORIES = set(['releases', 'targets'])
|
||||
|
@ -160,248 +53,14 @@ SHOW_MILESTONES = strtobool(os.environ.get('SHOW_MILESTONES', 'false'))
|
|||
SHOW_EPICS = strtobool(os.environ.get('SHOW_EPICS', 'false'))
|
||||
|
||||
|
||||
class GitHubIssue:
|
||||
def __init__(self, repo_id, issue_number, data):
|
||||
self.repo_id = repo_id
|
||||
self.issue_number = issue_number
|
||||
self.milestone = None
|
||||
|
||||
if data is not None:
|
||||
labels = [label['name'] for label in data['labels']['nodes']]
|
||||
self.title = data['title']
|
||||
self.is_release = 'C-release' in labels
|
||||
self.is_target = 'C-target' in labels
|
||||
self.is_pr = 'merged' in data
|
||||
self.is_committed = 'S-committed' in labels
|
||||
self.is_in_progress = 'S-in-progress' in labels
|
||||
self.waiting_on_review = 'S-waiting-on-review' in labels
|
||||
self.url = data['url']
|
||||
self.state = 'closed' if data['state'] in ['CLOSED', 'MERGED'] else 'open'
|
||||
if 'milestone' in data and data['milestone']:
|
||||
self.milestone = data['milestone']['title']
|
||||
else:
|
||||
# If we can't fetch issue data, assume we don't care.
|
||||
self.title = ''
|
||||
self.url = None
|
||||
self.is_release = False
|
||||
self.is_target = False
|
||||
self.is_pr = False
|
||||
self.is_committed = False
|
||||
self.is_in_progress = False
|
||||
self.waiting_on_review = False
|
||||
self.state = 'closed'
|
||||
|
||||
def __repr__(self):
|
||||
if self.repo_id in REPOS:
|
||||
repo = REPOS[self.repo_id]
|
||||
# Shorten the representation of long repo names.
|
||||
if repo[0] == 'Electric-Coin-Company':
|
||||
repo = ('ECC', repo[1])
|
||||
repo = '/'.join(repo)
|
||||
return '%s#%d' % (repo, self.issue_number)
|
||||
else:
|
||||
return 'Unknown'
|
||||
|
||||
def __eq__(self, other):
|
||||
return (self.repo_id, self.issue_number) == (other.repo_id, other.issue_number)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.repo_id, self.issue_number))
|
||||
|
||||
def any_cat(self, categories):
|
||||
release_cat = self.is_release if 'releases' in categories else False
|
||||
targets_cat = self.is_target if 'targets' in categories else False
|
||||
return release_cat or targets_cat
|
||||
|
||||
def fetch_issues(op, issues):
|
||||
repos = set([repo for (repo, _) in issues])
|
||||
repos = {repo: [issue for (r, issue) in issues if r == repo] for repo in repos}
|
||||
|
||||
for (repo, issues) in repos.items():
|
||||
conn = op.repository(
|
||||
owner=REPOS[repo][0],
|
||||
name=REPOS[repo][1],
|
||||
__alias__='repo%d' % repo,
|
||||
)
|
||||
|
||||
for issue in issues:
|
||||
res = conn.issue_or_pull_request(number=issue, __alias__='issue%d' % issue)
|
||||
for typ in [schema.Issue, schema.PullRequest]:
|
||||
node = res.__as__(typ)
|
||||
node.labels(first=50).nodes().name()
|
||||
node.state()
|
||||
node.milestone().title()
|
||||
node.title()
|
||||
node.url()
|
||||
if typ == schema.PullRequest:
|
||||
node.merged()
|
||||
|
||||
def download_issues(endpoint, nodes):
|
||||
issues = [(repo, issue) for (repo, issue) in nodes if repo in REPOS]
|
||||
|
||||
ret = {}
|
||||
|
||||
# Ensure that any graph nodes from ZenHub that are not in the repos we care about have
|
||||
# default entries, to simplify subsequent graph manipulation code.
|
||||
for (repo, issue) in [(repo, issue) for (repo, issue) in nodes if repo not in REPOS]:
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, None)
|
||||
|
||||
def chunks(lst, n):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
|
||||
for issues in chunks(issues, 50):
|
||||
op = Operation(schema.Query)
|
||||
fetch_issues(op, issues)
|
||||
|
||||
d = endpoint(op)
|
||||
data = (op + d)
|
||||
|
||||
for (repo, issue) in issues:
|
||||
repo_data = data['repo%d' % repo]
|
||||
issue_key = 'issue%d' % issue
|
||||
# If GITHUB_TOKEN doesn't have permission to read from a particular private
|
||||
# repository in REPOS, GitHub returns an empty repo_data section.
|
||||
issue_data = repo_data[issue_key] if issue_key in repo_data else None
|
||||
ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data)
|
||||
|
||||
return ret
|
||||
|
||||
def fetch_workspace_graph(op, workspace_id, repos, cursor):
|
||||
dependencies = op.workspace(id=workspace_id).issue_dependencies(
|
||||
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
|
||||
# not the GitHub repo IDs (which the previous REST API used).
|
||||
# repository_ids=repos,
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
dependencies.nodes.id()
|
||||
dependencies.nodes.blocked_issue.number()
|
||||
dependencies.nodes.blocked_issue.repository.gh_id()
|
||||
dependencies.nodes.blocking_issue.number()
|
||||
dependencies.nodes.blocking_issue.repository.gh_id()
|
||||
dependencies.page_info.has_next_page()
|
||||
dependencies.page_info.end_cursor()
|
||||
|
||||
def get_dependency_graph(endpoint, workspace_id, repos):
|
||||
edges = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_workspace_graph(op, workspace_id, repos, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = (op + d)
|
||||
|
||||
dependencies = data.workspace.issue_dependencies
|
||||
edges += [
|
||||
(
|
||||
(node.blocking_issue.repository.gh_id, node.blocking_issue.number),
|
||||
(node.blocked_issue.repository.gh_id, node.blocked_issue.number),
|
||||
) for node in dependencies.nodes
|
||||
]
|
||||
|
||||
if dependencies.page_info.has_next_page:
|
||||
cursor = dependencies.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return nx.DiGraph(edges)
|
||||
|
||||
def fetch_epics(op, workspace_id, repos, cursor):
|
||||
epics = op.workspace(id=workspace_id).epics(
|
||||
# TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here,
|
||||
# not the GitHub repo IDs (which the previous REST API used).
|
||||
# repository_ids=repos,
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
epics.nodes.id()
|
||||
epics.nodes.issue.number()
|
||||
epics.nodes.issue.repository.gh_id()
|
||||
epics.page_info.has_next_page()
|
||||
epics.page_info.end_cursor()
|
||||
|
||||
def get_epics(endpoint, workspace_id, repos):
|
||||
epics = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_epics(op, workspace_id, repos, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = (op + d)
|
||||
|
||||
epics_page = data.workspace.epics
|
||||
epics += [
|
||||
(node.id, (node.issue.repository.gh_id, node.issue.number))
|
||||
for node in epics_page.nodes
|
||||
]
|
||||
|
||||
if epics_page.page_info.has_next_page:
|
||||
cursor = epics_page.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return epics
|
||||
|
||||
def fetch_epic_issues(op, workspace_id, epic_id, cursor):
|
||||
epic = op.workspace(id=workspace_id).epics(ids=[epic_id])
|
||||
child_issues = epic.nodes.child_issues(
|
||||
first=100,
|
||||
after=cursor,
|
||||
)
|
||||
child_issues.nodes.number()
|
||||
child_issues.nodes.repository.gh_id()
|
||||
child_issues.page_info.has_next_page()
|
||||
child_issues.page_info.end_cursor()
|
||||
|
||||
def get_epic_issues(endpoint, workspace_id, epic_id):
|
||||
epic_issues = []
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
op = Operation(zenhub_schema.Query)
|
||||
fetch_epic_issues(op, workspace_id, epic_id, cursor)
|
||||
|
||||
d = endpoint(op)
|
||||
data = (op + d)
|
||||
|
||||
epic = data.workspace.epics.nodes[0]
|
||||
epic_issues += [
|
||||
(node.repository.gh_id, node.number)
|
||||
for node in epic.child_issues.nodes
|
||||
]
|
||||
|
||||
if epic.child_issues.page_info.has_next_page:
|
||||
cursor = epic.child_issues.page_info.end_cursor
|
||||
print('.', end='', flush=True)
|
||||
else:
|
||||
print()
|
||||
break
|
||||
|
||||
return epic_issues
|
||||
|
||||
def main():
|
||||
gapi = HTTPEndpoint(
|
||||
'https://api.github.com/graphql',
|
||||
{'Authorization': 'bearer %s' % GITHUB_TOKEN},
|
||||
)
|
||||
zapi = HTTPEndpoint(
|
||||
'https://api.zenhub.com/public/graphql',
|
||||
{'Authorization': 'Bearer %s' % ZENHUB_TOKEN},
|
||||
)
|
||||
gapi = github.api(GITHUB_TOKEN)
|
||||
zapi = zenhub.api(ZENHUB_TOKEN)
|
||||
|
||||
# Build the full dependency graph from ZenHub's per-workspace API.
|
||||
print('Fetching graph')
|
||||
dg = nx.compose_all([
|
||||
get_dependency_graph(zapi, workspace_id, repos)
|
||||
zenhub.get_dependency_graph(zapi, workspace_id, repos)
|
||||
for (workspace_id, repos) in WORKSPACES.items()
|
||||
if len(repos) > 0
|
||||
])
|
||||
|
@ -412,10 +71,10 @@ def main():
|
|||
epics_issues = []
|
||||
for (workspace_id, repos) in WORKSPACES.items():
|
||||
if len(repos) > 0:
|
||||
epics_issues += get_epics(zapi, workspace_id, repos)
|
||||
epics_issues += zenhub.get_epics(zapi, workspace_id, repos)
|
||||
epics_issues = set(epics_issues)
|
||||
|
||||
epics_mapping = download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues])
|
||||
epics_mapping = github.download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues], REPOS)
|
||||
epics_mapping = {k: v for (k, v) in epics_mapping.items() if v.state != 'closed'}
|
||||
issues_by_epic = {}
|
||||
for (i, ((repo_id, epic_id), epic)) in enumerate(epics_mapping.items()):
|
||||
|
@ -428,7 +87,7 @@ def main():
|
|||
id for (id, gh_ref) in epics_issues
|
||||
if gh_ref == (repo_id, epic_id)
|
||||
][0]
|
||||
issues = set(get_epic_issues(zapi, workspace_id, epic_id))
|
||||
issues = set(zenhub.get_epic_issues(zapi, workspace_id, epic_id))
|
||||
issues_by_epic[epic] = issues
|
||||
for i in issues:
|
||||
# zapi.dependencies only returns nodes that have some connection,
|
||||
|
@ -448,7 +107,7 @@ def main():
|
|||
dg = nx.subgraph(dg, terminate_at.union(*ancestors))
|
||||
|
||||
# Fetch the issues within the graph.
|
||||
mapping = download_issues(gapi, dg.nodes)
|
||||
mapping = github.download_issues(gapi, dg.nodes, REPOS)
|
||||
|
||||
# Relabel the graph
|
||||
dg = nx.relabel_nodes(dg, mapping)
|
||||
|
|
Loading…
Reference in New Issue