zcashd/zcutil/make-release.py

795 lines
22 KiB
Python
Executable File

#!/usr/bin/env python3
import os
import re
import sys
import logging
import argparse
import subprocess
import traceback
import unittest
import random
from datetime import date, datetime, timedelta, timezone
from io import StringIO
from functools import wraps
def main(args=sys.argv[1:]):
"""
Perform the final Zcash release process up to the git tag.
"""
opts = parse_args(args)
chdir_to_repo(opts.REPO)
initialize_logging()
logging.debug('argv %r', sys.argv)
try:
main_logged(
opts.REVISION,
opts.RELEASE_VERSION,
opts.RELEASE_PREV,
opts.RELEASE_FROM,
opts.RELEASE_HEIGHT,
opts.HOTFIX,
)
except SystemExit as e:
logging.error(str(e))
raise SystemExit(1)
except:
logging.error(traceback.format_exc())
raise SystemExit(2)
def parse_args(args):
p = argparse.ArgumentParser(description=main.__doc__)
p.add_argument(
'--repo',
dest='REPO',
type=str,
help='Path to repository root.',
)
p.add_argument(
'--hotfix',
action='store_true',
dest='HOTFIX',
help='Use if this is a hotfix release from a non-master branch.',
)
p.add_argument(
'REVISION',
type=GitHash.parse_arg,
help='The git commit hash from which to construct the release.',
)
p.add_argument(
'RELEASE_VERSION',
type=Version.parse_arg,
help='The release version: vX.Y.Z',
)
p.add_argument(
'RELEASE_PREV',
type=Version.parse_arg,
help='The previously released version.',
)
p.add_argument(
'RELEASE_FROM',
type=Version.parse_arg,
help='The previously released non-beta non-RC version. May be the same as RELEASE_PREV.',
)
p.add_argument(
'RELEASE_HEIGHT',
type=int,
help='A block height approximately occurring on release day.',
)
return p.parse_args(args)
# Top-level flow:
def main_logged(revision, release, releaseprev, releasefrom, releaseheight, hotfix):
verify_dependencies([
('help2man', None),
('debchange', 'devscripts'),
])
verify_tags(revision, releaseprev, releasefrom)
verify_version(release, releaseprev, hotfix)
initialize_git(revision, release, hotfix)
verify_dependency_updates()
patch_version_in_files(release, releaseprev)
patch_release_height(releaseheight)
commit('Versioning changes for {}.'.format(release.novtext))
build()
gen_manpages()
commit('Updated manpages for {}.'.format(release.novtext))
gen_release_notes(release, releasefrom)
update_debian_changelog(release)
commit(
'Updated release notes and changelog for {}.'.format(
release.novtext,
),
)
update_book(release, releaseheight)
commit('Updated book for {}.'.format(release.novtext))
def phase(message):
def deco(f):
@wraps(f)
def g(*a, **kw):
logging.info('%s', message)
return f(*a, **kw)
return g
return deco
@phase('Checking release script dependencies.')
def verify_dependencies(dependencies):
for (dependency, pkg) in dependencies:
try:
sh_log(dependency, '--version')
except OSError:
raise SystemExit(
"Missing dependency {}{}".format(
dependency,
" (part of {} Debian package)".format(pkg) if pkg else "",
),
)
@phase('Checking dependency updates.')
def verify_dependency_updates():
try:
sh_log('./qa/zcash/updatecheck.py')
except SystemExit:
raise SystemExit("Dependency update check failed. Either some updates have not been correctly postponed, or the .updatecheck-token file is missing.")
@phase('Checking tags.')
def verify_tags(revision, releaseprev, releasefrom):
candidates = []
# Any tag beginning with a 'v' followed by [1-9] must be a version
# matching our Version parser. Tags beginning with v0 may exist from
# upstream and those do not follow our schema and are silently
# ignored. Any other tag is silently ignored.
candidatergx = re.compile('^v[1-9].*$')
for tag in sh_out('git', 'tag', '--list', '--merged', revision.value).splitlines():
if candidatergx.match(tag):
v = Version.parse(tag)
if v is not None:
candidates.append(v)
candidates.sort()
try:
latest = candidates[-1]
except IndexError:
raise SystemExit(
'No previous releases found by `git tag --list --merged {}`.'
.format(
revision.value
),
)
if releaseprev != latest:
raise SystemExit(
'The latest candidate in `git tag --list --merged {} is {} not {}'
.format(
revision.value,
latest.vtext,
releaseprev.vtext,
),
)
candidates.reverse()
prev_tags = []
for candidate in candidates:
if releasefrom == candidate:
break
else:
prev_tags.append(candidate)
else:
raise SystemExit(
'{} does not appear in `git tag --list --merged {}`'
.format(
releasefrom.vtext,
revision.value,
),
)
for tag in prev_tags:
if not tag.betarc:
raise SystemExit(
'{} appears to be a more recent non-beta non-RC release than {}'
.format(
tag.vtext,
releasefrom.vtext,
),
)
@phase('Checking version.')
def verify_version(release, releaseprev, hotfix):
if not hotfix:
return
expected = Version(
releaseprev.major,
releaseprev.minor,
releaseprev.patch + 1,
releaseprev.betarc,
None
)
if release != expected:
raise SystemExit(
"Expected {!r}, given {!r}".format(
expected, release,
),
)
@phase('Initializing git.')
def initialize_git(revision, release, hotfix):
junk = sh_out('git', 'status', '--porcelain')
if junk.strip():
raise SystemExit('There are uncommitted changes:\n' + junk)
branch = 'release-' + release.vtext
logging.info(
'Creating release branch {} from revision {}.'
.format(
branch,
revision.value
)
)
sh_log('git', 'checkout', '-b', branch, revision.value)
return branch
@phase('Patching versioning in files.')
def patch_version_in_files(release, releaseprev):
patch_README(release, releaseprev)
patch_clientversion_h(release)
patch_configure_ac(release)
patch_gitian_linux_yml(release, releaseprev, 'contrib/gitian-descriptors/gitian-linux.yml')
patch_gitian_linux_yml(release, releaseprev, 'contrib/gitian-descriptors/gitian-linux-parallel.yml')
@phase('Patching release height for end-of-support halt.')
def patch_release_height(releaseheight):
rgx = re.compile(
r'^(static const int APPROX_RELEASE_HEIGHT = )\d+(;)$',
)
with PathPatcher('src/deprecation.h') as (inf, outf):
for line in inf:
m = rgx.match(line)
if m is None:
outf.write(line)
else:
[prefix, suffix] = m.groups()
outf.write(
'{}{}{}\n'.format(
prefix,
releaseheight,
suffix,
),
)
@phase('Building...')
def build():
base_dir = os.getcwd()
depends_dir = os.path.join(base_dir, 'depends')
src_dir = os.path.join(base_dir, 'src')
nproc = sh_out('nproc').strip()
sh_progress([
'Staging boost...',
'Staging libevent...',
'Staging zeromq...',
'Staging libsodium...',
"Leaving directory '%s'" % depends_dir,
'config.status: creating libzcash_script.pc',
"Entering directory '%s'" % src_dir,
'httpserver.cpp',
'torcontrol.cpp',
'gtest/test_tautology.cpp',
'gtest/test_metrics.cpp',
'test/equihash_tests.cpp',
'test/util_tests.cpp',
"Leaving directory '%s'" % src_dir,
], './zcutil/build.sh', '-j', nproc)
@phase('Generating manpages.')
def gen_manpages():
sh_log('./contrib/devtools/gen-manpages.sh')
@phase('Generating release notes.')
def gen_release_notes(release, releasefrom):
release_notes = [
'./zcutil/release-notes.py',
'--version',
release.novtext,
'--prev',
releasefrom.vtext,
]
if not release.betarc:
release_notes.append('--clear')
sh_log(*release_notes)
sh_log(
'git',
'add',
'./doc/authors.md',
'./doc/release-notes/release-notes-{}.md'.format(release.novtext),
)
@phase('Updating debian changelog.')
def update_debian_changelog(release):
os.environ['DEBEMAIL'] = 'team@electriccoin.co'
os.environ['DEBFULLNAME'] = 'Electric Coin Company'
sh_log(
'debchange',
'--newversion', release.debversion,
'--distribution', 'stable',
'--changelog', './contrib/debian/changelog',
'{} release.'.format(release.novtext),
)
@phase('Updating book.')
def update_book(release, releaseheight):
patch_book_release_support(release, releaseheight)
# Helper code:
def commit(message):
logging.info('Committing: %r', message)
fullmsg = 'make-release.py: {}'.format(message)
sh_log('git', 'commit', '--all', '-m', fullmsg)
def chdir_to_repo(repo):
if repo is None:
dn = os.path.dirname
repo = dn(dn(os.path.abspath(sys.argv[0])))
os.chdir(repo)
def patch_README(release, releaseprev):
with PathPatcher('README.md') as (inf, outf):
firstline = inf.readline()
assert firstline == 'Zcash {}\n'.format(releaseprev.novtext), \
repr(firstline)
outf.write('Zcash {}\n'.format(release.novtext))
outf.write(inf.read())
def patch_clientversion_h(release):
_patch_build_defs(
release,
'src/clientversion.h',
(r'^(#define CLIENT_VERSION_(MAJOR|MINOR|REVISION|BUILD|IS_RELEASE))'
r' \d+()$'),
)
def patch_configure_ac(release):
_patch_build_defs(
release,
'configure.ac',
(r'^(define\(_CLIENT_VERSION_(MAJOR|MINOR|REVISION|BUILD|IS_RELEASE),)'
r' \d+(\))$'),
)
def patch_gitian_linux_yml(release, releaseprev, path):
with PathPatcher(path) as (inf, outf):
outf.write(inf.readline())
secondline = inf.readline()
assert secondline == 'name: "zcash-{}"\n'.format(
releaseprev.novtext
), repr(secondline)
outf.write('name: "zcash-{}"\n'.format(release.novtext))
outf.write(inf.read())
def patch_book_release_support(release, releaseheight):
with PathPatcher('doc/book/src/user/release-support.md') as (inf, outf):
# Find the start marker.
cur_line = inf.readline()
while not 'RELEASE_SCRIPT_START_MARKER' in cur_line:
outf.write(cur_line)
cur_line = inf.readline()
outf.write(cur_line)
# The next two lines are the table heading.
for _ in range(2):
outf.write(inf.readline())
# The remaining lines before the end marker are table rows.
table_rows = []
cur_line = inf.readline()
while not 'RELEASE_SCRIPT_END_MARKER' in cur_line:
[row_ver, row_released, row_halt, row_eos] = cur_line.strip('| \n').split(' | ')
row_released = date.fromisoformat(row_released)
row_eos = date.fromisoformat(row_eos)
table_rows.append((row_ver, row_released, int(row_halt), row_eos))
cur_line = inf.readline()
# Prune rows for releases that have reached EoS.
today = datetime.now(timezone.utc).date()
table_rows = [row for row in table_rows if row[3] >= today]
# Add a row for this release.
with open('src/deprecation.h', 'r', encoding='utf8') as f:
prefix = 'RELEASE_TO_DEPRECATION_WEEKS = '
for line in f:
if prefix in line:
weeks_to_eos = int(line.split(prefix)[1].split(';')[0])
eos = today + timedelta(weeks=weeks_to_eos)
table_rows.append((release.novtext, today, releaseheight, eos))
# Write out the updated table rows.
for row in table_rows:
outf.write('| %s | %s | %s | %s |\n' % row)
# Write out the end marker and the rest of the page.
outf.write(cur_line)
outf.write(inf.read())
def _patch_build_defs(release, path, pattern):
rgx = re.compile(pattern)
with PathPatcher(path) as (inf, outf):
for line in inf:
m = rgx.match(line)
if m:
prefix, label, suffix = m.groups()
repl = {
'MAJOR': release.major,
'MINOR': release.minor,
'REVISION': release.patch,
'BUILD': release.build,
'IS_RELEASE': (
'false' if release.build < 50 else 'true'
),
}[label]
outf.write('{} {}{}\n'.format(prefix, repl, suffix))
else:
outf.write(line)
def initialize_logging():
logname = './zcash-make-release.log'
fmtr = logging.Formatter(
'%(asctime)s L%(lineno)-4d %(levelname)-5s | %(message)s',
'%Y-%m-%d %H:%M:%S'
)
hout = logging.StreamHandler(sys.stdout)
hout.setLevel(logging.INFO)
hout.setFormatter(fmtr)
hpath = logging.FileHandler(logname, mode='a')
hpath.setLevel(logging.DEBUG)
hpath.setFormatter(fmtr)
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.addHandler(hout)
root.addHandler(hpath)
logging.info('zcash make-release.py debug log: %r', logname)
def sh_out(*args):
logging.debug('Run (out): %r', args)
return subprocess.check_output(args).decode()
def sh_log(*args):
PIPE = subprocess.PIPE
STDOUT = subprocess.STDOUT
try:
p = subprocess.Popen(args, stdout=PIPE, stderr=STDOUT, stdin=None)
except OSError:
logging.error('Error launching %r...', args)
raise
logging.debug('Run (log PID %r): %r', p.pid, args)
for line in p.stdout:
logging.debug('> %s', line.decode().rstrip())
status = p.wait()
if status != 0:
raise SystemExit('Nonzero exit status: {!r}'.format(status))
def sh_progress(markers, *args):
try:
import progressbar
except:
sh_log(*args)
return
PIPE = subprocess.PIPE
STDOUT = subprocess.STDOUT
try:
p = subprocess.Popen(args, stdout=PIPE, stderr=STDOUT, stdin=None)
except OSError:
logging.error('Error launching %r...', args)
raise
pbar = progressbar.ProgressBar(max_value=len(markers))
marker = 0
pbar.update(marker)
logging.debug('Run (log PID %r): %r', p.pid, args)
for line in p.stdout:
line = line.decode()
logging.debug('> %s', line.rstrip())
for idx, val in enumerate(markers[marker:]):
if val in line:
marker += idx + 1
pbar.update(marker)
break
pbar.finish()
status = p.wait()
if status != 0:
raise SystemExit('Nonzero exit status: {!r}'.format(status))
class GitHash (object):
'''A git commit hash.'''
RGX = re.compile(
r'^([0-9a-f]{10,40})$',
)
@staticmethod
def parse_arg(text):
m = GitHash.RGX.match(text)
if m is None:
raise argparse.ArgumentTypeError(
'Could not parse revision {!r} against regex {}'.format(
text,
GitHash.RGX.pattern,
),
)
else:
assert len(m.groups()) == 1
[value] = m.groups()
return GitHash(value)
def __init__(self, value):
assert GitHash.RGX.match(value) is not None
self.value = value
class Version (object):
'''A release version.'''
RGX = re.compile(
r'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(beta|rc)?([1-9]\d*))?$',
)
@staticmethod
def parse(text):
m = Version.RGX.match(text)
if m is None:
return None
else:
[major, minor, patch, _, betarc, hyphen] = m.groups()
return Version(
int(major),
int(minor),
int(patch),
betarc,
int(hyphen) if hyphen is not None else None,
)
@staticmethod
def parse_arg(text):
v = Version.parse(text)
if v is None:
raise argparse.ArgumentTypeError(
'Could not parse version {!r} against regex {}'.format(
text,
Version.RGX.pattern,
),
)
else:
return v
def __init__(self, major, minor, patch, betarc, hyphen):
for i in [major, minor, patch]:
assert type(i) is int, i
assert betarc in {None, 'rc', 'beta'}, betarc
assert hyphen is None or type(hyphen) is int, hyphen
if betarc is not None:
assert hyphen is not None, (betarc, hyphen)
self.major = major
self.minor = minor
self.patch = patch
self.betarc = betarc
self.hyphen = hyphen
if hyphen is None:
self.build = 50
else:
assert hyphen > 0, hyphen
if betarc is None:
assert hyphen < 50, hyphen
self.build = 50 + hyphen
else:
assert hyphen < 26, hyphen
self.build = {'beta': 0, 'rc': 25}[betarc] + hyphen - 1
@property
def novtext(self):
return self._novtext(debian=False)
@property
def vtext(self):
return 'v' + self.novtext
@property
def debversion(self):
return self._novtext(debian=True)
def _novtext(self, debian):
novtext = '{}.{}.{}'.format(self.major, self.minor, self.patch)
if self.hyphen is None:
return novtext
else:
assert self.hyphen > 0, self.hyphen
if self.betarc is None:
assert self.hyphen < 50, self.hyphen
sep = '+' if debian else '-'
return '{}{}{}'.format(novtext, sep, self.hyphen)
else:
assert self.hyphen < 26, self.hyphen
sep = '~' if debian else '-'
return '{}{}{}{}'.format(
novtext,
sep,
self.betarc,
self.hyphen,
)
def __repr__(self):
return '<Version {}>'.format(self.vtext)
def _sort_tup(self):
if self.hyphen is None:
prio = 2
else:
prio = {'beta': 0, 'rc': 1, None: 3}[self.betarc]
return (
self.major,
self.minor,
self.patch,
prio,
self.hyphen,
)
def __lt__(self, other):
return self._sort_tup() < other._sort_tup()
def __eq__(self, other):
return self._sort_tup() == other._sort_tup()
class PathPatcher (object):
def __init__(self, path):
self._path = path
def __enter__(self):
logging.debug('Patching %r', self._path)
self._inf = open(self._path, 'r', encoding='utf8')
self._outf = StringIO()
return (self._inf, self._outf)
def __exit__(self, et, ev, tb):
if (et, ev, tb) == (None, None, None):
self._inf.close()
with open(self._path, 'w', encoding='utf8') as f:
f.write(self._outf.getvalue())
# Unit Tests
class TestVersion (unittest.TestCase):
ValidVersionsAndBuilds = [
# These are taken from: git tag --list | grep '^v1'
('v1.0.0-beta1', 0),
('v1.0.0-beta2', 1),
('v1.0.0-rc1', 25),
('v1.0.0-rc2', 26),
('v1.0.0-rc3', 27),
('v1.0.0-rc4', 28),
('v1.0.0', 50),
('v1.0.1', 50),
('v1.0.2', 50),
('v1.0.3', 50),
('v1.0.4', 50),
('v1.0.5', 50),
('v1.0.6', 50),
('v1.0.7-1', 51),
('v1.0.8', 50),
('v1.0.8-1', 51),
('v1.0.9', 50),
('v1.0.10', 50),
('v7.42.1000', 50),
]
ValidVersions = [
v
for (v, _)
in ValidVersionsAndBuilds
]
def test_arg_parse_and_vtext_identity(self):
for case in self.ValidVersions:
v = Version.parse_arg(case)
self.assertEqual(v.vtext, case)
def test_rev_parse(self):
sample = '958bcf2dac6d81d17797c0f58f176262a496cfd4'
rev = GitHash.parse_arg(sample)
self.assertEqual(rev.value, sample)
def test_arg_parse_negatives(self):
cases = [
'v07.0.0',
'v1.0.03',
'v1.2.3-0',
'v1.2.3-foobar',
'v1.2.3~0',
'v1.2.3+0',
'1.2.3',
]
for case in cases:
self.assertRaises(
argparse.ArgumentTypeError,
Version.parse_arg,
case,
)
def test_version_sort(self):
expected = [Version.parse_arg(v) for v in self.ValidVersions]
rng = random.Random()
rng.seed(0)
for _ in range(1024):
vec = list(expected)
rng.shuffle(vec)
vec.sort()
self.assertEqual(vec, expected)
def test_build_nums(self):
for (text, expected) in self.ValidVersionsAndBuilds:
version = Version.parse_arg(text)
self.assertEqual(version.build, expected)
if __name__ == '__main__':
if len(sys.argv) == 2 and sys.argv[1] == '--help':
main()
else:
actualargs = sys.argv
sys.argv = [sys.argv[0], '--verbose']
print('=== Self Test ===')
try:
unittest.main(verbosity=2)
except SystemExit as e:
if e.args[0] != 0:
raise
sys.argv = actualargs
print('=== Running ===')
main()