reproducible builds CI integration (#4337)
Enable reproducible builds on master. Overhaul build script, update docs accordingly.
This commit is contained in:
Alessio Treglia
GitHub
parent
e09ebf9ffa
commit
4b7d295eca
|
|
@ -370,6 +370,31 @@ jobs:
|
|||
docker login --password-stdin -u $DOCKER_USER <<<$DOCKER_PASS
|
||||
docker push tendermint/gaia:$CIRCLE_TAG
|
||||
|
||||
reproducible_builds:
|
||||
<<: *linux_defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- checkout
|
||||
- setup_remote_docker:
|
||||
docker_layer_caching: true
|
||||
- run:
|
||||
name: Build gaia
|
||||
no_output_timeout: 20m
|
||||
command: |
|
||||
sudo apt-get install -y ruby
|
||||
bash -x ./cmd/gaia/contrib/gitian-build.sh all
|
||||
for os in darwin linux windows; do
|
||||
cp gitian-build-${os}/result/gaia-${os}-res.yml .
|
||||
rm -rf gitian-build-${os}/
|
||||
done
|
||||
- store_artifacts:
|
||||
path: /go/src/github.com/cosmos/cosmos-sdk/gaia-darwin-res.yml
|
||||
- store_artifacts:
|
||||
path: /go/src/github.com/cosmos/cosmos-sdk/gaia-linux-res.yml
|
||||
- store_artifacts:
|
||||
path: /go/src/github.com/cosmos/cosmos-sdk/gaia-windows-res.yml
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
test-suite:
|
||||
|
|
@ -441,3 +466,10 @@ workflows:
|
|||
- upload_coverage:
|
||||
requires:
|
||||
- test_cover
|
||||
- reproducible_builds:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
requires:
|
||||
- setup_dependencies
|
||||
|
|
|
|||
|
|
@ -7,11 +7,10 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
THIS="${THIS_DIR}/$(basename ${BASH_SOURCE[0]})"
|
||||
GITIAN_CACHE_DIRNAME='.gitian-builder-cache'
|
||||
GO_DEBIAN_RELEASE='1.12.5-1'
|
||||
GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz"
|
||||
GO_TARBALL_URL="https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}/${GO_TARBALL}"
|
||||
|
||||
# Defaults
|
||||
|
||||
|
|
@ -19,6 +18,7 @@ DEFAULT_SIGN_COMMAND='gpg --detach-sign'
|
|||
DEFAULT_GAIA_SIGS=${GAIA_SIGS:-'gaia.sigs'}
|
||||
DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder'
|
||||
DEFAULT_GBUILD_FLAGS=''
|
||||
DEFAULT_SIGS_REPO='https://github.com/cosmos/gaia.sigs'
|
||||
|
||||
# Overrides
|
||||
|
||||
|
|
@ -30,147 +30,27 @@ GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}}
|
|||
|
||||
g_workdir=''
|
||||
g_gitian_cache=''
|
||||
g_cached_gitian=''
|
||||
g_cached_go_tarball=''
|
||||
g_sign_identity=''
|
||||
g_gitian_skip_download=''
|
||||
g_sigs_dir=''
|
||||
g_flag_commit=''
|
||||
|
||||
f_main() {
|
||||
local l_dirname \
|
||||
l_sdk \
|
||||
l_commit \
|
||||
l_platform \
|
||||
l_result \
|
||||
l_descriptor \
|
||||
l_release \
|
||||
l_sigs_dir
|
||||
|
||||
l_platform=$1
|
||||
l_sdk=$2
|
||||
l_sigs_dir=$3
|
||||
|
||||
pushd ${l_sdk}
|
||||
l_commit="$(git rev-parse HEAD)"
|
||||
l_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${l_platform}"
|
||||
popd
|
||||
|
||||
l_descriptor=${THIS_DIR}/gitian-descriptors/gitian-${l_platform}.yml
|
||||
[ -f ${l_descriptor} ]
|
||||
|
||||
if [ "${g_gitian_skip_download}" != "y" ]; then
|
||||
echo "Cloning ${GITIAN_REPO} to ${g_workdir}" >&2
|
||||
git clone ${GITIAN_REPO} ${g_workdir}
|
||||
fi
|
||||
|
||||
echo "Fetching Go sources" >&2
|
||||
f_ensure_go_source_tarball
|
||||
|
||||
echo "Prepare gitian-target docker image" >&2
|
||||
f_prep_docker_image
|
||||
|
||||
echo "Start the build" >&2
|
||||
f_build "${l_descriptor}" "${l_commit}"
|
||||
echo "You may find the result in $(echo ${g_workdir}/result/*.yml)" >&2
|
||||
|
||||
if [ -n "${g_sign_identity}" ]; then
|
||||
f_sign "${l_descriptor}" "${l_release}" "${l_sigs_dir}"
|
||||
echo "Build signed as ${g_sign_identity}, signatures can be found in ${l_sigs_dir}"
|
||||
f_verify "${l_descriptor}" "${l_release}" "${l_sigs_dir}"
|
||||
echo "Signatures in ${l_sigs_dir} have been verified"
|
||||
else
|
||||
echo "You can now sign the build with the following command:" >&2
|
||||
echo "cd ${g_workdir} ; bin/gsign -p 'gpg --detach-sign' -s GPG_IDENTITY --release=${l_release} ${l_descriptor}" >&2
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
f_prep_docker_image() {
|
||||
pushd ${g_workdir}
|
||||
bin/make-base-vm --docker --suite bionic --arch amd64
|
||||
popd
|
||||
}
|
||||
|
||||
f_ensure_go_source_tarball() {
|
||||
local l_cached_tar
|
||||
|
||||
l_cached_tar="${g_gitian_cache}/${GO_TARBALL}"
|
||||
mkdir -p ${g_workdir}/inputs
|
||||
if [ -f "${l_cached_tar}" ]; then
|
||||
echo "Fetching cached tarball from ${l_cached_tar}" >&2
|
||||
cp "${l_cached_tar}" ${g_workdir}/inputs/${GO_TARBALL}
|
||||
else
|
||||
f_download_go
|
||||
echo "Caching ${GO_TARBALL}" >&2
|
||||
cp ${g_workdir}/inputs/${GO_TARBALL} "${l_cached_tar}"
|
||||
fi
|
||||
}
|
||||
|
||||
f_download_go() {
|
||||
local l_remote
|
||||
|
||||
l_remote=https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}
|
||||
curl -L "${l_remote}/${GO_TARBALL}" > ${g_workdir}/inputs/${GO_TARBALL}
|
||||
}
|
||||
|
||||
f_build() {
|
||||
local l_sdk l_descriptor
|
||||
|
||||
l_descriptor=$1
|
||||
l_commit=$2
|
||||
|
||||
[ -f ${l_descriptor} ]
|
||||
|
||||
cd ${g_workdir}
|
||||
export USE_DOCKER=1
|
||||
bin/gbuild --commit cosmos-sdk="$l_commit" ${GBUILD_FLAGS} "$l_descriptor"
|
||||
libexec/stop-target || echo "warning: couldn't stop target" >&2
|
||||
}
|
||||
|
||||
f_sign() {
|
||||
local l_descriptor l_release_name l_sigs_dir
|
||||
|
||||
l_descriptor=$1
|
||||
l_release_name=$2
|
||||
l_sigs_dir=$3
|
||||
|
||||
pushd ${g_workdir}
|
||||
bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${l_sigs_dir}" --release=${l_release_name} ${l_descriptor}
|
||||
popd
|
||||
}
|
||||
|
||||
f_verify() {
|
||||
local l_descriptor l_release_name l_sigs_dir
|
||||
|
||||
l_descriptor=$1
|
||||
l_release_name=$2
|
||||
l_sigs_dir=$3
|
||||
|
||||
pushd ${g_workdir}
|
||||
bin/gverify --destination="${l_sigs_dir}" --release="${l_release_name}" ${l_descriptor}
|
||||
popd
|
||||
}
|
||||
|
||||
f_validate_platform() {
|
||||
case "${1}" in
|
||||
linux|darwin|windows)
|
||||
;;
|
||||
*)
|
||||
echo "invalid platform -- ${1}"
|
||||
exit 1
|
||||
esac
|
||||
}
|
||||
|
||||
f_abspath() {
|
||||
echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")"
|
||||
}
|
||||
|
||||
f_help() {
|
||||
cat >&2 <<EOF
|
||||
Usage: $(basename $0) [-h] GOOS GIT_REPO
|
||||
Launch a gitian build from the local clone of cosmos-sdk available at GIT_REPO.
|
||||
Usage: $(basename $0) [-h] PLATFORM
|
||||
Launch a gitian build from the current source directory for the given PLATFORM.
|
||||
The following platforms are supported:
|
||||
darwin
|
||||
linux
|
||||
windows
|
||||
all
|
||||
|
||||
Options:
|
||||
-h display this help and exit
|
||||
-d DIRNAME set working directory name and skip gitian-builder download
|
||||
-c clone the signatures repository and commit signatures;
|
||||
ignored if sign identity is not supplied
|
||||
-s IDENTITY sign build as IDENTITY
|
||||
|
||||
If a GPG identity is supplied via the -s flag, the build will be signed and verified.
|
||||
|
|
@ -183,34 +63,139 @@ variable \$SIGN_COMMAND.
|
|||
EOF
|
||||
}
|
||||
|
||||
while getopts ":d:s:h" opt; do
|
||||
|
||||
f_builddir() {
|
||||
printf '%s' "${g_workdir}/gitian-build-$1"
|
||||
}
|
||||
|
||||
f_prep_build() {
|
||||
local l_platforms \
|
||||
l_os \
|
||||
l_builddir
|
||||
|
||||
l_platforms="$1"
|
||||
|
||||
if [ -n "${g_flag_commit}" -a ! -d "${g_sigs_dir}" ]; then
|
||||
git clone ${DEFAULT_SIGS_REPO} "${g_sigs_dir}"
|
||||
fi
|
||||
|
||||
for l_os in ${l_platforms}; do
|
||||
l_builddir="$(f_builddir ${l_os})"
|
||||
|
||||
f_echo_stderr "Preparing build directory $(basename ${l_builddir}), restoring files from cache"
|
||||
cp -ar "${g_cached_gitian}" "${l_builddir}" >&2
|
||||
mkdir "${l_builddir}/inputs/"
|
||||
cp -v "${g_cached_go_tarball}" "${l_builddir}/inputs/"
|
||||
done
|
||||
}
|
||||
|
||||
f_build() {
|
||||
local l_descriptor
|
||||
|
||||
l_descriptor=$1
|
||||
|
||||
bin/gbuild --commit cosmos-sdk="$g_commit" ${GBUILD_FLAGS} "$l_descriptor"
|
||||
libexec/stop-target || f_echo_stderr "warning: couldn't stop target"
|
||||
}
|
||||
|
||||
f_sign_verify() {
|
||||
local l_descriptor
|
||||
|
||||
l_descriptor=$1
|
||||
|
||||
bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${g_sigs_dir}" --release=${g_release} ${l_descriptor}
|
||||
bin/gverify --destination="${g_sigs_dir}" --release="${g_release}" ${l_descriptor}
|
||||
}
|
||||
|
||||
f_commit_sig() {
|
||||
local l_release_name
|
||||
|
||||
l_release_name=$1
|
||||
|
||||
pushd "${g_sigs_dir}"
|
||||
git add . || echo "git add failed" >&2
|
||||
git commit -m "Add ${l_release_name} reproducible build" || echo "git commit failed" >&2
|
||||
popd
|
||||
}
|
||||
|
||||
f_prep_docker_image() {
|
||||
pushd $1
|
||||
bin/make-base-vm --docker --suite bionic --arch amd64
|
||||
popd
|
||||
}
|
||||
|
||||
f_ensure_cache() {
|
||||
g_gitian_cache="${g_workdir}/${GITIAN_CACHE_DIRNAME}"
|
||||
[ -d "${g_gitian_cache}" ] || mkdir "${g_gitian_cache}"
|
||||
|
||||
g_cached_go_tarball="${g_gitian_cache}/${GO_TARBALL}"
|
||||
if [ ! -f "${g_cached_go_tarball}" ]; then
|
||||
f_echo_stderr "${g_cached_go_tarball}: cache miss, caching..."
|
||||
curl -L "${GO_TARBALL_URL}" --output "${g_cached_go_tarball}"
|
||||
fi
|
||||
|
||||
g_cached_gitian="${g_gitian_cache}/gitian-builder"
|
||||
if [ ! -d "${g_cached_gitian}" ]; then
|
||||
f_echo_stderr "${g_cached_gitian}: cache miss, caching..."
|
||||
git clone ${GITIAN_REPO} "${g_cached_gitian}"
|
||||
fi
|
||||
}
|
||||
|
||||
f_demangle_platforms() {
|
||||
case "${1}" in
|
||||
all)
|
||||
printf '%s' 'darwin linux windows' ;;
|
||||
linux|darwin|windows)
|
||||
printf '%s' "${1}" ;;
|
||||
*)
|
||||
echo "invalid platform -- ${1}"
|
||||
exit 1
|
||||
esac
|
||||
}
|
||||
|
||||
f_echo_stderr() {
|
||||
echo $@ >&2
|
||||
}
|
||||
|
||||
|
||||
while getopts ":cs:h" opt; do
|
||||
case "${opt}" in
|
||||
h) f_help ; exit 0 ;;
|
||||
d) g_dirname="${OPTARG}" ;;
|
||||
c) g_flag_commit=y ;;
|
||||
s) g_sign_identity="${OPTARG}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
shift "$((OPTIND-1))"
|
||||
|
||||
g_platform="${1}"
|
||||
f_validate_platform "${g_platform}"
|
||||
g_platforms=$(f_demangle_platforms "${1}")
|
||||
g_workdir="$(pwd)"
|
||||
g_commit="$(git rev-parse HEAD)"
|
||||
g_sigs_dir=${GAIA_SIGS:-"${g_workdir}/${DEFAULT_GAIA_SIGS}"}
|
||||
|
||||
g_dirname="${g_dirname:-gitian-build-${g_platform}}"
|
||||
g_workdir="$(pwd)/${g_dirname}"
|
||||
if [ -d "${g_workdir}" ]; then
|
||||
echo "Directory ${g_workdir} exists and will be preserved" >&2
|
||||
g_gitian_skip_download=y
|
||||
fi
|
||||
f_ensure_cache
|
||||
|
||||
g_sdk="$(f_abspath ${2})"
|
||||
[ -d "${g_sdk}" ]
|
||||
f_prep_docker_image "${g_cached_gitian}"
|
||||
|
||||
g_sigs_dir=${GAIA_SIGS:-"$(pwd)/${DEFAULT_GAIA_SIGS}"}
|
||||
f_prep_build "${g_platforms}"
|
||||
|
||||
# create local cache directory for all gitian builds
|
||||
g_gitian_cache="$(pwd)/${GITIAN_CACHE_DIRNAME}"
|
||||
echo "Ensure cache directory ${g_gitian_cache} exists" >&2
|
||||
mkdir -p "${g_gitian_cache}"
|
||||
export USE_DOCKER=1
|
||||
for g_os in ${g_platforms}; do
|
||||
g_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${g_os}"
|
||||
g_descriptor="${g_workdir}/cmd/gaia/contrib/gitian-descriptors/gitian-${g_os}.yml"
|
||||
[ -f ${g_descriptor} ]
|
||||
g_builddir="$(f_builddir ${g_os})"
|
||||
|
||||
f_main "${g_platform}" "${g_sdk}" "${g_sigs_dir}"
|
||||
pushd "${g_builddir}"
|
||||
f_build "${g_descriptor}"
|
||||
if [ -n "${g_sign_identity}" ]; then
|
||||
f_sign_verify "${g_descriptor}"
|
||||
fi
|
||||
popd
|
||||
|
||||
if [ -n "${g_sign_identity}" -a -n "${g_flag_commit}" ]; then
|
||||
[ -d "${g_sigs_dir}/.git/" ] && f_commit_sig ${g_release} || f_echo_stderr "couldn't commit, ${g_sigs_dir} is not a git clone"
|
||||
fi
|
||||
done
|
||||
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ Run the following command to launch a build for `linux` and sign the final build
|
|||
report (replace `user@example.com` with the GPG identity you want to sign the report with):
|
||||
|
||||
```
|
||||
./cmd/gaia/contrib/gitian-build.sh -s user@example.com linux `pwd`
|
||||
./cmd/gaia/contrib/gitian-build.sh -s user@example.com linux
|
||||
```
|
||||
|
||||
The above command generates two directories in the current working directory:
|
||||
|
|
@ -57,7 +57,7 @@ for platform in darwin linux windows; do ./cmd/gaia/contrib/gitian-build.sh -s u
|
|||
If you want to generate unsigned builds, just remove the option `-s` from the command line:
|
||||
|
||||
```
|
||||
./cmd/gaia/contrib/gitian-build.sh linux `pwd`
|
||||
./cmd/gaia/contrib/gitian-build.sh linux
|
||||
```
|
||||
|
||||
At the end of the procedure, build results can be found in the `./gaia.sigs` directory:
|
||||
|
|
|
|||
Loading…
Reference in New Issue