From 4b7d295ecab7ac6d04665c6fdf776f0256ff7c95 Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Thu, 16 May 2019 19:05:56 +0100 Subject: [PATCH] reproducible builds CI integration (#4337) Enable reproducible builds on master. Overhaul build script, update docs accordingly. --- .circleci/config.yml | 32 +++ cmd/gaia/contrib/gitian-build.sh | 291 ++++++++++++------------- docs/cosmos-hub/reproducible-builds.md | 4 +- 3 files changed, 172 insertions(+), 155 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21a0d8d15..6dde65ea4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -370,6 +370,31 @@ jobs: docker login --password-stdin -u $DOCKER_USER <<<$DOCKER_PASS docker push tendermint/gaia:$CIRCLE_TAG + reproducible_builds: + <<: *linux_defaults + steps: + - attach_workspace: + at: /tmp/workspace + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build gaia + no_output_timeout: 20m + command: | + sudo apt-get install -y ruby + bash -x ./cmd/gaia/contrib/gitian-build.sh all + for os in darwin linux windows; do + cp gitian-build-${os}/result/gaia-${os}-res.yml . + rm -rf gitian-build-${os}/ + done + - store_artifacts: + path: /go/src/github.com/cosmos/cosmos-sdk/gaia-darwin-res.yml + - store_artifacts: + path: /go/src/github.com/cosmos/cosmos-sdk/gaia-linux-res.yml + - store_artifacts: + path: /go/src/github.com/cosmos/cosmos-sdk/gaia-windows-res.yml + workflows: version: 2 test-suite: @@ -441,3 +466,10 @@ workflows: - upload_coverage: requires: - test_cover + - reproducible_builds: + filters: + branches: + only: + - master + requires: + - setup_dependencies diff --git a/cmd/gaia/contrib/gitian-build.sh b/cmd/gaia/contrib/gitian-build.sh index eb1d65808..22e6d5133 100755 --- a/cmd/gaia/contrib/gitian-build.sh +++ b/cmd/gaia/contrib/gitian-build.sh @@ -7,11 +7,10 @@ set -euo pipefail -THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -THIS="${THIS_DIR}/$(basename ${BASH_SOURCE[0]})" GITIAN_CACHE_DIRNAME='.gitian-builder-cache' GO_DEBIAN_RELEASE='1.12.5-1' GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz" +GO_TARBALL_URL="https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}/${GO_TARBALL}" # Defaults @@ -19,6 +18,7 @@ DEFAULT_SIGN_COMMAND='gpg --detach-sign' DEFAULT_GAIA_SIGS=${GAIA_SIGS:-'gaia.sigs'} DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder' DEFAULT_GBUILD_FLAGS='' +DEFAULT_SIGS_REPO='https://github.com/cosmos/gaia.sigs' # Overrides @@ -30,147 +30,27 @@ GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}} g_workdir='' g_gitian_cache='' +g_cached_gitian='' +g_cached_go_tarball='' g_sign_identity='' -g_gitian_skip_download='' +g_sigs_dir='' +g_flag_commit='' -f_main() { - local l_dirname \ - l_sdk \ - l_commit \ - l_platform \ - l_result \ - l_descriptor \ - l_release \ - l_sigs_dir - - l_platform=$1 - l_sdk=$2 - l_sigs_dir=$3 - - pushd ${l_sdk} - l_commit="$(git rev-parse HEAD)" - l_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${l_platform}" - popd - - l_descriptor=${THIS_DIR}/gitian-descriptors/gitian-${l_platform}.yml - [ -f ${l_descriptor} ] - - if [ "${g_gitian_skip_download}" != "y" ]; then - echo "Cloning ${GITIAN_REPO} to ${g_workdir}" >&2 - git clone ${GITIAN_REPO} ${g_workdir} - fi - - echo "Fetching Go sources" >&2 - f_ensure_go_source_tarball - - echo "Prepare gitian-target docker image" >&2 - f_prep_docker_image - - echo "Start the build" >&2 - f_build "${l_descriptor}" "${l_commit}" - echo "You may find the result in $(echo ${g_workdir}/result/*.yml)" >&2 - - if [ -n "${g_sign_identity}" ]; then - f_sign "${l_descriptor}" "${l_release}" "${l_sigs_dir}" - echo "Build signed as ${g_sign_identity}, signatures can be found in ${l_sigs_dir}" - f_verify "${l_descriptor}" "${l_release}" "${l_sigs_dir}" - echo "Signatures in ${l_sigs_dir} have been verified" - else - echo "You can now sign the build with the following command:" >&2 - echo "cd ${g_workdir} ; bin/gsign -p 'gpg --detach-sign' -s GPG_IDENTITY --release=${l_release} ${l_descriptor}" >&2 - fi - - return 0 -} - -f_prep_docker_image() { - pushd ${g_workdir} - bin/make-base-vm --docker --suite bionic --arch amd64 - popd -} - -f_ensure_go_source_tarball() { - local l_cached_tar - - l_cached_tar="${g_gitian_cache}/${GO_TARBALL}" - mkdir -p ${g_workdir}/inputs - if [ -f "${l_cached_tar}" ]; then - echo "Fetching cached tarball from ${l_cached_tar}" >&2 - cp "${l_cached_tar}" ${g_workdir}/inputs/${GO_TARBALL} - else - f_download_go - echo "Caching ${GO_TARBALL}" >&2 - cp ${g_workdir}/inputs/${GO_TARBALL} "${l_cached_tar}" - fi -} - -f_download_go() { - local l_remote - - l_remote=https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE} - curl -L "${l_remote}/${GO_TARBALL}" > ${g_workdir}/inputs/${GO_TARBALL} -} - -f_build() { - local l_sdk l_descriptor - - l_descriptor=$1 - l_commit=$2 - - [ -f ${l_descriptor} ] - - cd ${g_workdir} - export USE_DOCKER=1 - bin/gbuild --commit cosmos-sdk="$l_commit" ${GBUILD_FLAGS} "$l_descriptor" - libexec/stop-target || echo "warning: couldn't stop target" >&2 -} - -f_sign() { - local l_descriptor l_release_name l_sigs_dir - - l_descriptor=$1 - l_release_name=$2 - l_sigs_dir=$3 - - pushd ${g_workdir} - bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${l_sigs_dir}" --release=${l_release_name} ${l_descriptor} - popd -} - -f_verify() { - local l_descriptor l_release_name l_sigs_dir - - l_descriptor=$1 - l_release_name=$2 - l_sigs_dir=$3 - - pushd ${g_workdir} - bin/gverify --destination="${l_sigs_dir}" --release="${l_release_name}" ${l_descriptor} - popd -} - -f_validate_platform() { - case "${1}" in - linux|darwin|windows) - ;; - *) - echo "invalid platform -- ${1}" - exit 1 - esac -} - -f_abspath() { - echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")" -} f_help() { cat >&2 <&2 + mkdir "${l_builddir}/inputs/" + cp -v "${g_cached_go_tarball}" "${l_builddir}/inputs/" + done +} + +f_build() { + local l_descriptor + + l_descriptor=$1 + + bin/gbuild --commit cosmos-sdk="$g_commit" ${GBUILD_FLAGS} "$l_descriptor" + libexec/stop-target || f_echo_stderr "warning: couldn't stop target" +} + +f_sign_verify() { + local l_descriptor + + l_descriptor=$1 + + bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${g_sigs_dir}" --release=${g_release} ${l_descriptor} + bin/gverify --destination="${g_sigs_dir}" --release="${g_release}" ${l_descriptor} +} + +f_commit_sig() { + local l_release_name + + l_release_name=$1 + + pushd "${g_sigs_dir}" + git add . || echo "git add failed" >&2 + git commit -m "Add ${l_release_name} reproducible build" || echo "git commit failed" >&2 + popd +} + +f_prep_docker_image() { + pushd $1 + bin/make-base-vm --docker --suite bionic --arch amd64 + popd +} + +f_ensure_cache() { + g_gitian_cache="${g_workdir}/${GITIAN_CACHE_DIRNAME}" + [ -d "${g_gitian_cache}" ] || mkdir "${g_gitian_cache}" + + g_cached_go_tarball="${g_gitian_cache}/${GO_TARBALL}" + if [ ! -f "${g_cached_go_tarball}" ]; then + f_echo_stderr "${g_cached_go_tarball}: cache miss, caching..." + curl -L "${GO_TARBALL_URL}" --output "${g_cached_go_tarball}" + fi + + g_cached_gitian="${g_gitian_cache}/gitian-builder" + if [ ! -d "${g_cached_gitian}" ]; then + f_echo_stderr "${g_cached_gitian}: cache miss, caching..." + git clone ${GITIAN_REPO} "${g_cached_gitian}" + fi +} + +f_demangle_platforms() { + case "${1}" in + all) + printf '%s' 'darwin linux windows' ;; + linux|darwin|windows) + printf '%s' "${1}" ;; + *) + echo "invalid platform -- ${1}" + exit 1 + esac +} + +f_echo_stderr() { + echo $@ >&2 +} + + +while getopts ":cs:h" opt; do case "${opt}" in h) f_help ; exit 0 ;; - d) g_dirname="${OPTARG}" ;; + c) g_flag_commit=y ;; s) g_sign_identity="${OPTARG}" ;; esac done shift "$((OPTIND-1))" -g_platform="${1}" -f_validate_platform "${g_platform}" +g_platforms=$(f_demangle_platforms "${1}") +g_workdir="$(pwd)" +g_commit="$(git rev-parse HEAD)" +g_sigs_dir=${GAIA_SIGS:-"${g_workdir}/${DEFAULT_GAIA_SIGS}"} -g_dirname="${g_dirname:-gitian-build-${g_platform}}" -g_workdir="$(pwd)/${g_dirname}" -if [ -d "${g_workdir}" ]; then - echo "Directory ${g_workdir} exists and will be preserved" >&2 - g_gitian_skip_download=y -fi +f_ensure_cache -g_sdk="$(f_abspath ${2})" -[ -d "${g_sdk}" ] +f_prep_docker_image "${g_cached_gitian}" -g_sigs_dir=${GAIA_SIGS:-"$(pwd)/${DEFAULT_GAIA_SIGS}"} +f_prep_build "${g_platforms}" -# create local cache directory for all gitian builds -g_gitian_cache="$(pwd)/${GITIAN_CACHE_DIRNAME}" -echo "Ensure cache directory ${g_gitian_cache} exists" >&2 -mkdir -p "${g_gitian_cache}" +export USE_DOCKER=1 +for g_os in ${g_platforms}; do + g_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${g_os}" + g_descriptor="${g_workdir}/cmd/gaia/contrib/gitian-descriptors/gitian-${g_os}.yml" + [ -f ${g_descriptor} ] + g_builddir="$(f_builddir ${g_os})" -f_main "${g_platform}" "${g_sdk}" "${g_sigs_dir}" + pushd "${g_builddir}" + f_build "${g_descriptor}" + if [ -n "${g_sign_identity}" ]; then + f_sign_verify "${g_descriptor}" + fi + popd + + if [ -n "${g_sign_identity}" -a -n "${g_flag_commit}" ]; then + [ -d "${g_sigs_dir}/.git/" ] && f_commit_sig ${g_release} || f_echo_stderr "couldn't commit, ${g_sigs_dir} is not a git clone" + fi +done + +exit 0 diff --git a/docs/cosmos-hub/reproducible-builds.md b/docs/cosmos-hub/reproducible-builds.md index 3370c3da6..4c3039c7b 100644 --- a/docs/cosmos-hub/reproducible-builds.md +++ b/docs/cosmos-hub/reproducible-builds.md @@ -39,7 +39,7 @@ Run the following command to launch a build for `linux` and sign the final build report (replace `user@example.com` with the GPG identity you want to sign the report with): ``` -./cmd/gaia/contrib/gitian-build.sh -s user@example.com linux `pwd` +./cmd/gaia/contrib/gitian-build.sh -s user@example.com linux ``` The above command generates two directories in the current working directory: @@ -57,7 +57,7 @@ for platform in darwin linux windows; do ./cmd/gaia/contrib/gitian-build.sh -s u If you want to generate unsigned builds, just remove the option `-s` from the command line: ``` -./cmd/gaia/contrib/gitian-build.sh linux `pwd` +./cmd/gaia/contrib/gitian-build.sh linux ``` At the end of the procedure, build results can be found in the `./gaia.sigs` directory: