diff --git a/.circleci/config.yml b/.circleci/config.yml index 705c5d283..733b5d40f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,8 +4,6 @@ defaults: &linux_defaults working_directory: /go/src/github.com/cosmos/cosmos-sdk docker: - image: circleci/golang:1.12.4 - environment: - GOBIN: /tmp/workspace/bin ############ @@ -25,8 +23,7 @@ set_macos_env: &macos_env command: | echo 'export PATH=$PATH:$HOME/go/bin' >> $BASH_ENV echo 'export GOPATH=$HOME/project' >> $BASH_ENV - echo 'export GOBIN=$GOPATH/bin' >> $BASH_ENV - echo 'export PATH=$PATH:$HOME/go/bin:$GOBIN' >> $BASH_ENV + echo 'export PATH=$PATH:$HOME/go/bin:$GOPATH/bin' >> $BASH_ENV echo 'export GO111MODULE=on' ############ @@ -40,12 +37,6 @@ docs_update: &docs_deploy environment: AWS_REGION: us-east-1 -deps: &dependencies - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - jobs: setup_dependencies: <<: *linux_defaults @@ -59,13 +50,11 @@ jobs: - run: name: tools command: | - export PATH="$GOBIN:$PATH" - make tools - - *dependencies + make tools TOOLS_DESTDIR=/tmp/workspace/bin - run: name: binaries command: | - export PATH="$GOBIN:$PATH" + export PATH=/tmp/workspace/bin:$PATH make go-mod-cache make install - save_cache: @@ -85,14 +74,13 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Lint source command: | - export PATH="$GOBIN:$PATH" + export PATH=/tmp/workspace/bin:$PATH make ci-lint integration_tests: @@ -102,14 +90,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test cli command: | - export PATH="$GOBIN:$PATH" make test_cli test_sim_gaia_nondeterminism: @@ -119,14 +105,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test individual module simulations command: | - export PATH="$GOBIN:$PATH" make test_sim_gaia_nondeterminism test_sim_gaia_fast: @@ -136,14 +120,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test full Gaia simulation command: | - export PATH="$GOBIN:$PATH" make test_sim_gaia_fast test_sim_gaia_import_export: @@ -153,14 +135,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test Gaia import/export simulation command: | - export PATH="$GOBIN:$PATH" make test_sim_gaia_import_export test_sim_gaia_simulation_after_import: @@ -170,14 +150,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test Gaia import/export simulation command: | - export PATH="$GOBIN:$PATH" make test_sim_gaia_simulation_after_import test_sim_gaia_multi_seed_long: @@ -187,14 +165,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test multi-seed Gaia simulation long command: | - export PATH="$GOBIN:$PATH" export GO111MODULE=on make runsim runsim 500 50 TestFullGaiaSimulation @@ -206,14 +182,12 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - run: name: Test multi-seed Gaia simulation short command: | - export PATH="$GOBIN:$PATH" export GO111MODULE=on make runsim runsim 50 10 TestFullGaiaSimulation @@ -225,7 +199,6 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - run: mkdir -p /tmp/logs - restore_cache: keys: @@ -233,7 +206,6 @@ jobs: - run: name: Run tests command: | - export PATH="$GOBIN:$PATH" export VERSION="$(git describe --tags --long | sed 's/v\(.*\)/\1/')" export GO111MODULE=on for pkg in $(go list ./... | grep -v github.com/cosmos/cosmos-sdk/cmd/gaia/cli_test | grep -v '/simulation' | circleci tests split --split-by=timings); do @@ -254,7 +226,6 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout - - *dependencies - run: name: gather command: | @@ -283,7 +254,6 @@ jobs: machine: image: circleci/classic:latest environment: - GOBIN: /home/circleci/.go_workspace/bin GOPATH: /home/circleci/.go_workspace/ GOOS: linux GOARCH: amd64 diff --git a/.pending/breaking/gaia/4027-gaiad-and-gaiac b/.pending/breaking/gaia/4027-gaiad-and-gaiac new file mode 100644 index 000000000..c5055047e --- /dev/null +++ b/.pending/breaking/gaia/4027-gaiad-and-gaiac @@ -0,0 +1,2 @@ +#4027 gaiad and gaiacli version commands do not return the checksum of the go.sum file shipped along with the source release tarball. +Go modules feature guarantees dependencies reproducibility and as long as binaries are built via the Makefile shipped with the sources, no dependendencies can break such guarantee. diff --git a/.pending/breaking/sdk/4262-GoSumHash-is-no b/.pending/breaking/sdk/4262-GoSumHash-is-no new file mode 100644 index 000000000..582c4a310 --- /dev/null +++ b/.pending/breaking/sdk/4262-GoSumHash-is-no @@ -0,0 +1 @@ +#4262 GoSumHash is no longer returned by the version command. diff --git a/Dockerfile b/Dockerfile index 517c63589..f779aae3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM golang:alpine AS build-env # Set up dependencies -ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev +ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev python # Set working directory for the build WORKDIR /go/src/github.com/cosmos/cosmos-sdk diff --git a/Makefile b/Makefile index 759fed493..302e9bccb 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,7 @@ PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation') VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//') COMMIT := $(shell git log -1 --format='%H') LEDGER_ENABLED ?= true -GOBIN ?= $(GOPATH)/bin -GOSUM := $(shell which gosum) +BINDIR ?= $(GOPATH)/bin export GO111MODULE = on @@ -42,16 +41,17 @@ endif build_tags += $(BUILD_TAGS) build_tags := $(strip $(build_tags)) +whitespace := +whitespace += $(whitespace) +comma := , +build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags)) + # process linker flags ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \ -X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \ -X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \ - -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)" - -ifneq ($(GOSUM),) -ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) go.sum) -endif + -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)" ifeq ($(WITH_CLEVELDB),yes) ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb @@ -119,6 +119,12 @@ draw_deps: tools clean: rm -rf snapcraft-local.yaml build/ +distclean: clean + rm -rf \ + gitian-build-darwin/ \ + gitian-build-linux/ \ + gitian-build-windows/ \ + .gitian-builder-cache/ ######################################## ### Documentation @@ -164,20 +170,20 @@ test_sim_gaia_fast: test_sim_gaia_import_export: runsim @echo "Running Gaia import/export simulation. This may take several minutes..." - $(GOBIN)/runsim 50 5 TestGaiaImportExport + $(BINDIR)/runsim 50 5 TestGaiaImportExport test_sim_gaia_simulation_after_import: runsim @echo "Running Gaia simulation-after-import. This may take several minutes..." - $(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport + $(BINDIR)/runsim 50 5 TestGaiaSimulationAfterImport test_sim_gaia_custom_genesis_multi_seed: runsim @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - $(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation + $(BINDIR)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation test_sim_gaia_multi_seed: runsim @echo "Running multi-seed Gaia simulation. This may take awhile!" - $(GOBIN)/runsim 400 5 TestFullGaiaSimulation + $(BINDIR)/runsim 400 5 TestFullGaiaSimulation test_sim_benchmark_invariants: @echo "Running simulation invariant benchmarks..." @@ -186,8 +192,8 @@ test_sim_benchmark_invariants: -SimulationCommit=true -SimulationSeed=57 -v -timeout 24h # Don't move it into tools - this will be gone once gaia has moved into the new repo -runsim: $(GOBIN)/runsim -$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go +runsim: $(BINDIR)/runsim +$(BINDIR)/runsim: cmd/gaia/contrib/runsim/main.go go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim SIM_NUM_BLOCKS ?= 500 diff --git a/cmd/gaia/Makefile b/cmd/gaia/Makefile index 024e0d318..518812cee 100644 --- a/cmd/gaia/Makefile +++ b/cmd/gaia/Makefile @@ -5,7 +5,6 @@ VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//') COMMIT := $(shell git log -1 --format='%H') LEDGER_ENABLED ?= true GOBIN ?= $(GOPATH)/bin -GOSUM := $(shell which gosum) export GO111MODULE = on @@ -41,16 +40,17 @@ endif build_tags += $(BUILD_TAGS) build_tags := $(strip $(build_tags)) +whitespace := +whitespace += $(whitespace) +comma := , +build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags)) + # process linker flags ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \ -X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \ -X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \ - -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)" - -ifneq ($(GOSUM),) -ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) ../../go.sum) -endif + -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)" ifeq ($(WITH_CLEVELDB),yes) ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb diff --git a/cmd/gaia/contrib/gitian-build.sh b/cmd/gaia/contrib/gitian-build.sh new file mode 100755 index 000000000..eb1d65808 --- /dev/null +++ b/cmd/gaia/contrib/gitian-build.sh @@ -0,0 +1,216 @@ +#!/bin/bash + +# symbol prefixes: +# g_ -> global +# l_ - local variable +# f_ -> function + +set -euo pipefail + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +THIS="${THIS_DIR}/$(basename ${BASH_SOURCE[0]})" +GITIAN_CACHE_DIRNAME='.gitian-builder-cache' +GO_DEBIAN_RELEASE='1.12.5-1' +GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz" + +# Defaults + +DEFAULT_SIGN_COMMAND='gpg --detach-sign' +DEFAULT_GAIA_SIGS=${GAIA_SIGS:-'gaia.sigs'} +DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder' +DEFAULT_GBUILD_FLAGS='' + +# Overrides + +SIGN_COMMAND=${SIGN_COMMAND:-${DEFAULT_SIGN_COMMAND}} +GITIAN_REPO=${GITIAN_REPO:-${DEFAULT_GITIAN_REPO}} +GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}} + +# Globals + +g_workdir='' +g_gitian_cache='' +g_sign_identity='' +g_gitian_skip_download='' + +f_main() { + local l_dirname \ + l_sdk \ + l_commit \ + l_platform \ + l_result \ + l_descriptor \ + l_release \ + l_sigs_dir + + l_platform=$1 + l_sdk=$2 + l_sigs_dir=$3 + + pushd ${l_sdk} + l_commit="$(git rev-parse HEAD)" + l_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${l_platform}" + popd + + l_descriptor=${THIS_DIR}/gitian-descriptors/gitian-${l_platform}.yml + [ -f ${l_descriptor} ] + + if [ "${g_gitian_skip_download}" != "y" ]; then + echo "Cloning ${GITIAN_REPO} to ${g_workdir}" >&2 + git clone ${GITIAN_REPO} ${g_workdir} + fi + + echo "Fetching Go sources" >&2 + f_ensure_go_source_tarball + + echo "Prepare gitian-target docker image" >&2 + f_prep_docker_image + + echo "Start the build" >&2 + f_build "${l_descriptor}" "${l_commit}" + echo "You may find the result in $(echo ${g_workdir}/result/*.yml)" >&2 + + if [ -n "${g_sign_identity}" ]; then + f_sign "${l_descriptor}" "${l_release}" "${l_sigs_dir}" + echo "Build signed as ${g_sign_identity}, signatures can be found in ${l_sigs_dir}" + f_verify "${l_descriptor}" "${l_release}" "${l_sigs_dir}" + echo "Signatures in ${l_sigs_dir} have been verified" + else + echo "You can now sign the build with the following command:" >&2 + echo "cd ${g_workdir} ; bin/gsign -p 'gpg --detach-sign' -s GPG_IDENTITY --release=${l_release} ${l_descriptor}" >&2 + fi + + return 0 +} + +f_prep_docker_image() { + pushd ${g_workdir} + bin/make-base-vm --docker --suite bionic --arch amd64 + popd +} + +f_ensure_go_source_tarball() { + local l_cached_tar + + l_cached_tar="${g_gitian_cache}/${GO_TARBALL}" + mkdir -p ${g_workdir}/inputs + if [ -f "${l_cached_tar}" ]; then + echo "Fetching cached tarball from ${l_cached_tar}" >&2 + cp "${l_cached_tar}" ${g_workdir}/inputs/${GO_TARBALL} + else + f_download_go + echo "Caching ${GO_TARBALL}" >&2 + cp ${g_workdir}/inputs/${GO_TARBALL} "${l_cached_tar}" + fi +} + +f_download_go() { + local l_remote + + l_remote=https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE} + curl -L "${l_remote}/${GO_TARBALL}" > ${g_workdir}/inputs/${GO_TARBALL} +} + +f_build() { + local l_sdk l_descriptor + + l_descriptor=$1 + l_commit=$2 + + [ -f ${l_descriptor} ] + + cd ${g_workdir} + export USE_DOCKER=1 + bin/gbuild --commit cosmos-sdk="$l_commit" ${GBUILD_FLAGS} "$l_descriptor" + libexec/stop-target || echo "warning: couldn't stop target" >&2 +} + +f_sign() { + local l_descriptor l_release_name l_sigs_dir + + l_descriptor=$1 + l_release_name=$2 + l_sigs_dir=$3 + + pushd ${g_workdir} + bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${l_sigs_dir}" --release=${l_release_name} ${l_descriptor} + popd +} + +f_verify() { + local l_descriptor l_release_name l_sigs_dir + + l_descriptor=$1 + l_release_name=$2 + l_sigs_dir=$3 + + pushd ${g_workdir} + bin/gverify --destination="${l_sigs_dir}" --release="${l_release_name}" ${l_descriptor} + popd +} + +f_validate_platform() { + case "${1}" in + linux|darwin|windows) + ;; + *) + echo "invalid platform -- ${1}" + exit 1 + esac +} + +f_abspath() { + echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")" +} + +f_help() { + cat >&2 <&2 + g_gitian_skip_download=y +fi + +g_sdk="$(f_abspath ${2})" +[ -d "${g_sdk}" ] + +g_sigs_dir=${GAIA_SIGS:-"$(pwd)/${DEFAULT_GAIA_SIGS}"} + +# create local cache directory for all gitian builds +g_gitian_cache="$(pwd)/${GITIAN_CACHE_DIRNAME}" +echo "Ensure cache directory ${g_gitian_cache} exists" >&2 +mkdir -p "${g_gitian_cache}" + +f_main "${g_platform}" "${g_sdk}" "${g_sigs_dir}" diff --git a/cmd/gaia/contrib/gitian-descriptors/gitian-darwin.yml b/cmd/gaia/contrib/gitian-descriptors/gitian-darwin.yml new file mode 100644 index 000000000..67cd69c5a --- /dev/null +++ b/cmd/gaia/contrib/gitian-descriptors/gitian-darwin.yml @@ -0,0 +1,116 @@ +--- +name: "gaia-darwin" +enable_cache: true +distro: "ubuntu" +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "bsdmainutils" +- "build-essential" +- "ca-certificates" +- "curl" +- "debhelper" +- "dpkg-dev" +- "devscripts" +- "fakeroot" +- "git" +- "golang-any" +- "xxd" +- "quilt" +remotes: +- "url": "https://github.com/cosmos/cosmos-sdk.git" + "dir": "cosmos-sdk" +files: +- "golang-debian-1.12.5-1.tar.gz" +script: | + set -e -o pipefail + + GO_SRC_RELEASE=golang-debian-1.12.5-1 + GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" + # Compile go and configure the environment + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export BUILD_DIR=`pwd` + tar xf "${GO_SRC_TARBALL}" + rm -f "${GO_SRC_TARBALL}" + [ -d "${GO_SRC_RELEASE}/" ] + mv "${GO_SRC_RELEASE}/" go/ + pushd go/ + QUILT_PATCHES=debian/patches quilt push -a + fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache + popd + + export GOOS=darwin + export GOROOT=${BUILD_DIR}/go + export GOPATH=${BUILD_DIR}/gopath + mkdir -p ${GOPATH}/bin + + export PATH_orig=${PATH} + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + + export ARCHS='386 amd64' + export GO111MODULE=on + + # Make release tarball + pushd cosmos-sdk + VERSION=$(git describe --tags | sed 's/^v//') + COMMIT=$(git log -1 --format='%H') + DISTNAME=gaia-${VERSION} + git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD + SOURCEDIST=`pwd`/`echo gaia-*.tar.gz` + popd + + # Correct tar file order + mkdir -p temp + pushd temp + tar xf $SOURCEDIST + rm $SOURCEDIST + find gaia-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST + popd + + # Prepare GOPATH and install deps + distsrc=${GOPATH}/src/github.com/cosmos/cosmos-sdk + mkdir -p ${distsrc} + pushd ${distsrc} + tar --strip-components=1 -xf $SOURCEDIST + go mod download + popd + + # Configure LDFLAGS for reproducible builds + LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ + -X github.com/cosmos/cosmos-sdk/version.Name=gaia \ + -X github.com/cosmos/cosmos-sdk/version.Version=${VERSION} \ + -X github.com/cosmos/cosmos-sdk/version.Commit=${COMMIT} \ + -X github.com/cosmos/cosmos-sdk/version.BuildTags=netgo,ledger" + + # Extract release tarball and build + for arch in ${ARCHS}; do + INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} + mkdir -p ${INSTALLPATH} + + # Build gaia tool suite + pushd ${distsrc} + for prog in gaiacli gaiad; do + GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ + -gcflags=all=-trimpath=${GOPATH} \ + -asmflags=all=-trimpath=${GOPATH} \ + -mod=readonly -tags "netgo ledger" \ + -ldflags="${LDFLAGS}" \ + -o ${INSTALLPATH}/${prog} ./cmd/gaia/cmd/${prog} + + done + popd # ${distsrc} + + pushd ${INSTALLPATH} + find -type f | sort | tar \ + --no-recursion --mode='u+rw,go+r-w,a+X' \ + --numeric-owner --sort=name \ + --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-darwin-${arch}.tar.gz + popd # installed + done + + rm -rf ${distsrc} + + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src diff --git a/cmd/gaia/contrib/gitian-descriptors/gitian-linux.yml b/cmd/gaia/contrib/gitian-descriptors/gitian-linux.yml new file mode 100644 index 000000000..f3112baa2 --- /dev/null +++ b/cmd/gaia/contrib/gitian-descriptors/gitian-linux.yml @@ -0,0 +1,115 @@ +--- +name: "gaia-linux" +enable_cache: true +distro: "ubuntu" +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "bsdmainutils" +- "build-essential" +- "ca-certificates" +- "curl" +- "debhelper" +- "dpkg-dev" +- "devscripts" +- "fakeroot" +- "git" +- "golang-any" +- "xxd" +- "quilt" +remotes: +- "url": "https://github.com/cosmos/cosmos-sdk.git" + "dir": "cosmos-sdk" +files: +- "golang-debian-1.12.5-1.tar.gz" +script: | + set -e -o pipefail + + GO_SRC_RELEASE=golang-debian-1.12.5-1 + GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" + # Compile go and configure the environment + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export BUILD_DIR=`pwd` + tar xf "${GO_SRC_TARBALL}" + rm -f "${GO_SRC_TARBALL}" + [ -d "${GO_SRC_RELEASE}/" ] + mv "${GO_SRC_RELEASE}/" go/ + pushd go/ + QUILT_PATCHES=debian/patches quilt push -a + fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache + popd + + export GOROOT=${BUILD_DIR}/go + export GOPATH=${BUILD_DIR}/gopath + mkdir -p ${GOPATH}/bin + + export PATH_orig=${PATH} + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + + export ARCHS='386 amd64 arm arm64' + export GO111MODULE=on + + # Make release tarball + pushd cosmos-sdk + VERSION=$(git describe --tags | sed 's/^v//') + COMMIT=$(git log -1 --format='%H') + DISTNAME=gaia-${VERSION} + git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD + SOURCEDIST=`pwd`/`echo gaia-*.tar.gz` + popd + + # Correct tar file order + mkdir -p temp + pushd temp + tar xf $SOURCEDIST + rm $SOURCEDIST + find gaia-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST + popd + + # Prepare GOPATH and install deps + distsrc=${GOPATH}/src/github.com/cosmos/cosmos-sdk + mkdir -p ${distsrc} + pushd ${distsrc} + tar --strip-components=1 -xf $SOURCEDIST + go mod download + popd + + # Configure LDFLAGS for reproducible builds + LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ + -X github.com/cosmos/cosmos-sdk/version.Name=gaia \ + -X github.com/cosmos/cosmos-sdk/version.Version=${VERSION} \ + -X github.com/cosmos/cosmos-sdk/version.Commit=${COMMIT} \ + -X github.com/cosmos/cosmos-sdk/version.BuildTags=netgo,ledger" + + # Extract release tarball and build + for arch in ${ARCHS}; do + INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} + mkdir -p ${INSTALLPATH} + + # Build gaia tool suite + pushd ${distsrc} + for prog in gaiacli gaiad; do + GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ + -gcflags=all=-trimpath=${GOPATH} \ + -asmflags=all=-trimpath=${GOPATH} \ + -mod=readonly -tags "netgo ledger" \ + -ldflags="${LDFLAGS}" \ + -o ${INSTALLPATH}/${prog} ./cmd/gaia/cmd/${prog} + + done + popd # ${distsrc} + + pushd ${INSTALLPATH} + find -type f | sort | tar \ + --no-recursion --mode='u+rw,go+r-w,a+X' \ + --numeric-owner --sort=name \ + --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-linux-${arch}.tar.gz + popd # installed + done + + rm -rf ${distsrc} + + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src diff --git a/cmd/gaia/contrib/gitian-descriptors/gitian-windows.yml b/cmd/gaia/contrib/gitian-descriptors/gitian-windows.yml new file mode 100644 index 000000000..4ad03c328 --- /dev/null +++ b/cmd/gaia/contrib/gitian-descriptors/gitian-windows.yml @@ -0,0 +1,116 @@ +--- +name: "gaia-windows" +enable_cache: true +distro: "ubuntu" +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "bsdmainutils" +- "build-essential" +- "ca-certificates" +- "curl" +- "debhelper" +- "dpkg-dev" +- "devscripts" +- "fakeroot" +- "git" +- "golang-any" +- "xxd" +- "quilt" +remotes: +- "url": "https://github.com/cosmos/cosmos-sdk.git" + "dir": "cosmos-sdk" +files: +- "golang-debian-1.12.5-1.tar.gz" +script: | + set -e -o pipefail + + GO_SRC_RELEASE=golang-debian-1.12.5-1 + GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" + # Compile go and configure the environment + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export BUILD_DIR=`pwd` + tar xf "${GO_SRC_TARBALL}" + rm -f "${GO_SRC_TARBALL}" + [ -d "${GO_SRC_RELEASE}/" ] + mv "${GO_SRC_RELEASE}/" go/ + pushd go/ + QUILT_PATCHES=debian/patches quilt push -a + fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache + popd + + export GOROOT=${BUILD_DIR}/go + export GOPATH=${BUILD_DIR}/gopath + mkdir -p ${GOPATH}/bin + + export PATH_orig=${PATH} + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + + export ARCHS='386 amd64' + export GO111MODULE=on + + # Make release tarball + pushd cosmos-sdk + VERSION=$(git describe --tags | sed 's/^v//') + COMMIT=$(git log -1 --format='%H') + DISTNAME=gaia-${VERSION} + git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD + SOURCEDIST=`pwd`/`echo gaia-*.tar.gz` + popd + + # Correct tar file order + mkdir -p temp + pushd temp + tar xf $SOURCEDIST + rm $SOURCEDIST + find gaia-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST + popd + + # Prepare GOPATH and install deps + distsrc=${GOPATH}/src/github.com/cosmos/cosmos-sdk + mkdir -p ${distsrc} + pushd ${distsrc} + tar --strip-components=1 -xf $SOURCEDIST + go mod download + popd + + # Configure LDFLAGS for reproducible builds + LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ + -X github.com/cosmos/cosmos-sdk/version.Name=gaia \ + -X github.com/cosmos/cosmos-sdk/version.Version=${VERSION} \ + -X github.com/cosmos/cosmos-sdk/version.Commit=${COMMIT} \ + -X github.com/cosmos/cosmos-sdk/version.BuildTags=netgo,ledger" + + # Extract release tarball and build + for arch in ${ARCHS}; do + INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} + mkdir -p ${INSTALLPATH} + + # Build gaia tool suite + pushd ${distsrc} + for prog in gaiacli gaiad; do + exe=${prog}.exe + GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ + -gcflags=all=-trimpath=${GOPATH} \ + -asmflags=all=-trimpath=${GOPATH} \ + -mod=readonly -tags "netgo ledger" \ + -ldflags="${LDFLAGS}" \ + -o ${INSTALLPATH}/${exe} ./cmd/gaia/cmd/${prog} + + done + popd # ${distsrc} + + pushd ${INSTALLPATH} + find -type f | sort | tar \ + --no-recursion --mode='u+rw,go+r-w,a+X' \ + --numeric-owner --sort=name \ + --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-windows-${arch}.tar.gz + popd # installed + done + + rm -rf ${distsrc} + + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src diff --git a/cmd/gaia/contrib/gitian-keys/README.md b/cmd/gaia/contrib/gitian-keys/README.md new file mode 100644 index 000000000..645014337 --- /dev/null +++ b/cmd/gaia/contrib/gitian-keys/README.md @@ -0,0 +1,29 @@ +## PGP keys of Gitian builders and Gaia Developers + +The file `keys.txt` contains fingerprints of the public keys of Gitian builders +and active developers. + +The associated keys are mainly used to sign git commits or the build results +of Gitian builds. + +The most recent version of each pgp key can be found on most PGP key servers. + +Fetch the latest version from the key server to see if any key was revoked in +the meantime. +To fetch the latest version of all pgp keys in your gpg homedir, + +```sh +gpg --refresh-keys +``` + +To fetch keys of Gitian builders and active core developers, feed the list of +fingerprints of the primary keys into gpg: + +```sh +while read fingerprint keyholder_name; \ +do gpg --keyserver hkp://subset.pool.sks-keyservers.net \ +--recv-keys ${fingerprint}; done < ./keys.txt +``` + +Add your key to the list if you are a Gaia core developer or you have +provided Gitian signatures for two major or minor releases of Gaia. diff --git a/cmd/gaia/contrib/gitian-keys/keys.txt b/cmd/gaia/contrib/gitian-keys/keys.txt new file mode 100644 index 000000000..91330ae0b --- /dev/null +++ b/cmd/gaia/contrib/gitian-keys/keys.txt @@ -0,0 +1 @@ +04160004A8276E40BB9890FBE8A48AE5311D765A Alessio Treglia diff --git a/cmd/gaia/sims.mk b/cmd/gaia/sims.mk index c6776594f..6dee8236a 100644 --- a/cmd/gaia/sims.mk +++ b/cmd/gaia/sims.mk @@ -3,8 +3,8 @@ ######################################## ### Simulations -runsim: $(GOBIN)/runsim -$(GOBIN)/runsim: contrib/runsim/main.go +runsim: $(GOPATH)/bin/runsim +$(GOPATH)/bin/runsim: contrib/runsim/main.go go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim sim-gaia-nondeterminism: @@ -23,20 +23,20 @@ sim-gaia-fast: sim-gaia-import-export: runsim @echo "Running Gaia import/export simulation. This may take several minutes..." - $(GOBIN)/runsim 50 5 TestGaiaImportExport + $(GOPATH)/bin/runsim 50 5 TestGaiaImportExport sim-gaia-simulation-after-import: runsim @echo "Running Gaia simulation-after-import. This may take several minutes..." - $(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport + $(GOPATH)/bin/runsim 50 5 TestGaiaSimulationAfterImport sim-gaia-custom-genesis-multi-seed: runsim @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - $(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation + $(GOPATH)/bin/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation sim-gaia-multi-seed: runsim @echo "Running multi-seed Gaia simulation. This may take awhile!" - $(GOBIN)/runsim 400 5 TestFullGaiaSimulation + $(GOPATH)/bin/runsim 400 5 TestFullGaiaSimulation sim-benchmark-invariants: @echo "Running simulation invariant benchmarks..." diff --git a/contrib/devtools/Makefile b/contrib/devtools/Makefile index e07d16f4b..56a378fca 100644 --- a/contrib/devtools/Makefile +++ b/contrib/devtools/Makefile @@ -42,20 +42,21 @@ mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd) # tools ### -GOLANGCI_LINT = $(GOBIN)/golangci-lint -STATIK = $(GOBIN)/statik -GOIMPORTS = $(GOBIN)/goimports -GOSUM = $(GOBIN)/gosum -CLOG = $(GOBIN)/clog +TOOLS_DESTDIR ?= $(GOPATH)/bin + +GOLANGCI_LINT = $(TOOLS_DESTDIR)/golangci-lint +STATIK = $(TOOLS_DESTDIR)/statik +GOIMPORTS = $(TOOLS_DESTDIR)/goimports +CLOG = $(TOOLS_DESTDIR)/clog all: tools tools: tools-stamp -tools-stamp: $(GOLANGCI_LINT) $(STATIK) $(GOIMPORTS) $(GOSUM) $(CLOG) +tools-stamp: $(STATIK) $(GOIMPORTS) $(CLOG) $(GOLANGCI_LINT) touch $@ -$(GOLANGCI_LINT): $(mkfile_dir)/install-golangci-lint.sh $(GOSUM) - bash $(mkfile_dir)/install-golangci-lint.sh $(GOBIN) $(GOLANGCI_LINT_VERSION) $(GOLANGCI_LINT_HASHSUM) +$(GOLANGCI_LINT): $(mkfile_dir)/install-golangci-lint.sh + bash $(mkfile_dir)/install-golangci-lint.sh $(TOOLS_DESTDIR) $(GOLANGCI_LINT_VERSION) $(GOLANGCI_LINT_HASHSUM) $(STATIK): $(call go_install,rakyll,statik,v0.1.5) @@ -63,14 +64,11 @@ $(STATIK): $(GOIMPORTS): go get golang.org/x/tools/cmd/goimports@v0.0.0-20190114222345-bf090417da8b -$(GOSUM): ./contrib/devtools/gosum/main.go - go install -mod=readonly ./$(&2 "${CURL}" -sfL "https://raw.githubusercontent.com/golangci/golangci-lint/${VERSION}/install.sh" > "${installer}" echo "Checking hashsum ..." >&2 -[ "${HASHSUM}" = "$(${GOSUM} ${installer})" ] +[ "${HASHSUM}" = "$(f_sha256 ${installer})" ] chmod +x "${installer}" echo "Launching installer ..." >&2 diff --git a/docs/cosmos-hub/installation.md b/docs/cosmos-hub/installation.md index ae17a9666..8895e9688 100644 --- a/docs/cosmos-hub/installation.md +++ b/docs/cosmos-hub/installation.md @@ -4,13 +4,12 @@ This guide will explain how to install the `gaiad` and `gaiacli` entrypoints ont ### Install Go -Install `go` by following the [official docs](https://golang.org/doc/install). Remember to set your `$GOPATH`, `$GOBIN`, and `$PATH` environment variables, for example: +Install `go` by following the [official docs](https://golang.org/doc/install). Remember to set your `$GOPATH` and `$PATH` environment variables, for example: ```bash mkdir -p $HOME/go/bin echo "export GOPATH=$HOME/go" >> ~/.bash_profile -echo "export GOBIN=\$GOPATH/bin" >> ~/.bash_profile -echo "export PATH=\$PATH:\$GOBIN" >> ~/.bash_profile +echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.bash_profile source ~/.bash_profile ``` diff --git a/docs/cosmos-hub/reproducible-builds.md b/docs/cosmos-hub/reproducible-builds.md new file mode 100644 index 000000000..3370c3da6 --- /dev/null +++ b/docs/cosmos-hub/reproducible-builds.md @@ -0,0 +1,82 @@ +## Build Gaia Deterministically + +Gitian is the deterministic build process that is used to build the Gaia executables. It provides a way to be reasonably sure that the executables are really built from the git source. It also makes sure that the same, tested dependencies are used and statically built into the executable. + +Multiple developers build the source code by following a specific descriptor ("recipe"), cryptographically sign the result, and upload the resulting signature. These results are compared and only if they match, the build is accepted and provided for download. + +More independent Gitian builders are needed, which is why this guide exists. It is preferred you follow these steps yourself instead of using someone else's VM image to avoid 'contaminating' the build. + +This page contains all instructions required to build and sign reproducible Gaia binaries for Linux, Mac OS X, and Windows. + +## Prerequisites + +Make sure your system satisfy minimum requisites as outlined in https://github.com/devrandom/gitian-builder#prerequisites. + +All the following instructions have been tested on *Ubuntu 18.04.2 LTS* with *docker 18.06.1-ce* and *docker 18.09.6-ce*. + +If you are on Mac OS X, make sure you have prepended your `PATH` environment variable with GNU coreutils's path before running the build script: + +``` +export PATH=/usr/local/opt/coreutils/libexec/gnubin/:$PATH +``` + +## Build and sign + +Clone cosmos-sdk: + +``` +git clone git@github.com:cosmos/cosmos-sdk +``` + +Checkout the commit, branch, or release tag you want to build: + +``` +cd cosmos-sdk/ +git checkout v0.35.0 +``` + +Run the following command to launch a build for `linux` and sign the final build +report (replace `user@example.com` with the GPG identity you want to sign the report with): + +``` +./cmd/gaia/contrib/gitian-build.sh -s user@example.com linux `pwd` +``` + +The above command generates two directories in the current working directory: +* `gitian-build-linux` containing the `gitian-builder` clone used to drive the build process. +* `gaia.sigs` containing the signed build report. + +Replace `linux` in the above command with `darwin` or `windows` to run builds for Mac OS X and Windows respectively. +Run the following command to build binaries for all platforms (`darwin`, `linux`, and `windows`): + +``` +cd cosmos-sdk/ +for platform in darwin linux windows; do ./cmd/gaia/contrib/gitian-build.sh -s user@example.com $platform `pwd`; done +``` + +If you want to generate unsigned builds, just remove the option `-s` from the command line: + +``` +./cmd/gaia/contrib/gitian-build.sh linux `pwd` +``` + +At the end of the procedure, build results can be found in the `./gaia.sigs` directory: + +Please refer to the `cmd/gaia/contrib/gitian-build.sh`'s help screen for further information on its usage. + +## Signatures upload + +Once signatures are generated, they could be uploaded to gaia's dedicated repository: https://github.com/cosmos/gaia.sigs. + +## Troubleshooting + +### Docker gitian-target container cannot be killed + +The issue is due to a relatively recent kernel apparmor change, [see here](https://github.com/moby/moby/issues/36809#issuecomment-379325713) for more information on a potential mitigation for the issue. + +On Ubuntu 18.04, when the container hangs and `docker` is unable to kill it you can try to use `pkill` to forcibly terminate `containerd-shim`: + +``` +sudo pkill containerd-shim +docker system prune +``` diff --git a/docs/translations/cn/cosmos-hub/installation.md b/docs/translations/cn/cosmos-hub/installation.md index 5c0bba93e..bc0a9c233 100644 --- a/docs/translations/cn/cosmos-hub/installation.md +++ b/docs/translations/cn/cosmos-hub/installation.md @@ -9,8 +9,7 @@ ```bash mkdir -p $HOME/go/bin echo "export GOPATH=$HOME/go" >> ~/.bash_profile -echo "export GOBIN=\$GOPATH/bin" >> ~/.bash_profile -echo "export PATH=\$PATH:\$GOBIN" >> ~/.bash_profile +echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.bash_profile echo "export GO111MODULE=on" >> ~/.bash_profile source ~/.bash_profile ``` diff --git a/docs/translations/kr/gaia/installation.md b/docs/translations/kr/gaia/installation.md index da390e18b..7c71696cf 100755 --- a/docs/translations/kr/gaia/installation.md +++ b/docs/translations/kr/gaia/installation.md @@ -4,13 +4,12 @@ ### Go 설치하기 -공식 [Go 문서](https://golang.org/doc/install)를 따라서 `go`를 설치하십시오. `$GOPATH`, `$GOBIN`, 그리고 `$PATH`의 환경을 꼭 세팅하세요. 예시: +공식 [Go 문서](https://golang.org/doc/install)를 따라서 `go`를 설치하십시오. `$GOPATH`, 그리고 `$PATH`의 환경을 꼭 세팅하세요. 예시: ```bash mkdir -p $HOME/go/bin echo "export GOPATH=$HOME/go" >> ~/.bash_profile -echo "export GOBIN=\$GOPATH/bin" >> ~/.bash_profile -echo "export PATH=\$PATH:\$GOBIN" >> ~/.bash_profile +echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.bash_profile echo "export GO111MODULE=on" >> ~/.bash_profile source ~/.bash_profile ``` diff --git a/version/version.go b/version/version.go index c46f9deca..86274dfea 100644 --- a/version/version.go +++ b/version/version.go @@ -6,9 +6,8 @@ // Configure the version command // // The version command can be just added to your cobra root command. -// At build time, the variables Name, Version, Commit, GoSumHash, and -// BuildTags can be passed as build flags as shown in the following -// example: +// At build time, the variables Name, Version, Commit, and BuildTags +// can be passed as build flags as shown in the following example: // // go build -X github.com/cosmos/cosmos-sdk/version.Name=dapp \ // -X github.com/cosmos/cosmos-sdk/version.Version=1.0 \ @@ -38,7 +37,6 @@ type versionInfo struct { Name string `json:"name"` Version string `json:"version"` GitCommit string `json:"commit"` - GoSumHash string `json:"gosum_hash"` BuildTags string `json:"build_tags"` GoVersion string `json:"go"` } @@ -46,9 +44,8 @@ type versionInfo struct { func (v versionInfo) String() string { return fmt.Sprintf(`%s: %s git commit: %s -go.sum hash: %s build tags: %s -%s`, v.Name, v.Version, v.GitCommit, v.GoSumHash, v.BuildTags, v.GoVersion) +%s`, v.Name, v.Version, v.GitCommit, v.BuildTags, v.GoVersion) } func newVersionInfo() versionInfo { @@ -56,7 +53,6 @@ func newVersionInfo() versionInfo { Name, Version, Commit, - GoSumHash, BuildTags, fmt.Sprintf("go version %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH)} }