certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

R4R: Infrastructure for reproducible builds (#4262) 

This change set introduces support for building gaia with gitian
on the following GOOS/GOARCH pairs:

- darwin/386
- darwin/amd64
- linux/386
- linux/amd64
- linux/arm
- linux/arm64
- windows/386
- windows/amd64

cmd/gaia/contrib/gitian-descriptors/ contains gitian descriptor files.

cmd/gaia/contrib/gitian-keys/ contains:
- a keys.txt file that is meant to list core developers and gitian
  builders PGP keys. 
- README.me to provide instructions on how to import the keys
  into one's personal GPG keyring.

The gosum utility is removed, so is the go.sum hashsum bit from
gaiacli/gaiad version string. It was meant to be a provisional
mitigation to the lack of a reproducible build process.

GOBIN is removed from all Makefiles. When GOBIN is set, go
refuses to cross-compiles binaries for foreign architectures.
export GOBIN=$GOPATH/bin is unnecessary anyway as by
default go install places built binaries in $GOPATH/bin.
Developers are required to update their enviornment files and
replace $GOBIN with $GOPATH/bin in PATH.

circleci configuration file is amended accordingly.

Closes: #4027
Closes: #4280
This commit is contained in:
Alessio Treglia 2019-05-14 00:33:34 +02:00 committed by GitHub
parent 3439101421
commit c0486aa532
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 739 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.circleci/config.yml
View File

@ -4,8 +4,6 @@ defaults: &linux_defaults
working_directory: /go/src/github.com/cosmos/cosmos-sdk
docker:
- image: circleci/golang:1.12.4
environment:
GOBIN: /tmp/workspace/bin
############
@ -25,8 +23,7 @@ set_macos_env: &macos_env
command: |
echo 'export PATH=$PATH:$HOME/go/bin' >> $BASH_ENV
echo 'export GOPATH=$HOME/project' >> $BASH_ENV
echo 'export GOBIN=$GOPATH/bin' >> $BASH_ENV
echo 'export PATH=$PATH:$HOME/go/bin:$GOBIN' >> $BASH_ENV
echo 'export PATH=$PATH:$HOME/go/bin:$GOPATH/bin' >> $BASH_ENV
echo 'export GO111MODULE=on'
############
@ -40,12 +37,6 @@ docs_update: &docs_deploy
environment:
AWS_REGION: us-east-1
deps: &dependencies
run:
name: dependencies
command: |
export PATH="$GOBIN:$PATH"
jobs:
setup_dependencies:
<<: *linux_defaults
@ -59,13 +50,11 @@ jobs:
- run:
name: tools
command: |
export PATH="$GOBIN:$PATH"
make tools
- *dependencies
make tools TOOLS_DESTDIR=/tmp/workspace/bin
- run:
name: binaries
command: |
export PATH="$GOBIN:$PATH"
export PATH=/tmp/workspace/bin:$PATH
make go-mod-cache
make install
- save_cache:
@ -85,14 +74,13 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Lint source
command: |
export PATH="$GOBIN:$PATH"
export PATH=/tmp/workspace/bin:$PATH
make ci-lint
integration_tests:
@ -102,14 +90,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test cli
command: |
export PATH="$GOBIN:$PATH"
make test_cli
test_sim_gaia_nondeterminism:
@ -119,14 +105,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test individual module simulations
command: |
export PATH="$GOBIN:$PATH"
make test_sim_gaia_nondeterminism
test_sim_gaia_fast:
@ -136,14 +120,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test full Gaia simulation
command: |
export PATH="$GOBIN:$PATH"
make test_sim_gaia_fast
test_sim_gaia_import_export:
@ -153,14 +135,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test Gaia import/export simulation
command: |
export PATH="$GOBIN:$PATH"
make test_sim_gaia_import_export
test_sim_gaia_simulation_after_import:
@ -170,14 +150,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test Gaia import/export simulation
command: |
export PATH="$GOBIN:$PATH"
make test_sim_gaia_simulation_after_import
test_sim_gaia_multi_seed_long:
@ -187,14 +165,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test multi-seed Gaia simulation long
command: |
export PATH="$GOBIN:$PATH"
export GO111MODULE=on
make runsim
runsim 500 50 TestFullGaiaSimulation
@ -206,14 +182,12 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
- run:
name: Test multi-seed Gaia simulation short
command: |
export PATH="$GOBIN:$PATH"
export GO111MODULE=on
make runsim
runsim 50 10 TestFullGaiaSimulation
@ -225,7 +199,6 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- run: mkdir -p /tmp/logs
- restore_cache:
keys:
@ -233,7 +206,6 @@ jobs:
- run:
name: Run tests
command: |
export PATH="$GOBIN:$PATH"
export VERSION="$(git describe --tags --long | sed 's/v\(.*\)/\1/')"
export GO111MODULE=on
for pkg in $(go list ./... | grep -v github.com/cosmos/cosmos-sdk/cmd/gaia/cli_test | grep -v '/simulation' | circleci tests split --split-by=timings); do
@ -254,7 +226,6 @@ jobs:
- attach_workspace:
at: /tmp/workspace
- checkout
- *dependencies
- run:
name: gather
command: |
@ -283,7 +254,6 @@ jobs:
machine:
image: circleci/classic:latest
environment:
GOBIN: /home/circleci/.go_workspace/bin
GOPATH: /home/circleci/.go_workspace/
GOOS: linux
GOARCH: amd64

2
.pending/breaking/gaia/4027-gaiad-and-gaiac Normal file
View File

@ -0,0 +1,2 @@
#4027 gaiad and gaiacli version commands do not return the checksum of the go.sum file shipped along with the source release tarball.
Go modules feature guarantees dependencies reproducibility and as long as binaries are built via the Makefile shipped with the sources, no dependendencies can break such guarantee.

1
.pending/breaking/sdk/4262-GoSumHash-is-no Normal file
View File

@ -0,0 +1 @@
#4262 GoSumHash is no longer returned by the version command.

2
Dockerfile
View File

@ -5,7 +5,7 @@
FROM golang:alpine AS build-env
# Set up dependencies
ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev
ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev python
# Set working directory for the build
WORKDIR /go/src/github.com/cosmos/cosmos-sdk

32
Makefile
View File

@ -5,8 +5,7 @@ PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation')
VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//')
COMMIT := $(shell git log -1 --format='%H')
LEDGER_ENABLED ?= true
GOBIN ?= $(GOPATH)/bin
GOSUM := $(shell which gosum)
BINDIR ?= $(GOPATH)/bin
export GO111MODULE = on
@ -42,16 +41,17 @@ endif
build_tags += $(BUILD_TAGS)
build_tags := $(strip $(build_tags))
whitespace :=
whitespace += $(whitespace)
comma := ,
build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags))
# process linker flags
ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \
-X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \
-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
-X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)"
ifneq ($(GOSUM),)
ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) go.sum)
endif
-X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)"
ifeq ($(WITH_CLEVELDB),yes)
ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb
@ -119,6 +119,12 @@ draw_deps: tools
clean:
rm -rf snapcraft-local.yaml build/
distclean: clean
rm -rf \
gitian-build-darwin/ \
gitian-build-linux/ \
gitian-build-windows/ \
.gitian-builder-cache/
########################################
### Documentation
@ -164,20 +170,20 @@ test_sim_gaia_fast:
test_sim_gaia_import_export: runsim
@echo "Running Gaia import/export simulation. This may take several minutes..."
$(GOBIN)/runsim 50 5 TestGaiaImportExport
$(BINDIR)/runsim 50 5 TestGaiaImportExport
test_sim_gaia_simulation_after_import: runsim
@echo "Running Gaia simulation-after-import. This may take several minutes..."
$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
$(BINDIR)/runsim 50 5 TestGaiaSimulationAfterImport
test_sim_gaia_custom_genesis_multi_seed: runsim
@echo "Running multi-seed custom genesis simulation..."
@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
$(BINDIR)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
test_sim_gaia_multi_seed: runsim
@echo "Running multi-seed Gaia simulation. This may take awhile!"
$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
$(BINDIR)/runsim 400 5 TestFullGaiaSimulation
test_sim_benchmark_invariants:
@echo "Running simulation invariant benchmarks..."
@ -186,8 +192,8 @@ test_sim_benchmark_invariants:
-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
# Don't move it into tools - this will be gone once gaia has moved into the new repo
runsim: $(GOBIN)/runsim
$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go
runsim: $(BINDIR)/runsim
$(BINDIR)/runsim: cmd/gaia/contrib/runsim/main.go
go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
SIM_NUM_BLOCKS ?= 500

12
cmd/gaia/Makefile
View File

@ -5,7 +5,6 @@ VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//')
COMMIT := $(shell git log -1 --format='%H')
LEDGER_ENABLED ?= true
GOBIN ?= $(GOPATH)/bin
GOSUM := $(shell which gosum)
export GO111MODULE = on
@ -41,16 +40,17 @@ endif
build_tags += $(BUILD_TAGS)
build_tags := $(strip $(build_tags))
whitespace :=
whitespace += $(whitespace)
comma := ,
build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags))
# process linker flags
ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \
-X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \
-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
-X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)"
ifneq ($(GOSUM),)
ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) ../../go.sum)
endif
-X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)"
ifeq ($(WITH_CLEVELDB),yes)
ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb

216
cmd/gaia/contrib/gitian-build.sh Executable file
View File

@ -0,0 +1,216 @@
#!/bin/bash
# symbol prefixes:
# g_ -> global
# l_ - local variable
# f_ -> function
set -euo pipefail
THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
THIS="${THIS_DIR}/$(basename ${BASH_SOURCE[0]})"
GITIAN_CACHE_DIRNAME='.gitian-builder-cache'
GO_DEBIAN_RELEASE='1.12.5-1'
GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz"
# Defaults
DEFAULT_SIGN_COMMAND='gpg --detach-sign'
DEFAULT_GAIA_SIGS=${GAIA_SIGS:-'gaia.sigs'}
DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder'
DEFAULT_GBUILD_FLAGS=''
# Overrides
SIGN_COMMAND=${SIGN_COMMAND:-${DEFAULT_SIGN_COMMAND}}
GITIAN_REPO=${GITIAN_REPO:-${DEFAULT_GITIAN_REPO}}
GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}}
# Globals
g_workdir=''
g_gitian_cache=''
g_sign_identity=''
g_gitian_skip_download=''
f_main() {
local l_dirname \
l_sdk \
l_commit \
l_platform \
l_result \
l_descriptor \
l_release \
l_sigs_dir
l_platform=$1
l_sdk=$2
l_sigs_dir=$3
pushd ${l_sdk}
l_commit="$(git rev-parse HEAD)"
l_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${l_platform}"
popd
l_descriptor=${THIS_DIR}/gitian-descriptors/gitian-${l_platform}.yml
[ -f ${l_descriptor} ]
if [ "${g_gitian_skip_download}" != "y" ]; then
echo "Cloning ${GITIAN_REPO} to ${g_workdir}" >&2
git clone ${GITIAN_REPO} ${g_workdir}
fi
echo "Fetching Go sources" >&2
f_ensure_go_source_tarball
echo "Prepare gitian-target docker image" >&2
f_prep_docker_image
echo "Start the build" >&2
f_build "${l_descriptor}" "${l_commit}"
echo "You may find the result in $(echo ${g_workdir}/result/*.yml)" >&2
if [ -n "${g_sign_identity}" ]; then
f_sign "${l_descriptor}" "${l_release}" "${l_sigs_dir}"
echo "Build signed as ${g_sign_identity}, signatures can be found in ${l_sigs_dir}"
f_verify "${l_descriptor}" "${l_release}" "${l_sigs_dir}"
echo "Signatures in ${l_sigs_dir} have been verified"
else
echo "You can now sign the build with the following command:" >&2
echo "cd ${g_workdir} ; bin/gsign -p 'gpg --detach-sign' -s GPG_IDENTITY --release=${l_release} ${l_descriptor}" >&2
fi
return 0
}
f_prep_docker_image() {
pushd ${g_workdir}
bin/make-base-vm --docker --suite bionic --arch amd64
popd
}
f_ensure_go_source_tarball() {
local l_cached_tar
l_cached_tar="${g_gitian_cache}/${GO_TARBALL}"
mkdir -p ${g_workdir}/inputs
if [ -f "${l_cached_tar}" ]; then
echo "Fetching cached tarball from ${l_cached_tar}" >&2
cp "${l_cached_tar}" ${g_workdir}/inputs/${GO_TARBALL}
else
f_download_go
echo "Caching ${GO_TARBALL}" >&2
cp ${g_workdir}/inputs/${GO_TARBALL} "${l_cached_tar}"
fi
}
f_download_go() {
local l_remote
l_remote=https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}
curl -L "${l_remote}/${GO_TARBALL}" > ${g_workdir}/inputs/${GO_TARBALL}
}
f_build() {
local l_sdk l_descriptor
l_descriptor=$1
l_commit=$2
[ -f ${l_descriptor} ]
cd ${g_workdir}
export USE_DOCKER=1
bin/gbuild --commit cosmos-sdk="$l_commit" ${GBUILD_FLAGS} "$l_descriptor"
libexec/stop-target || echo "warning: couldn't stop target" >&2
}
f_sign() {
local l_descriptor l_release_name l_sigs_dir
l_descriptor=$1
l_release_name=$2
l_sigs_dir=$3
pushd ${g_workdir}
bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${l_sigs_dir}" --release=${l_release_name} ${l_descriptor}
popd
}
f_verify() {
local l_descriptor l_release_name l_sigs_dir
l_descriptor=$1
l_release_name=$2
l_sigs_dir=$3
pushd ${g_workdir}
bin/gverify --destination="${l_sigs_dir}" --release="${l_release_name}" ${l_descriptor}
popd
}
f_validate_platform() {
case "${1}" in
linux|darwin|windows)
;;
*)
echo "invalid platform -- ${1}"
exit 1
esac
}
f_abspath() {
echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")"
}
f_help() {
cat >&2 <<EOF
Usage: $(basename $0) [-h] GOOS GIT_REPO
Launch a gitian build from the local clone of cosmos-sdk available at GIT_REPO.
Options:
-h display this help and exit
-d DIRNAME set working directory name and skip gitian-builder download
-s IDENTITY sign build as IDENTITY
If a GPG identity is supplied via the -s flag, the build will be signed and verified.
The signature will be saved in '${DEFAULT_GAIA_SIGS}/'. An alternative output directory
for signatures can be supplied via the environment variable \$GAIA_SIGS.
The default signing command used to sign the build is '$DEFAULT_SIGN_COMMAND'.
An alternative signing command can be supplied via the environment
variable \$SIGN_COMMAND.
EOF
}
while getopts ":d:s:h" opt; do
case "${opt}" in
h) f_help ; exit 0 ;;
d) g_dirname="${OPTARG}" ;;
s) g_sign_identity="${OPTARG}" ;;
esac
done
shift "$((OPTIND-1))"
g_platform="${1}"
f_validate_platform "${g_platform}"
g_dirname="${g_dirname:-gitian-build-${g_platform}}"
g_workdir="$(pwd)/${g_dirname}"
if [ -d "${g_workdir}" ]; then
echo "Directory ${g_workdir} exists and will be preserved" >&2
g_gitian_skip_download=y
fi
g_sdk="$(f_abspath ${2})"
[ -d "${g_sdk}" ]
g_sigs_dir=${GAIA_SIGS:-"$(pwd)/${DEFAULT_GAIA_SIGS}"}
# create local cache directory for all gitian builds
g_gitian_cache="$(pwd)/${GITIAN_CACHE_DIRNAME}"
echo "Ensure cache directory ${g_gitian_cache} exists" >&2
mkdir -p "${g_gitian_cache}"
f_main "${g_platform}" "${g_sdk}" "${g_sigs_dir}"

116
cmd/gaia/contrib/gitian-descriptors/gitian-darwin.yml Normal file
View File

@ -0,0 +1,116 @@
---
name: "gaia-darwin"
enable_cache: true
distro: "ubuntu"
suites:
- "bionic"
architectures:
- "amd64"
packages:
- "bsdmainutils"
- "build-essential"
- "ca-certificates"
- "curl"
- "debhelper"
- "dpkg-dev"
- "devscripts"
- "fakeroot"
- "git"
- "golang-any"
- "xxd"
- "quilt"
remotes:
- "url": "https://github.com/cosmos/cosmos-sdk.git"
"dir": "cosmos-sdk"
files:
- "golang-debian-1.12.5-1.tar.gz"
script: |
set -e -o pipefail
GO_SRC_RELEASE=golang-debian-1.12.5-1
GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz"
# Compile go and configure the environment
export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
export BUILD_DIR=`pwd`
tar xf "${GO_SRC_TARBALL}"
rm -f "${GO_SRC_TARBALL}"
[ -d "${GO_SRC_RELEASE}/" ]
mv "${GO_SRC_RELEASE}/" go/
pushd go/
QUILT_PATCHES=debian/patches quilt push -a
fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache
popd
export GOOS=darwin
export GOROOT=${BUILD_DIR}/go
export GOPATH=${BUILD_DIR}/gopath
mkdir -p ${GOPATH}/bin
export PATH_orig=${PATH}
export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
export ARCHS='386 amd64'
export GO111MODULE=on
# Make release tarball
pushd cosmos-sdk
VERSION=$(git describe --tags | sed 's/^v//')
COMMIT=$(git log -1 --format='%H')
DISTNAME=gaia-${VERSION}
git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD
SOURCEDIST=`pwd`/`echo gaia-*.tar.gz`
popd
# Correct tar file order
mkdir -p temp
pushd temp
tar xf $SOURCEDIST
rm $SOURCEDIST
find gaia-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST
popd
# Prepare GOPATH and install deps
distsrc=${GOPATH}/src/github.com/cosmos/cosmos-sdk
mkdir -p ${distsrc}
pushd ${distsrc}
tar --strip-components=1 -xf $SOURCEDIST
go mod download
popd
# Configure LDFLAGS for reproducible builds
LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \
-X github.com/cosmos/cosmos-sdk/version.Name=gaia \
-X github.com/cosmos/cosmos-sdk/version.Version=${VERSION} \
-X github.com/cosmos/cosmos-sdk/version.Commit=${COMMIT} \
-X github.com/cosmos/cosmos-sdk/version.BuildTags=netgo,ledger"
# Extract release tarball and build
for arch in ${ARCHS}; do
INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch}
mkdir -p ${INSTALLPATH}
# Build gaia tool suite
pushd ${distsrc}
for prog in gaiacli gaiad; do
GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \
-gcflags=all=-trimpath=${GOPATH} \
-asmflags=all=-trimpath=${GOPATH} \
-mod=readonly -tags "netgo ledger" \
-ldflags="${LDFLAGS}" \
-o ${INSTALLPATH}/${prog} ./cmd/gaia/cmd/${prog}
done
popd # ${distsrc}
pushd ${INSTALLPATH}
find -type f | sort | tar \
--no-recursion --mode='u+rw,go+r-w,a+X' \
--numeric-owner --sort=name \
--owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-darwin-${arch}.tar.gz
popd # installed
done
rm -rf ${distsrc}
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src

115
cmd/gaia/contrib/gitian-descriptors/gitian-linux.yml Normal file
View File

@ -0,0 +1,115 @@
---
name: "gaia-linux"
enable_cache: true
distro: "ubuntu"
suites:
- "bionic"
architectures:
- "amd64"
packages:
- "bsdmainutils"
- "build-essential"
- "ca-certificates"
- "curl"
- "debhelper"
- "dpkg-dev"
- "devscripts"
- "fakeroot"
- "git"
- "golang-any"
- "xxd"
- "quilt"
remotes:
- "url": "https://github.com/cosmos/cosmos-sdk.git"
"dir": "cosmos-sdk"
files:
- "golang-debian-1.12.5-1.tar.gz"
script: |
set -e -o pipefail
GO_SRC_RELEASE=golang-debian-1.12.5-1
GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz"
# Compile go and configure the environment
export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
export BUILD_DIR=`pwd`
tar xf "${GO_SRC_TARBALL}"
rm -f "${GO_SRC_TARBALL}"
[ -d "${GO_SRC_RELEASE}/" ]
mv "${GO_SRC_RELEASE}/" go/
pushd go/
QUILT_PATCHES=debian/patches quilt push -a
fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache
popd
export GOROOT=${BUILD_DIR}/go
export GOPATH=${BUILD_DIR}/gopath
mkdir -p ${GOPATH}/bin
export PATH_orig=${PATH}
export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
export ARCHS='386 amd64 arm arm64'
export GO111MODULE=on
# Make release tarball
pushd cosmos-sdk
VERSION=$(git describe --tags | sed 's/^v//')
COMMIT=$(git log -1 --format='%H')
DISTNAME=gaia-${VERSION}
git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD
SOURCEDIST=`pwd`/`echo gaia-*.tar.gz`
popd
# Correct tar file order
mkdir -p temp
pushd temp
tar xf $SOURCEDIST
rm $SOURCEDIST
find gaia-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST
popd
# Prepare GOPATH and install deps
distsrc=${GOPATH}/src/github.com/cosmos/cosmos-sdk
mkdir -p ${distsrc}
pushd ${distsrc}
tar --strip-components=1 -xf $SOURCEDIST
go mod download
popd
# Configure LDFLAGS for reproducible builds
LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \
-X github.com/cosmos/cosmos-sdk/version.Name=gaia \
-X github.com/cosmos/cosmos-sdk/version.Version=${VERSION} \
-X github.com/cosmos/cosmos-sdk/version.Commit=${COMMIT} \
-X github.com/cosmos/cosmos-sdk/version.BuildTags=netgo,ledger"
# Extract release tarball and build
for arch in ${ARCHS}; do
INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch}
mkdir -p ${INSTALLPATH}
# Build gaia tool suite
pushd ${distsrc}
for prog in gaiacli gaiad; do
GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \
-gcflags=all=-trimpath=${GOPATH} \
-asmflags=all=-trimpath=${GOPATH} \
-mod=readonly -tags "netgo ledger" \
-ldflags="${LDFLAGS}" \
-o ${INSTALLPATH}/${prog} ./cmd/gaia/cmd/${prog}
done
popd # ${distsrc}
pushd ${INSTALLPATH}
find -type f | sort | tar \
--no-recursion --mode='u+rw,go+r-w,a+X' \
--numeric-owner --sort=name \
--owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-linux-${arch}.tar.gz
popd # installed
done
rm -rf ${distsrc}
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src

116
cmd/gaia/contrib/gitian-descriptors/gitian-windows.yml Normal file
View File

@ -0,0 +1,116 @@
---
name: "gaia-windows"
enable_cache: true
distro: "ubuntu"
suites:
- "bionic"
architectures:
- "amd64"
packages:
- "bsdmainutils"
- "build-essential"
- "ca-certificates"
- "curl"
- "debhelper"
- "dpkg-dev"
- "devscripts"
- "fakeroot"
- "git"
- "golang-any"
- "xxd"
- "quilt"
remotes:
- "url": "https://github.com/cosmos/cosmos-sdk.git"
"dir": "cosmos-sdk"
files:
- "golang-debian-1.12.5-1.tar.gz"
script: |
set -e -o pipefail
GO_SRC_RELEASE=golang-debian-1.12.5-1
GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz"
# Compile go and configure the environment
export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
export BUILD_DIR=`pwd`
tar xf "${GO_SRC_TARBALL}"
rm -f "${GO_SRC_TARBALL}"
[ -d "${GO_SRC_RELEASE}/" ]
mv "${GO_SRC_RELEASE}/" go/
pushd go/
QUILT_PATCHES=debian/patches quilt push -a
fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache
popd
export GOROOT=${BUILD_DIR}/go
export GOPATH=${BUILD_DIR}/gopath
mkdir -p ${GOPATH}/bin
export PATH_orig=${PATH}
export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
export ARCHS='386 amd64'
export GO111MODULE=on
# Make release tarball
pushd cosmos-sdk
VERSION=$(git describe --tags | sed 's/^v//')
COMMIT=$(git log -1 --format='%H')
DISTNAME=gaia-${VERSION}
git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD
SOURCEDIST=`pwd`/`echo gaia-*.tar.gz`
popd
# Correct tar file order
mkdir -p temp
pushd temp
tar xf $SOURCEDIST
rm $SOURCEDIST
find gaia-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST
popd
# Prepare GOPATH and install deps
distsrc=${GOPATH}/src/github.com/cosmos/cosmos-sdk
mkdir -p ${distsrc}
pushd ${distsrc}
tar --strip-components=1 -xf $SOURCEDIST
go mod download
popd
# Configure LDFLAGS for reproducible builds
LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \
-X github.com/cosmos/cosmos-sdk/version.Name=gaia \
-X github.com/cosmos/cosmos-sdk/version.Version=${VERSION} \
-X github.com/cosmos/cosmos-sdk/version.Commit=${COMMIT} \
-X github.com/cosmos/cosmos-sdk/version.BuildTags=netgo,ledger"
# Extract release tarball and build
for arch in ${ARCHS}; do
INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch}
mkdir -p ${INSTALLPATH}
# Build gaia tool suite
pushd ${distsrc}
for prog in gaiacli gaiad; do
exe=${prog}.exe
GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \
-gcflags=all=-trimpath=${GOPATH} \
-asmflags=all=-trimpath=${GOPATH} \
-mod=readonly -tags "netgo ledger" \
-ldflags="${LDFLAGS}" \
-o ${INSTALLPATH}/${exe} ./cmd/gaia/cmd/${prog}
done
popd # ${distsrc}
pushd ${INSTALLPATH}
find -type f | sort | tar \
--no-recursion --mode='u+rw,go+r-w,a+X' \
--numeric-owner --sort=name \
--owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-windows-${arch}.tar.gz
popd # installed
done
rm -rf ${distsrc}
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src

29
cmd/gaia/contrib/gitian-keys/README.md Normal file
View File

@ -0,0 +1,29 @@
## PGP keys of Gitian builders and Gaia Developers
The file `keys.txt` contains fingerprints of the public keys of Gitian builders
and active developers.
The associated keys are mainly used to sign git commits or the build results
of Gitian builds.
The most recent version of each pgp key can be found on most PGP key servers.
Fetch the latest version from the key server to see if any key was revoked in
the meantime.
To fetch the latest version of all pgp keys in your gpg homedir,
```sh
gpg --refresh-keys
```
To fetch keys of Gitian builders and active core developers, feed the list of
fingerprints of the primary keys into gpg:
```sh
while read fingerprint keyholder_name; \
do gpg --keyserver hkp://subset.pool.sks-keyservers.net \
--recv-keys ${fingerprint}; done < ./keys.txt
```
Add your key to the list if you are a Gaia core developer or you have
provided Gitian signatures for two major or minor releases of Gaia.

1
cmd/gaia/contrib/gitian-keys/keys.txt Normal file
View File

@ -0,0 +1 @@
04160004A8276E40BB9890FBE8A48AE5311D765A Alessio Treglia

12
cmd/gaia/sims.mk
View File

@ -3,8 +3,8 @@
########################################
### Simulations
runsim: $(GOBIN)/runsim
$(GOBIN)/runsim: contrib/runsim/main.go
runsim: $(GOPATH)/bin/runsim
$(GOPATH)/bin/runsim: contrib/runsim/main.go
go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
sim-gaia-nondeterminism:
@ -23,20 +23,20 @@ sim-gaia-fast:
sim-gaia-import-export: runsim
@echo "Running Gaia import/export simulation. This may take several minutes..."
$(GOBIN)/runsim 50 5 TestGaiaImportExport
$(GOPATH)/bin/runsim 50 5 TestGaiaImportExport
sim-gaia-simulation-after-import: runsim
@echo "Running Gaia simulation-after-import. This may take several minutes..."
$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
$(GOPATH)/bin/runsim 50 5 TestGaiaSimulationAfterImport
sim-gaia-custom-genesis-multi-seed: runsim
@echo "Running multi-seed custom genesis simulation..."
@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
$(GOPATH)/bin/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
sim-gaia-multi-seed: runsim
@echo "Running multi-seed Gaia simulation. This may take awhile!"
$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
$(GOPATH)/bin/runsim 400 5 TestFullGaiaSimulation
sim-benchmark-invariants:
@echo "Running simulation invariant benchmarks..."

22
contrib/devtools/Makefile
View File

@ -42,20 +42,21 @@ mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd)
# tools
###
GOLANGCI_LINT = $(GOBIN)/golangci-lint
STATIK = $(GOBIN)/statik
GOIMPORTS = $(GOBIN)/goimports
GOSUM = $(GOBIN)/gosum
CLOG = $(GOBIN)/clog
TOOLS_DESTDIR ?= $(GOPATH)/bin
GOLANGCI_LINT = $(TOOLS_DESTDIR)/golangci-lint
STATIK = $(TOOLS_DESTDIR)/statik
GOIMPORTS = $(TOOLS_DESTDIR)/goimports
CLOG = $(TOOLS_DESTDIR)/clog
all: tools
tools: tools-stamp
tools-stamp: $(GOLANGCI_LINT) $(STATIK) $(GOIMPORTS) $(GOSUM) $(CLOG)
tools-stamp: $(STATIK) $(GOIMPORTS) $(CLOG) $(GOLANGCI_LINT)
touch $@
$(GOLANGCI_LINT): $(mkfile_dir)/install-golangci-lint.sh $(GOSUM)
bash $(mkfile_dir)/install-golangci-lint.sh $(GOBIN) $(GOLANGCI_LINT_VERSION) $(GOLANGCI_LINT_HASHSUM)
$(GOLANGCI_LINT): $(mkfile_dir)/install-golangci-lint.sh
bash $(mkfile_dir)/install-golangci-lint.sh $(TOOLS_DESTDIR) $(GOLANGCI_LINT_VERSION) $(GOLANGCI_LINT_HASHSUM)
$(STATIK):
$(call go_install,rakyll,statik,v0.1.5)
@ -63,14 +64,11 @@ $(STATIK):
$(GOIMPORTS):
go get golang.org/x/tools/cmd/goimports@v0.0.0-20190114222345-bf090417da8b
$(GOSUM): ./contrib/devtools/gosum/main.go
go install -mod=readonly ./$(<D)/
$(CLOG):
$(call go_install,alessio,clog,1)
tools-clean:
rm -f $(GOLANGCI_LINT) $(STATIK) $(GOIMPORTS) $(GOSUM) $(CLOG)
rm -f $(STATIK) $(GOIMPORTS) $(CLOG) $(GOLANGCI_LINT)
rm -f tools-stamp
.PHONY: all tools tools-clean

27
contrib/devtools/gosum/main.go
View File

@ -1,27 +0,0 @@
package main
import (
"crypto/sha256"
"fmt"
"io"
"log"
"os"
)
// DONTCOVER
// nolint: errcheck
func main() {
f, err := os.Open(os.Args[1])
if err != nil {
log.Fatal(err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
log.Fatal(err)
}
fmt.Printf("%x", h.Sum(nil))
}

9
contrib/devtools/install-golangci-lint.sh
View File

@ -2,6 +2,12 @@
set -euo pipefail
f_sha256() {
local l_file
l_file=$1
python -sBc "import hashlib;print(hashlib.sha256(open('$l_file','rb').read()).hexdigest())"
}
installer="$(mktemp)"
trap "rm -f ${installer}" EXIT
@ -9,13 +15,12 @@ GOBIN="${1}"
VERSION="${2}"
HASHSUM="${3}"
CURL="$(which curl)"
GOSUM="$(which gosum)"
echo "Downloading golangci-lint ${VERSION} installer ..." >&2
"${CURL}" -sfL "https://raw.githubusercontent.com/golangci/golangci-lint/${VERSION}/install.sh" > "${installer}"
echo "Checking hashsum ..." >&2
[ "${HASHSUM}" = "$(${GOSUM} ${installer})" ]
[ "${HASHSUM}" = "$(f_sha256 ${installer})" ]
chmod +x "${installer}"
echo "Launching installer ..." >&2

5
docs/cosmos-hub/installation.md
View File

@ -4,13 +4,12 @@ This guide will explain how to install the `gaiad` and `gaiacli` entrypoints ont
### Install Go
Install `go` by following the [official docs](https://golang.org/doc/install). Remember to set your `$GOPATH`, `$GOBIN`, and `$PATH` environment variables, for example:
Install `go` by following the [official docs](https://golang.org/doc/install). Remember to set your `$GOPATH` and `$PATH` environment variables, for example:
```bash
mkdir -p $HOME/go/bin
echo "export GOPATH=$HOME/go" >> ~/.bash_profile
echo "export GOBIN=\$GOPATH/bin" >> ~/.bash_profile
echo "export PATH=\$PATH:\$GOBIN" >> ~/.bash_profile
echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.bash_profile
source ~/.bash_profile
```

82
docs/cosmos-hub/reproducible-builds.md Normal file
View File

@ -0,0 +1,82 @@
## Build Gaia Deterministically
Gitian is the deterministic build process that is used to build the Gaia executables. It provides a way to be reasonably sure that the executables are really built from the git source. It also makes sure that the same, tested dependencies are used and statically built into the executable.
Multiple developers build the source code by following a specific descriptor ("recipe"), cryptographically sign the result, and upload the resulting signature. These results are compared and only if they match, the build is accepted and provided for download.
More independent Gitian builders are needed, which is why this guide exists. It is preferred you follow these steps yourself instead of using someone else's VM image to avoid 'contaminating' the build.
This page contains all instructions required to build and sign reproducible Gaia binaries for Linux, Mac OS X, and Windows.
## Prerequisites
Make sure your system satisfy minimum requisites as outlined in https://github.com/devrandom/gitian-builder#prerequisites.
All the following instructions have been tested on *Ubuntu 18.04.2 LTS* with *docker 18.06.1-ce* and *docker 18.09.6-ce*.
If you are on Mac OS X, make sure you have prepended your `PATH` environment variable with GNU coreutils's path before running the build script:
```
export PATH=/usr/local/opt/coreutils/libexec/gnubin/:$PATH
```
## Build and sign
Clone cosmos-sdk:
```
git clone git@github.com:cosmos/cosmos-sdk
```
Checkout the commit, branch, or release tag you want to build:
```
cd cosmos-sdk/
git checkout v0.35.0
```
Run the following command to launch a build for `linux` and sign the final build
report (replace `user@example.com` with the GPG identity you want to sign the report with):
```
./cmd/gaia/contrib/gitian-build.sh -s user@example.com linux `pwd`
```
The above command generates two directories in the current working directory:
* `gitian-build-linux` containing the `gitian-builder` clone used to drive the build process.
* `gaia.sigs` containing the signed build report.
Replace `linux` in the above command with `darwin` or `windows` to run builds for Mac OS X and Windows respectively.
Run the following command to build binaries for all platforms (`darwin`, `linux`, and `windows`):
```
cd cosmos-sdk/
for platform in darwin linux windows; do ./cmd/gaia/contrib/gitian-build.sh -s user@example.com $platform `pwd`; done
```
If you want to generate unsigned builds, just remove the option `-s` from the command line:
```
./cmd/gaia/contrib/gitian-build.sh linux `pwd`
```
At the end of the procedure, build results can be found in the `./gaia.sigs` directory:
Please refer to the `cmd/gaia/contrib/gitian-build.sh`'s help screen for further information on its usage.
## Signatures upload
Once signatures are generated, they could be uploaded to gaia's dedicated repository: https://github.com/cosmos/gaia.sigs.
## Troubleshooting
### Docker gitian-target container cannot be killed
The issue is due to a relatively recent kernel apparmor change, [see here](https://github.com/moby/moby/issues/36809#issuecomment-379325713) for more information on a potential mitigation for the issue.
On Ubuntu 18.04, when the container hangs and `docker` is unable to kill it you can try to use `pkill` to forcibly terminate `containerd-shim`:
```
sudo pkill containerd-shim
docker system prune
```

3
docs/translations/cn/cosmos-hub/installation.md
View File

@ -9,8 +9,7 @@
```bash
mkdir -p $HOME/go/bin
echo "export GOPATH=$HOME/go" >> ~/.bash_profile
echo "export GOBIN=\$GOPATH/bin" >> ~/.bash_profile
echo "export PATH=\$PATH:\$GOBIN" >> ~/.bash_profile
echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.bash_profile
echo "export GO111MODULE=on" >> ~/.bash_profile
source ~/.bash_profile
```

5
docs/translations/kr/gaia/installation.md
View File

@ -4,13 +4,12 @@
### Go 설치하기
공식 [Go 문서](https://golang.org/doc/install)를 따라서 `go`를 설치하십시오. `$GOPATH`, `$GOBIN`, 그리고 `$PATH`의 환경을 꼭 세팅하세요. 예시:
공식 [Go 문서](https://golang.org/doc/install)를 따라서 `go`를 설치하십시오. `$GOPATH`, 그리고 `$PATH`의 환경을 꼭 세팅하세요. 예시:
```bash
mkdir -p $HOME/go/bin
echo "export GOPATH=$HOME/go" >> ~/.bash_profile
echo "export GOBIN=\$GOPATH/bin" >> ~/.bash_profile
echo "export PATH=\$PATH:\$GOBIN" >> ~/.bash_profile
echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.bash_profile
echo "export GO111MODULE=on" >> ~/.bash_profile
source ~/.bash_profile
```

10
version/version.go
View File

@ -6,9 +6,8 @@
// Configure the version command
//
// The version command can be just added to your cobra root command.
// At build time, the variables Name, Version, Commit, GoSumHash, and
// BuildTags can be passed as build flags as shown in the following
// example:
// At build time, the variables Name, Version, Commit, and BuildTags
// can be passed as build flags as shown in the following example:
//
// go build -X github.com/cosmos/cosmos-sdk/version.Name=dapp \
// -X github.com/cosmos/cosmos-sdk/version.Version=1.0 \
@ -38,7 +37,6 @@ type versionInfo struct {
Name string `json:"name"`
Version string `json:"version"`
GitCommit string `json:"commit"`
GoSumHash string `json:"gosum_hash"`
BuildTags string `json:"build_tags"`
GoVersion string `json:"go"`
}
@ -46,9 +44,8 @@ type versionInfo struct {
func (v versionInfo) String() string {
return fmt.Sprintf(`%s: %s
git commit: %s
go.sum hash: %s
build tags: %s
%s`, v.Name, v.Version, v.GitCommit, v.GoSumHash, v.BuildTags, v.GoVersion)
%s`, v.Name, v.Version, v.GitCommit, v.BuildTags, v.GoVersion)
}
func newVersionInfo() versionInfo {
@ -56,7 +53,6 @@ func newVersionInfo() versionInfo {
Name,
Version,
Commit,
GoSumHash,
BuildTags,
fmt.Sprintf("go version %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH)}
}