Speed up CircleCI builds
To achieve faster feedback cycles for our feature PRs this change reduces the average buildtime from 35 to ~6min by utilising their new 2.0 offering based on docker and nomad. We make use of parallel build steps wherever possible so that the duration is determined by the slowest test suite (p2p). This is an intermediate step until we move our CI/CD completely on-premise for more control and added security.
This commit is contained in:
parent
8d81a259c7
commit
b7ce89e568
|
@ -0,0 +1,222 @@
|
|||
version: 2
|
||||
|
||||
defaults: &defaults
|
||||
working_directory: /go/src/github.com/tendermint/tendermint
|
||||
docker:
|
||||
- image: circleci/golang:1.10.0
|
||||
environment:
|
||||
GOBIN: /tmp/workspace/bin
|
||||
|
||||
jobs:
|
||||
setup_dependencies:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- run: mkdir -p /tmp/workspace/bin
|
||||
- run: mkdir -p /tmp/workspace/profiles
|
||||
- checkout
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-pkg-cache
|
||||
- run:
|
||||
name: tools
|
||||
command: |
|
||||
export PATH="$GOBIN:$PATH"
|
||||
make get_tools
|
||||
- run:
|
||||
name: dependencies
|
||||
command: |
|
||||
export PATH="$GOBIN:$PATH"
|
||||
make get_vendor_deps
|
||||
- run:
|
||||
name: binaries
|
||||
command: |
|
||||
export PATH="$GOBIN:$PATH"
|
||||
make install
|
||||
- persist_to_workspace:
|
||||
root: /tmp/workspace
|
||||
paths:
|
||||
- bin
|
||||
- profiles
|
||||
- save_cache:
|
||||
key: v1-pkg-cache
|
||||
paths:
|
||||
- /go/pkg
|
||||
- save_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
paths:
|
||||
- /go/src/github.com/tendermint/tendermint
|
||||
|
||||
setup_abci:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-pkg-cache
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run:
|
||||
name: Checkout abci
|
||||
command: |
|
||||
commit=$(bash scripts/dep_utils/parse.sh abci)
|
||||
go get -v -u -d github.com/tendermint/abci/...
|
||||
cd /go/src/github.com/tendermint/abci
|
||||
git checkout "$commit"
|
||||
- run:
|
||||
working_directory: /go/src/github.com/tendermint/abci
|
||||
name: Install abci
|
||||
command: |
|
||||
set -ex
|
||||
export PATH="$GOBIN:$PATH"
|
||||
make get_tools
|
||||
make get_vendor_deps
|
||||
make install
|
||||
- run: ls -lah /tmp/workspace/bin
|
||||
- persist_to_workspace:
|
||||
root: /tmp/workspace
|
||||
paths:
|
||||
- "bin/abci*"
|
||||
|
||||
lint:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-pkg-cache
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run:
|
||||
name: metalinter
|
||||
command: |
|
||||
set -ex
|
||||
export PATH="$GOBIN:$PATH"
|
||||
make metalinter
|
||||
|
||||
test_apps:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-pkg-cache
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils
|
||||
- run:
|
||||
name: Run tests
|
||||
command: bash test/app/test.sh
|
||||
|
||||
test_cover:
|
||||
<<: *defaults
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-pkg-cache
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
|
||||
id=$(basename "$pkg")
|
||||
|
||||
go test -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg"
|
||||
done
|
||||
- persist_to_workspace:
|
||||
root: /tmp/workspace
|
||||
paths:
|
||||
- "profiles/*"
|
||||
|
||||
test_libs:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-pkg-cache
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run:
|
||||
name: Run tests
|
||||
command: bash test/test_libs.sh
|
||||
|
||||
test_persistence:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-pkg-cache
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run:
|
||||
name: Run tests
|
||||
command: bash test/persist/test_failure_indices.sh
|
||||
|
||||
test_p2p:
|
||||
environment:
|
||||
GOBIN: /home/circleci/.go_workspace/bin
|
||||
GOPATH: /home/circleci/.go_workspace
|
||||
machine:
|
||||
image: circleci/classic:latest
|
||||
steps:
|
||||
- checkout
|
||||
- run: mkdir -p $GOPATH/src/github.com/tendermint
|
||||
- run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
|
||||
- run: bash test/circleci/p2p.sh
|
||||
|
||||
upload_coverage:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /tmp/workspace
|
||||
- restore_cache:
|
||||
key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
|
||||
- run:
|
||||
name: gather
|
||||
command: |
|
||||
set -ex
|
||||
|
||||
echo "mode: atomic" > coverage.txt
|
||||
for prof in $(ls /tmp/workspace/profiles/); do
|
||||
tail -n +2 /tmp/workspace/profiles/"$prof" >> coverage.txt
|
||||
done
|
||||
- run:
|
||||
name: upload
|
||||
command: bash <(curl -s https://codecov.io/bash) -f coverage.txt
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
test-suite:
|
||||
jobs:
|
||||
- setup_dependencies
|
||||
- setup_abci:
|
||||
requires:
|
||||
- setup_dependencies
|
||||
- lint:
|
||||
requires:
|
||||
- setup_dependencies
|
||||
- test_apps:
|
||||
requires:
|
||||
- setup_abci
|
||||
- test_cover:
|
||||
requires:
|
||||
- setup_dependencies
|
||||
- test_libs:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
- master
|
||||
requires:
|
||||
- setup_dependencies
|
||||
- test_persistence:
|
||||
requires:
|
||||
- setup_abci
|
||||
- test_p2p
|
||||
- upload_coverage:
|
||||
requires:
|
||||
- test_cover
|
8
Makefile
8
Makefile
|
@ -1,6 +1,6 @@
|
|||
GOTOOLS = \
|
||||
github.com/golang/dep/cmd/dep \
|
||||
# gopkg.in/alecthomas/gometalinter.v2
|
||||
gopkg.in/alecthomas/gometalinter.v2
|
||||
PACKAGES=$(shell go list ./... | grep -v '/vendor/')
|
||||
BUILD_TAGS?=tendermint
|
||||
BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
|
||||
|
@ -40,7 +40,7 @@ check_tools:
|
|||
get_tools:
|
||||
@echo "--> Installing tools"
|
||||
go get -u -v $(GOTOOLS)
|
||||
# @gometalinter.v2 --install
|
||||
@gometalinter.v2 --install
|
||||
|
||||
update_tools:
|
||||
@echo "--> Updating tools"
|
||||
|
@ -79,7 +79,7 @@ test:
|
|||
|
||||
test_race:
|
||||
@echo "--> Running go test --race"
|
||||
@go test -v -race $(PACKAGES)
|
||||
@go test -race $(PACKAGES)
|
||||
|
||||
test_integrations:
|
||||
@bash ./test/test.sh
|
||||
|
@ -105,7 +105,7 @@ fmt:
|
|||
|
||||
metalinter:
|
||||
@echo "--> Running linter"
|
||||
gometalinter.v2 --vendor --deadline=600s --disable-all \
|
||||
@gometalinter.v2 --vendor --deadline=600s --disable-all \
|
||||
--enable=deadcode \
|
||||
--enable=gosimple \
|
||||
--enable=misspell \
|
||||
|
|
35
circle.yml
35
circle.yml
|
@ -1,35 +0,0 @@
|
|||
---
|
||||
machine:
|
||||
environment:
|
||||
MACH_PREFIX: tendermint-test-mach
|
||||
DOCKER_VERSION: 1.10.0
|
||||
DOCKER_MACHINE_VERSION: 0.9.0
|
||||
GOPATH: "$HOME/.go_project"
|
||||
PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
|
||||
PROJECT_PATH: "$PROJECT_PARENT_PATH/$CIRCLE_PROJECT_REPONAME"
|
||||
PATH: "$HOME/.go_project/bin:${PATH}"
|
||||
hosts:
|
||||
localhost: 127.0.0.1
|
||||
|
||||
dependencies:
|
||||
override:
|
||||
- curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | sudo bash -s -- $DOCKER_VERSION
|
||||
- sudo start docker
|
||||
- sudo curl -sSL -o /usr/bin/docker-machine "https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-`uname -s`-`uname -m`"; sudo chmod 0755 /usr/bin/docker-machine
|
||||
- mkdir -p "$PROJECT_PARENT_PATH"
|
||||
- ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH"
|
||||
post:
|
||||
- go version
|
||||
- docker version
|
||||
- docker-machine version
|
||||
|
||||
test:
|
||||
override:
|
||||
- cd "$PROJECT_PATH" && set -o pipefail && make test_integrations 2>&1 | tee test_integrations.log:
|
||||
timeout: 1800
|
||||
post:
|
||||
- cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
|
||||
- cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
|
||||
- cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"
|
||||
- cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker.log"
|
||||
- cd "${CIRCLE_ARTIFACTS}" && tar czf logs.tar.gz *.log
|
|
@ -248,6 +248,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) {
|
|||
require.Nil(err)
|
||||
|
||||
peer := sw.Peers().Get(rp.ID())
|
||||
require.NotNil(peer)
|
||||
|
||||
// simulate failure by closing connection
|
||||
pc.CloseConn()
|
||||
|
@ -274,10 +275,11 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) {
|
|||
pc, err := newOutboundPeerConn(rp.Addr(), DefaultPeerConfig(), true, sw.nodeKey.PrivKey)
|
||||
// sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodeKey.PrivKey,
|
||||
require.Nil(err)
|
||||
err = sw.addPeer(pc)
|
||||
require.Nil(err)
|
||||
|
||||
require.Nil(sw.addPeer(pc))
|
||||
|
||||
peer := sw.Peers().Get(rp.ID())
|
||||
require.NotNil(peer)
|
||||
|
||||
// simulate failure by closing connection
|
||||
pc.CloseConn()
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
#! /bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
set +u
|
||||
if [[ "$DEP" == "" ]]; then
|
||||
DEP=$GOPATH/src/github.com/tendermint/tendermint/Gopkg.lock
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#! /bin/bash
|
||||
set -e
|
||||
set -ex
|
||||
|
||||
#- kvstore over socket, curl
|
||||
#- counter over socket, curl
|
||||
|
@ -8,6 +8,7 @@ set -e
|
|||
|
||||
# TODO: install everything
|
||||
|
||||
export PATH="$GOBIN:$PATH"
|
||||
export TMHOME=$HOME/.tendermint_app
|
||||
|
||||
function kvstore_over_socket(){
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
#! /bin/bash
|
||||
set -eux
|
||||
|
||||
# Get the directory of where this script is.
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
LOGS_DIR="$DIR/../logs"
|
||||
echo
|
||||
echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
|
||||
rm -rf "$LOGS_DIR"
|
||||
mkdir -p "$LOGS_DIR"
|
||||
|
||||
set +e
|
||||
echo
|
||||
echo "* [$(date +"%T")] removing run_test container"
|
||||
docker rm -vf run_test
|
||||
set -e
|
||||
|
||||
echo
|
||||
echo "* [$(date +"%T")] starting rsyslog container"
|
||||
docker rm -f rsyslog || true
|
||||
docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
|
||||
|
||||
set +u
|
||||
if [[ "$SKIP_BUILD" == "" ]]; then
|
||||
echo
|
||||
echo "* [$(date +"%T")] building docker image"
|
||||
bash "$DIR/../docker/build.sh"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "* [$(date +"%T")] running p2p tests on a local docker network"
|
||||
bash "$DIR/../p2p/test.sh" tester
|
|
@ -10,6 +10,7 @@ RUN apt-get update && \
|
|||
|
||||
# Setup tendermint repo
|
||||
ENV REPO $GOPATH/src/github.com/tendermint/tendermint
|
||||
ENV GOBIN $GOPATH/bin
|
||||
WORKDIR $REPO
|
||||
|
||||
# Install the vendored dependencies before copying code
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
#! /bin/bash
|
||||
|
||||
export PATH="$GOBIN:$PATH"
|
||||
export TMHOME=$HOME/.tendermint_persist
|
||||
|
||||
rm -rf "$TMHOME"
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
#! /bin/bash
|
||||
set -e
|
||||
set -ex
|
||||
|
||||
export PATH="$GOBIN:$PATH"
|
||||
|
||||
# Get the parent directory of where this script is.
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
|
@ -18,6 +20,7 @@ for lib in "${LIBS[@]}"; do
|
|||
|
||||
echo "Testing $lib ..."
|
||||
cd "$GOPATH/src/github.com/tendermint/$lib"
|
||||
make get_tools
|
||||
make get_vendor_deps
|
||||
make test
|
||||
if [[ "$?" != 0 ]]; then
|
||||
|
|
|
@ -19,7 +19,6 @@ import (
|
|||
|
||||
const (
|
||||
defaultConnDeadlineSeconds = 3
|
||||
defaultDialRetryIntervalSeconds = 1
|
||||
defaultDialRetryMax = 10
|
||||
)
|
||||
|
||||
|
@ -56,7 +55,7 @@ type socketClient struct {
|
|||
// Check that socketClient implements PrivValidator2.
|
||||
var _ types.PrivValidator2 = (*socketClient)(nil)
|
||||
|
||||
// NewsocketClient returns an instance of socketClient.
|
||||
// NewSocketClient returns an instance of socketClient.
|
||||
func NewSocketClient(
|
||||
logger log.Logger,
|
||||
socketAddr string,
|
||||
|
@ -448,7 +447,7 @@ func readMsg(r io.Reader) (PrivValidatorSocketMsg, error) {
|
|||
|
||||
w, ok := read.(struct{ PrivValidatorSocketMsg })
|
||||
if !ok {
|
||||
return nil, errors.New("unknwon type")
|
||||
return nil, errors.New("unknown type")
|
||||
}
|
||||
|
||||
return w.PrivValidatorSocketMsg, nil
|
||||
|
|
Loading…
Reference in New Issue