From 1275458c3f0e5da47b27215f8d8800856be6e93e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 14 Feb 2017 18:53:22 +0400 Subject: [PATCH 1/4] collect and add docker logs to CircleCI artifacts (Refs #387) How: 1) we start syslog docker container 2) all other containers use syslog logging driver to ship their logs to that container --- .gitignore | 1 + circle.yml | 4 ++-- test/p2p/peer.sh | 4 ++++ test/test.sh | 45 +++++++++++++++++++++++++++++++-------------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 4d8d3395..c4694b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ remote_dump vendor .vagrant test/p2p/data/ +test/logs .glide diff --git a/circle.yml b/circle.yml index e77b578b..50ffbd01 100644 --- a/circle.yml +++ b/circle.yml @@ -29,5 +29,5 @@ test: post: - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}" - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt - - cd "$PROJECT_PATH" && [[ -f coverage.txt ]] && mv coverage.txt "${CIRCLE_ARTIFACTS}" - - sudo cp /var/log/upstart/docker.log "${CIRCLE_ARTIFACTS}" + - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}" + - cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker_logs.txt" diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh index 76314f58..19586b12 100644 --- a/test/p2p/peer.sh +++ b/test/p2p/peer.sh @@ -21,4 +21,8 @@ docker run -d \ --name local_testnet_$ID \ --entrypoint tendermint \ -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \ + --log-driver=syslog \ + --log-opt syslog-address=udp://127.0.0.1:5514 \ + --log-opt syslog-facility=daemon \ + --log-opt tag="{{.Name}}" \ $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY diff --git a/test/test.sh b/test/test.sh index 9e42235e..2abafc32 100644 --- a/test/test.sh +++ b/test/test.sh @@ -1,17 +1,30 @@ #! /bin/bash set -eu -# Top Level Testing Script -# See the github.com/tendermint/tendermint/test/README.md +# Get the directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -echo "" -echo "* building docker image" -bash ./test/docker/build.sh +LOGS_DIR="$DIR/logs" +echo +echo "* [$(date +"%T")] cleaning up $LOGS_DIR" +rm -rf "$LOGS_DIR" +mkdir -p "$LOGS_DIR" -echo "" -echo "* running go tests and app tests in docker container" +echo +echo "* [$(date +"%T")] starting rsyslog container" +docker rm -f rsyslog || true +docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog + +echo +echo "* [$(date +"%T")] building docker image" +bash "$DIR/docker/build.sh" + +echo +echo "* [$(date +"%T")] running go tests and app tests in docker container" # sometimes its helpful to mount the local test folder -# -v $GOPATH/src/github.com/tendermint/tendermint/test:/go/src/github.com/tendermint/tendermint/test +# -v $DIR:/go/src/github.com/tendermint/tendermint/test docker run --name run_test -t tester bash test/run_test.sh # copy the coverage results out of docker container @@ -19,16 +32,20 @@ docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt . # test basic network connectivity # by starting a local testnet and checking peers connect and make blocks -echo "" -echo "* running p2p tests on a local docker network" -bash test/p2p/test.sh tester +echo +echo "* [$(date +"%T")] running p2p tests on a local docker network" +bash "$DIR/p2p/test.sh" tester # only run the cloud benchmark for releases -BRANCH=`git rev-parse --abbrev-ref HEAD` +BRANCH=$(git rev-parse --abbrev-ref HEAD) if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then - echo "" + echo echo "TODO: run network tests" #echo "* branch $BRANCH; running mintnet/netmon throughput benchmark" # TODO: replace mintnet - #bash test/net/test.sh + #bash "$DIR/net/test.sh" fi + +echo +echo "* [$(date +"%T")] stopping rsyslog container" +docker rm -f rsyslog From b5bb1657d29341428de425ca2548187f3c9e0e54 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 14 Feb 2017 19:09:03 +0400 Subject: [PATCH 2/4] increase memory quota for Vagrant users `make test_integrations` requires > 2Gi of memory --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 0acce1ac..fbbdc70f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,7 +5,7 @@ Vagrant.configure("2") do |config| config.vm.box = "ubuntu/trusty64" config.vm.provider "virtualbox" do |v| - v.memory = 2048 + v.memory = 3072 v.cpus = 2 end From ad354c4c483d7211a9ab2940833227af92815d0f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 14 Feb 2017 19:23:28 +0400 Subject: [PATCH 3/4] stop containers to avoid futher errors Error: ``` Failed to remove container (rsyslog): Error response from daemon: Unable to remove filesystem ``` --- test/p2p/test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 0f29aa19..4d91aaba 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -31,3 +31,6 @@ bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP # test killing all peers bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3 + +# stop the existing testnet and remove local network +bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N From d0ca0cb308d6793b8d1ab48c4a4f29b7763b7951 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 14 Feb 2017 20:40:23 +0400 Subject: [PATCH 4/4] otherwise we end up with unmount errors due to busy disk ``` local_testnet_1 Failed to remove container (local_testnet_1): Error response from daemon: Unable to remove filesystem for 74e1104c705d4f21137bbe9e6710f33511948fc04d26475b482d1314fe70d537: remove /var/lib/docker/containers/74e1104c705d4f21137bbe9e6710f33511948fc04d26475b482d1314fe70d537/shm: device or resource busy local_testnet_2 Failed to remove container (local_testnet_2): Error response from daemon: Unable to remove filesystem for 6f2ec44e01afd85875c1d9958afa957f1623ed9464df2b4fcac06feb8254145e: remove /var/lib/docker/containers/6f2ec44e01afd85875c1d9958afa957f1623ed9464df2b4fcac06feb8254145e/shm: device or resource busy local_testnet_3 Failed to remove container (local_testnet_3): Error response from daemon: Unable to remove filesystem for 6160c73d45376920d90473a487a07f0e283e383011a0d23050b22cd82ab5d255: remove /var/lib/docker/containers/6160c73d45376920d90473a487a07f0e283e383011a0d23050b22cd82ab5d255/shm: device or resource busy local_testnet_4 Failed to remove container (local_testnet_4): Error response from daemon: Driver btrfs failed to remove root filesystem c93a30d66ff3710617915a8f6075755ebc45aebf73569420225105cc81003e2f: Failed to destroy btrfs snapshot /var/lib/docker/btrfs/subvolumes for cac56069a3682b6d133e239f479fb82a67c4ccd09b877794d9f440a1125d59d6: operation not permitted * [15:37:40] stopping rsyslog container Failed to remove container (rsyslog): Error response from daemon: Driver btrfs failed to remove root filesystem 477287a170a48156e663c55092257064e1f14770d0b6c634a9972893636d9451: Failed to destroy btrfs snapshot /var/lib/docker/btrfs/subvolumes for 76b66d582a478743c099f479bf88eb334ad303ef607f91021dfa58d28c9161eb: operation not permitted make: *** [test_integrations] Error 1 ``` --- test/p2p/test.sh | 3 --- test/test.sh | 4 ---- 2 files changed, 7 deletions(-) diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 4d91aaba..0f29aa19 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -31,6 +31,3 @@ bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP # test killing all peers bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3 - -# stop the existing testnet and remove local network -bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N diff --git a/test/test.sh b/test/test.sh index 2abafc32..9fbfd7d3 100644 --- a/test/test.sh +++ b/test/test.sh @@ -45,7 +45,3 @@ if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then # TODO: replace mintnet #bash "$DIR/net/test.sh" fi - -echo -echo "* [$(date +"%T")] stopping rsyslog container" -docker rm -f rsyslog