From 081ab1146d89f07d5d9225a4e9aa78e1d0e30a76 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 10:01:54 -0300 Subject: [PATCH 01/80] defining basic e2e tests ci --- .ci/run_e2e_tests.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .ci/run_e2e_tests.sh diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh new file mode 100644 index 0000000..222fd1b --- /dev/null +++ b/.ci/run_e2e_tests.sh @@ -0,0 +1,34 @@ +LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" +LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" +GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" + +#bash "${KURTOSIS_PATH}"/scripts/build_image.sh ${LATEST_KURTOSIS_TAG} +#docker pull ${LATEST_CONTROLLER_TAG} + +(docker run -v /var/run/docker.sock:/var/run/docker.sock \ +--env DEFAULT_GECKO_IMAGE="${DEFAULT_GECKO_IMAGE}" \ +--env TEST_CONTROLLER_IMAGE="${LATEST_CONTROLLER_TAG}" \ +${LATEST_KURTOSIS_TAG}) & + +kurtosis_pid=$! + +sleep 15 +kill ${kurtosis_pid} + +ACTUAL_EXIT_STATUS=$(docker ps -a --latest --filter ancestor=${LATEST_CONTROLLER_TAG} --format="{{.Status}}") +EXPECTED_EXIT_STATUS="Exited \(0\).*" + +echo "${ACTUAL_EXIT_STATUS}" + +# Clear containers. +echo "Clearing kurtosis testnet containers." +docker rm $(docker stop $(docker ps -a -q --filter ancestor="${GECKO_IMAGE}" --format="{{.ID}}")) >/dev/null + +if [[ ${ACTUAL_EXIT_STATUS} =~ ${EXPECTED_EXIT_STATUS} ]] +then + echo "Kurtosis test succeeded." + exit 0 +else + echo "Kurtosis test failed." + exit 1 +fi From 3eb9efded9f2fd4e5cf440b7c7869030c9f88375 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 10:03:02 -0300 Subject: [PATCH 02/80] modifying travisci yaml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a919fa0..1725f39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ install: script: - if [ "$TRAVIS_OS_NAME" = "osx" ]; then .ci/runscript_osx.sh; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then .ci/runscript_linux.sh; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then .ci/runscript_linux.sh; .ci/run_e2e_tests.sh; fi #Need to push to docker hub only from one build after_success: From 37b7440788f771a19c58c923eadcab8d4b1503a3 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 10:06:36 -0300 Subject: [PATCH 03/80] pulling images --- .ci/run_e2e_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 222fd1b..d154b99 100644 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -2,8 +2,8 @@ LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" -#bash "${KURTOSIS_PATH}"/scripts/build_image.sh ${LATEST_KURTOSIS_TAG} -#docker pull ${LATEST_CONTROLLER_TAG} +docker pull ${LATEST_CONTROLLER_TAG} +docker pull ${LATEST_KURTOSIS_TAG} (docker run -v /var/run/docker.sock:/var/run/docker.sock \ --env DEFAULT_GECKO_IMAGE="${DEFAULT_GECKO_IMAGE}" \ From d85a01631087d023b9e9155e196d91ded801a84a Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 10:09:38 -0300 Subject: [PATCH 04/80] e2e tests must be runnable --- .ci/run_e2e_tests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .ci/run_e2e_tests.sh diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh old mode 100644 new mode 100755 From 0165827857e700424f51d504bc2dcb56411c3776 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 10:28:21 -0300 Subject: [PATCH 05/80] removing osx build to debug kurtosis runs; --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1725f39..3b88aaf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ jobs: include: - os: linux dist: bionic - - os: osx - osx_image: xcode11.4 + #- os: osx + # osx_image: xcode11.4 services: - docker env: From 374062aa1ac0ca3e54a6a8b8531e28117ec35345 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 12:56:16 -0300 Subject: [PATCH 06/80] sleeping 90 --- .ci/run_e2e_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index d154b99..84ae038 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -12,7 +12,7 @@ ${LATEST_KURTOSIS_TAG}) & kurtosis_pid=$! -sleep 15 +sleep 90 kill ${kurtosis_pid} ACTUAL_EXIT_STATUS=$(docker ps -a --latest --filter ancestor=${LATEST_CONTROLLER_TAG} --format="{{.Status}}") From dea626aea362c2e36e2b758ab56139ce3478eefd Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 13:10:05 -0300 Subject: [PATCH 07/80] putting in debugging statements --- .ci/run_e2e_tests.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 84ae038..ebf9cef 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,3 +1,5 @@ +set -x + LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" @@ -5,8 +7,11 @@ GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" docker pull ${LATEST_CONTROLLER_TAG} docker pull ${LATEST_KURTOSIS_TAG} +docker image ls +echo "MY GECKO IMAGE: ${GECKO_IMAGE}" + (docker run -v /var/run/docker.sock:/var/run/docker.sock \ ---env DEFAULT_GECKO_IMAGE="${DEFAULT_GECKO_IMAGE}" \ +--env DEFAULT_GECKO_IMAGE="${GECKO_IMAGE}" \ --env TEST_CONTROLLER_IMAGE="${LATEST_CONTROLLER_TAG}" \ ${LATEST_KURTOSIS_TAG}) & From 2036c7233afe6e12fd3816af8c99ee35ca222013 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 13:50:32 -0300 Subject: [PATCH 08/80] building with docker deploy in scripts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3b88aaf..85a2176 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then .ci/before_install_linux.sh; fi install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then DOCKER_BUILDKIT=1 docker build --progress plain -t $DOCKERHUB_REPO:$COMMIT . ; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then DOCKER_BUILDKIT=1 docker build --progress plain -t $DOCKERHUB_REPO:$COMMIT -f "./scripts/Dockerfile.deploy" ; fi script: - if [ "$TRAVIS_OS_NAME" = "osx" ]; then .ci/runscript_osx.sh; fi From c6954227eb0235bf4e5f8f97fded70dc735cff5c Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 14:09:29 -0300 Subject: [PATCH 09/80] build image script --- .ci/run_e2e_tests.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index ebf9cef..2091485 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -2,11 +2,20 @@ set -x LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" -GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" +#GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" docker pull ${LATEST_CONTROLLER_TAG} docker pull ${LATEST_KURTOSIS_TAG} +SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) +SRC_PATH=$(dirname "${SCRIPTS_PATH}") + +# build docker image we need +echo $(pwd) +bash ${SRC_PATH}/scripts/build_image.sh +# get docker image label +GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) + docker image ls echo "MY GECKO IMAGE: ${GECKO_IMAGE}" From 6fac7c244625cf01fd5ca465371e5a11b3adf6e6 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 14:13:07 -0300 Subject: [PATCH 10/80] fixing travis yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85a2176..bb57fc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then .ci/before_install_linux.sh; fi install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then DOCKER_BUILDKIT=1 docker build --progress plain -t $DOCKERHUB_REPO:$COMMIT -f "./scripts/Dockerfile.deploy" ; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then DOCKER_BUILDKIT=1 docker build --progress plain -t $DOCKERHUB_REPO:$COMMIT .; fi script: - if [ "$TRAVIS_OS_NAME" = "osx" ]; then .ci/runscript_osx.sh; fi From f8cea1d2999601accfa4786c8d95c01d7db36806 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 14:25:11 -0300 Subject: [PATCH 11/80] travisyml now minimally modified to include kurtosis tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bb57fc9..f596a65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ jobs: include: - os: linux dist: bionic - #- os: osx - # osx_image: xcode11.4 + - os: osx + osx_image: xcode11.4 services: - docker env: From 72330d494ce537e8cb5d26c02d0a08c0ea6a3ddd Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 14:25:47 -0300 Subject: [PATCH 12/80] removing debugging statements --- .ci/run_e2e_tests.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 2091485..56d778d 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,8 +1,5 @@ -set -x - LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" -#GECKO_IMAGE="${DOCKERHUB_REPO}":"$COMMIT" docker pull ${LATEST_CONTROLLER_TAG} docker pull ${LATEST_KURTOSIS_TAG} @@ -11,14 +8,10 @@ SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) SRC_PATH=$(dirname "${SCRIPTS_PATH}") # build docker image we need -echo $(pwd) bash ${SRC_PATH}/scripts/build_image.sh # get docker image label GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) -docker image ls -echo "MY GECKO IMAGE: ${GECKO_IMAGE}" - (docker run -v /var/run/docker.sock:/var/run/docker.sock \ --env DEFAULT_GECKO_IMAGE="${GECKO_IMAGE}" \ --env TEST_CONTROLLER_IMAGE="${LATEST_CONTROLLER_TAG}" \ @@ -32,8 +25,6 @@ kill ${kurtosis_pid} ACTUAL_EXIT_STATUS=$(docker ps -a --latest --filter ancestor=${LATEST_CONTROLLER_TAG} --format="{{.Status}}") EXPECTED_EXIT_STATUS="Exited \(0\).*" -echo "${ACTUAL_EXIT_STATUS}" - # Clear containers. echo "Clearing kurtosis testnet containers." docker rm $(docker stop $(docker ps -a -q --filter ancestor="${GECKO_IMAGE}" --format="{{.ID}}")) >/dev/null From 1ae9c76c5344a135bc3d4f2cbcbe0223aeeb8789 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 31 May 2020 14:48:33 -0300 Subject: [PATCH 13/80] isolating the kurtosis testing --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f596a65..bb57fc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ jobs: include: - os: linux dist: bionic - - os: osx - osx_image: xcode11.4 + #- os: osx + # osx_image: xcode11.4 services: - docker env: From be6be7ae1c1e06fc29d352770836f7f0065cc63d Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Thu, 18 Jun 2020 01:45:58 -0400 Subject: [PATCH 14/80] reverted breaking changes --- genesis/genesis_test.go | 4 +- genesis/network_id.go | 2 +- main/params.go | 2 +- network/commands.go | 10 +- network/peer.go | 20 ++-- node/node.go | 2 +- .../add_default_subnet_delegator_tx.go | 2 +- .../add_default_subnet_delegator_tx_test.go | 94 +++++++++---------- 8 files changed, 69 insertions(+), 67 deletions(-) diff --git a/genesis/genesis_test.go b/genesis/genesis_test.go index c80767d..292fdee 100644 --- a/genesis/genesis_test.go +++ b/genesis/genesis_test.go @@ -26,8 +26,8 @@ func TestNetworkName(t *testing.T) { if name := NetworkName(EverestID); name != EverestName { t.Fatalf("NetworkID was incorrectly named. Result: %s ; Expected: %s", name, EverestName) } - if name := NetworkName(TestnetID); name != EverestName { - t.Fatalf("NetworkID was incorrectly named. Result: %s ; Expected: %s", name, EverestName) + if name := NetworkName(DenaliID); name != DenaliName { + t.Fatalf("NetworkID was incorrectly named. Result: %s ; Expected: %s", name, DenaliName) } if name := NetworkName(4294967295); name != "network-4294967295" { t.Fatalf("NetworkID was incorrectly named. Result: %s ; Expected: %s", name, "network-4294967295") diff --git a/genesis/network_id.go b/genesis/network_id.go index f318a36..880583e 100644 --- a/genesis/network_id.go +++ b/genesis/network_id.go @@ -18,7 +18,7 @@ var ( DenaliID uint32 = 3 EverestID uint32 = 4 - TestnetID uint32 = 4 + TestnetID uint32 = 3 LocalID uint32 = 12345 MainnetName = "mainnet" diff --git a/main/params.go b/main/params.go index 468a599..1e526b3 100644 --- a/main/params.go +++ b/main/params.go @@ -30,7 +30,7 @@ import ( ) const ( - dbVersion = "v0.6.0" + dbVersion = "v0.5.0" ) // Results of parsing the CLI diff --git a/network/commands.go b/network/commands.go index a5a9006..06fc31b 100644 --- a/network/commands.go +++ b/network/commands.go @@ -170,21 +170,23 @@ const ( Version GetPeerList PeerList - Ping - Pong // Bootstrapping: GetAcceptedFrontier AcceptedFrontier GetAccepted Accepted - GetAncestors - MultiPut // Consensus: Get Put PushQuery PullQuery Chits + + // TODO: Reorder these messages when we transition to everest + GetAncestors + MultiPut + Ping + Pong ) // Defines the messages that can be sent/received with this network diff --git a/network/peer.go b/network/peer.go index 9fd801f..409d7f6 100644 --- a/network/peer.go +++ b/network/peer.go @@ -64,7 +64,7 @@ func (p *peer) Start() { // Initially send the version to the peer go p.Version() go p.requestVersion() - go p.sendPings() + // go p.sendPings() } func (p *peer) sendPings() { @@ -107,10 +107,10 @@ func (p *peer) requestVersion() { func (p *peer) ReadMessages() { defer p.Close() - if err := p.conn.SetReadDeadline(p.net.clock.Time().Add(p.net.pingPongTimeout)); err != nil { - p.net.log.Verbo("error on setting the connection read timeout %s", err) - return - } + // if err := p.conn.SetReadDeadline(p.net.clock.Time().Add(p.net.pingPongTimeout)); err != nil { + // p.net.log.Verbo("error on setting the connection read timeout %s", err) + // return + // } pendingBuffer := wrappers.Packer{} readBuffer := make([]byte, 1<<10) @@ -246,11 +246,11 @@ func (p *peer) handle(msg Msg) { currentTime := p.net.clock.Time() atomic.StoreInt64(&p.lastReceived, currentTime.Unix()) - if err := p.conn.SetReadDeadline(currentTime.Add(p.net.pingPongTimeout)); err != nil { - p.net.log.Verbo("error on setting the connection read timeout %s, closing the connection", err) - p.Close() - return - } + // if err := p.conn.SetReadDeadline(currentTime.Add(p.net.pingPongTimeout)); err != nil { + // p.net.log.Verbo("error on setting the connection read timeout %s, closing the connection", err) + // p.Close() + // return + // } op := msg.Op() msgMetrics := p.net.message(op) diff --git a/node/node.go b/node/node.go index eae8cfd..752c78d 100644 --- a/node/node.go +++ b/node/node.go @@ -57,7 +57,7 @@ var ( genesisHashKey = []byte("genesisID") // Version is the version of this code - Version = version.NewDefaultVersion("avalanche", 0, 6, 0) + Version = version.NewDefaultVersion("avalanche", 0, 5, 5) versionParser = version.NewDefaultParser() ) diff --git a/vms/platformvm/add_default_subnet_delegator_tx.go b/vms/platformvm/add_default_subnet_delegator_tx.go index 9881652..3012d84 100644 --- a/vms/platformvm/add_default_subnet_delegator_tx.go +++ b/vms/platformvm/add_default_subnet_delegator_tx.go @@ -128,7 +128,7 @@ func (tx *addDefaultSubnetDelegatorTx) SemanticVerify(db database.Database) (*ve // The account if this block's proposal is committed and the validator is // added to the pending validator set. (Increase the account's nonce; // decrease its balance.) - newAccount, err := account.Remove(tx.Wght, tx.Nonce) // Remove also removes the fee + newAccount, err := account.Remove(0, tx.Nonce) // Remove also removes the fee if err != nil { return nil, nil, nil, nil, permError{err} } diff --git a/vms/platformvm/add_default_subnet_delegator_tx_test.go b/vms/platformvm/add_default_subnet_delegator_tx_test.go index 9380001..4b6fe80 100644 --- a/vms/platformvm/add_default_subnet_delegator_tx_test.go +++ b/vms/platformvm/add_default_subnet_delegator_tx_test.go @@ -335,9 +335,9 @@ func TestAddDefaultSubnetDelegatorTxSemanticVerify(t *testing.T) { } tx, err = vm.newAddDefaultSubnetDelegatorTx( - defaultNonce+1, // nonce - defaultStakeAmount, // weight - uint64(newTimestamp.Unix()), // start time + defaultNonce+1, // nonce + defaultStakeAmount, // weight + uint64(newTimestamp.Unix()), // start time uint64(newTimestamp.Add(MinimumStakingDuration).Unix()), // end time defaultKey.PublicKey().Address(), // node ID defaultKey.PublicKey().Address(), // destination @@ -387,51 +387,51 @@ func TestAddDefaultSubnetDelegatorTxSemanticVerify(t *testing.T) { } txFee = txFeeSaved // Reset tx fee - // Case 8: fail verification for spending more funds than it has - tx, err = vm.newAddDefaultSubnetDelegatorTx( - defaultNonce+1, - defaultBalance*2, // weight - uint64(defaultValidateStartTime.Unix()), // start time - uint64(defaultValidateEndTime.Unix()), // end time - defaultKey.PublicKey().Address(), // node ID - defaultKey.PublicKey().Address(), // destination - testNetworkID, // network ID - defaultKey, // tx fee payer - ) - if err != nil { - t.Fatal(err) - } - _, _, _, _, err = tx.SemanticVerify(vm.DB) - if err == nil { - t.Fatal("should have failed verification because payer account spent twice the account's balance") - } + // // Case 8: fail verification for spending more funds than it has + // tx, err = vm.newAddDefaultSubnetDelegatorTx( + // defaultNonce+1, + // defaultBalance*2, // weight + // uint64(defaultValidateStartTime.Unix()), // start time + // uint64(defaultValidateEndTime.Unix()), // end time + // defaultKey.PublicKey().Address(), // node ID + // defaultKey.PublicKey().Address(), // destination + // testNetworkID, // network ID + // defaultKey, // tx fee payer + // ) + // if err != nil { + // t.Fatal(err) + // } + // _, _, _, _, err = tx.SemanticVerify(vm.DB) + // if err == nil { + // t.Fatal("should have failed verification because payer account spent twice the account's balance") + // } - // Case 9: Confirm balance is correct - tx, err = vm.newAddDefaultSubnetDelegatorTx( - defaultNonce+1, - defaultStakeAmount, // weight - uint64(defaultValidateStartTime.Unix()), // start time - uint64(defaultValidateEndTime.Unix()), // end time - defaultKey.PublicKey().Address(), // node ID - defaultKey.PublicKey().Address(), // destination - testNetworkID, // network ID - defaultKey, // tx fee payer - ) - if err != nil { - t.Fatal(err) - } + // // Case 9: Confirm balance is correct + // tx, err = vm.newAddDefaultSubnetDelegatorTx( + // defaultNonce+1, + // defaultStakeAmount, // weight + // uint64(defaultValidateStartTime.Unix()), // start time + // uint64(defaultValidateEndTime.Unix()), // end time + // defaultKey.PublicKey().Address(), // node ID + // defaultKey.PublicKey().Address(), // destination + // testNetworkID, // network ID + // defaultKey, // tx fee payer + // ) + // if err != nil { + // t.Fatal(err) + // } - onCommitDB, _, _, _, err := tx.SemanticVerify(vm.DB) - if err != nil { - t.Fatal(err) - } - account, err := tx.vm.getAccount(onCommitDB, defaultKey.PublicKey().Address()) - if err != nil { - t.Fatal(err) - } - balance := account.Balance + // onCommitDB, _, _, _, err := tx.SemanticVerify(vm.DB) + // if err != nil { + // t.Fatal(err) + // } + // account, err := tx.vm.getAccount(onCommitDB, defaultKey.PublicKey().Address()) + // if err != nil { + // t.Fatal(err) + // } + // balance := account.Balance - if balance != defaultBalance-(defaultStakeAmount+txFee) { - t.Fatalf("balance was not updated correctly after subnet delegator tx") - } + // if balance != defaultBalance-(defaultStakeAmount+txFee) { + // t.Fatalf("balance was not updated correctly after subnet delegator tx") + // } } From f78d7b3caf6a438c286d64f038449a9c02638da3 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Thu, 18 Jun 2020 13:34:04 -0400 Subject: [PATCH 15/80] use transitive dependencies when possible with transaction dependencies --- vms/avm/base_tx.go | 11 +++- vms/avm/base_tx_test.go | 112 ++++++++++++++++---------------- vms/avm/create_asset_tx_test.go | 8 +-- vms/avm/export_tx_test.go | 16 ++--- vms/avm/import_tx.go | 11 +++- vms/avm/import_tx_test.go | 10 +-- vms/avm/operation_test.go | 18 ++--- vms/avm/operation_tx.go | 13 +++- vms/avm/prefixed_state_test.go | 6 +- vms/avm/service.go | 8 +-- vms/avm/service_test.go | 8 +-- vms/avm/state_test.go | 2 +- vms/avm/static_service_test.go | 12 ++-- vms/avm/tx.go | 4 +- vms/avm/tx_test.go | 12 ++-- vms/avm/unique_tx.go | 27 ++++---- vms/avm/vm.go | 8 +-- vms/avm/vm_test.go | 60 +++++++++-------- 18 files changed, 186 insertions(+), 160 deletions(-) diff --git a/vms/avm/base_tx.go b/vms/avm/base_tx.go index 0ab3fa4..cf4371f 100644 --- a/vms/avm/base_tx.go +++ b/vms/avm/base_tx.go @@ -9,8 +9,8 @@ import ( "github.com/ava-labs/gecko/database" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" - "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/utils/codec" + "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/vms/components/verify" ) @@ -46,8 +46,8 @@ func (t *BaseTx) InputUTXOs() []*ava.UTXOID { return utxos } -// AssetIDs returns the IDs of the assets this transaction depends on -func (t *BaseTx) AssetIDs() ids.Set { +// ConsumedAssetIDs returns the IDs of the assets this transaction consumes +func (t *BaseTx) ConsumedAssetIDs() ids.Set { assets := ids.Set{} for _, in := range t.Ins { assets.Add(in.AssetID()) @@ -55,6 +55,11 @@ func (t *BaseTx) AssetIDs() ids.Set { return assets } +// AssetIDs returns the IDs of the assets this transaction depends on +func (t *BaseTx) AssetIDs() ids.Set { + return t.ConsumedAssetIDs() +} + // NumCredentials returns the number of expected credentials func (t *BaseTx) NumCredentials() int { return len(t.Ins) } diff --git a/vms/avm/base_tx_test.go b/vms/avm/base_tx_test.go index 163ef5c..9f7bbdd 100644 --- a/vms/avm/base_tx_test.go +++ b/vms/avm/base_tx_test.go @@ -77,7 +77,7 @@ func TestBaseTxSerialization(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -87,7 +87,7 @@ func TestBaseTxSerialization(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -125,7 +125,7 @@ func TestBaseTxGetters(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -135,7 +135,7 @@ func TestBaseTxGetters(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -162,6 +162,10 @@ func TestBaseTxGetters(t *testing.T) { t.Fatalf("Wrong number of assets returned") } else if !assets.Contains(asset) { t.Fatalf("Wrong asset returned") + } else if assets := tx.ConsumedAssetIDs(); assets.Len() != 1 { + t.Fatalf("Wrong number of consumed assets returned") + } else if !assets.Contains(asset) { + t.Fatalf("Wrong consumed asset returned") } else if utxos := tx.UTXOs(); len(utxos) != 1 { t.Fatalf("Wrong number of utxos returned") } else if utxo := utxos[0]; !utxo.TxID.Equals(txID) { @@ -179,7 +183,7 @@ func TestBaseTxSyntacticVerify(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -189,7 +193,7 @@ func TestBaseTxSyntacticVerify(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -230,7 +234,7 @@ func TestBaseTxSyntacticVerifyWrongNetworkID(t *testing.T) { tx := &BaseTx{ NetID: 0, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -240,7 +244,7 @@ func TestBaseTxSyntacticVerifyWrongNetworkID(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -272,7 +276,7 @@ func TestBaseTxSyntacticVerifyWrongChainID(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: ids.Empty, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -282,7 +286,7 @@ func TestBaseTxSyntacticVerifyWrongChainID(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -315,7 +319,7 @@ func TestBaseTxSyntacticVerifyInvalidOutput(t *testing.T) { NetID: networkID, BCID: chainID, Outs: []*ava.TransferableOutput{nil}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -348,7 +352,7 @@ func TestBaseTxSyntacticVerifyUnsortedOutputs(t *testing.T) { NetID: networkID, BCID: chainID, Outs: []*ava.TransferableOutput{ - &ava.TransferableOutput{ + { Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 2, @@ -358,7 +362,7 @@ func TestBaseTxSyntacticVerifyUnsortedOutputs(t *testing.T) { }, }, }, - &ava.TransferableOutput{ + { Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 1, @@ -370,7 +374,7 @@ func TestBaseTxSyntacticVerifyUnsortedOutputs(t *testing.T) { }, }, Ins: []*ava.TransferableInput{ - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -403,7 +407,7 @@ func TestBaseTxSyntacticVerifyInvalidInput(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -428,7 +432,7 @@ func TestBaseTxSyntacticVerifyInputOverflow(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -439,7 +443,7 @@ func TestBaseTxSyntacticVerifyInputOverflow(t *testing.T) { }, }}, Ins: []*ava.TransferableInput{ - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -457,7 +461,7 @@ func TestBaseTxSyntacticVerifyInputOverflow(t *testing.T) { }, }, }, - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -491,7 +495,7 @@ func TestBaseTxSyntacticVerifyOutputOverflow(t *testing.T) { NetID: networkID, BCID: chainID, Outs: []*ava.TransferableOutput{ - &ava.TransferableOutput{ + { Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 2, @@ -501,7 +505,7 @@ func TestBaseTxSyntacticVerifyOutputOverflow(t *testing.T) { }, }, }, - &ava.TransferableOutput{ + { Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: math.MaxUint64, @@ -512,7 +516,7 @@ func TestBaseTxSyntacticVerifyOutputOverflow(t *testing.T) { }, }, }, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -544,7 +548,7 @@ func TestBaseTxSyntacticVerifyInsufficientFunds(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: math.MaxUint64, @@ -554,7 +558,7 @@ func TestBaseTxSyntacticVerifyInsufficientFunds(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -586,7 +590,7 @@ func TestBaseTxSyntacticVerifyUninitialized(t *testing.T) { tx := &BaseTx{ NetID: networkID, BCID: chainID, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: asset}, Out: &secp256k1fx.TransferOutput{ Amt: 12345, @@ -596,7 +600,7 @@ func TestBaseTxSyntacticVerifyUninitialized(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, @@ -633,7 +637,7 @@ func TestBaseTxSemanticVerify(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -702,7 +706,7 @@ func TestBaseTxSemanticVerifyUnknownFx(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -754,7 +758,7 @@ func TestBaseTxSemanticVerifyWrongAssetID(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -825,11 +829,11 @@ func TestBaseTxSemanticVerifyUnauthorizedFx(t *testing.T) { genesisBytes, issuer, []*common.Fx{ - &common.Fx{ + { ID: ids.Empty, Fx: &secp256k1fx.Fx{}, }, - &common.Fx{ + { ID: ids.NewID([32]byte{1}), Fx: &testFx{}, }, @@ -863,7 +867,7 @@ func TestBaseTxSemanticVerifyUnauthorizedFx(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -923,7 +927,7 @@ func TestBaseTxSemanticVerifyInvalidSignature(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -941,9 +945,7 @@ func TestBaseTxSemanticVerifyInvalidSignature(t *testing.T) { }} tx.Creds = append(tx.Creds, &secp256k1fx.Credential{ - Sigs: [][crypto.SECP256K1RSigLen]byte{ - [crypto.SECP256K1RSigLen]byte{}, - }, + Sigs: [][crypto.SECP256K1RSigLen]byte{{}}, }) b, err := vm.codec.Marshal(tx) @@ -977,7 +979,7 @@ func TestBaseTxSemanticVerifyMissingUTXO(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.Empty, OutputIndex: 1, @@ -1044,7 +1046,7 @@ func TestBaseTxSemanticVerifyInvalidUTXO(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: math.MaxUint32, @@ -1107,7 +1109,7 @@ func TestBaseTxSemanticVerifyPendingInvalidUTXO(t *testing.T) { pendingTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -1122,7 +1124,7 @@ func TestBaseTxSemanticVerifyPendingInvalidUTXO(t *testing.T) { }, }, }}, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: genesisTx.ID()}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, @@ -1179,7 +1181,7 @@ func TestBaseTxSemanticVerifyPendingInvalidUTXO(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: txID, OutputIndex: 2, @@ -1241,7 +1243,7 @@ func TestBaseTxSemanticVerifyPendingWrongAssetID(t *testing.T) { pendingTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -1256,7 +1258,7 @@ func TestBaseTxSemanticVerifyPendingWrongAssetID(t *testing.T) { }, }, }}, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: genesisTx.ID()}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, @@ -1313,7 +1315,7 @@ func TestBaseTxSemanticVerifyPendingWrongAssetID(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: txID, OutputIndex: 0, @@ -1381,11 +1383,11 @@ func TestBaseTxSemanticVerifyPendingUnauthorizedFx(t *testing.T) { genesisBytes, issuer, []*common.Fx{ - &common.Fx{ + { ID: ids.NewID([32]byte{1}), Fx: &secp256k1fx.Fx{}, }, - &common.Fx{ + { ID: ids.Empty, Fx: &testFx{}, }, @@ -1419,7 +1421,7 @@ func TestBaseTxSemanticVerifyPendingUnauthorizedFx(t *testing.T) { pendingTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -1434,7 +1436,7 @@ func TestBaseTxSemanticVerifyPendingUnauthorizedFx(t *testing.T) { }, }, }}, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: genesisTx.ID()}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, @@ -1491,7 +1493,7 @@ func TestBaseTxSemanticVerifyPendingUnauthorizedFx(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: txID, OutputIndex: 0, @@ -1543,11 +1545,11 @@ func TestBaseTxSemanticVerifyPendingInvalidSignature(t *testing.T) { genesisBytes, issuer, []*common.Fx{ - &common.Fx{ + { ID: ids.NewID([32]byte{1}), Fx: &secp256k1fx.Fx{}, }, - &common.Fx{ + { ID: ids.Empty, Fx: &testFx{}, }, @@ -1581,7 +1583,7 @@ func TestBaseTxSemanticVerifyPendingInvalidSignature(t *testing.T) { pendingTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -1596,7 +1598,7 @@ func TestBaseTxSemanticVerifyPendingInvalidSignature(t *testing.T) { }, }, }}, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: genesisTx.ID()}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, @@ -1653,7 +1655,7 @@ func TestBaseTxSemanticVerifyPendingInvalidSignature(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: txID, OutputIndex: 0, @@ -1671,9 +1673,7 @@ func TestBaseTxSemanticVerifyPendingInvalidSignature(t *testing.T) { }} tx.Creds = append(tx.Creds, &secp256k1fx.Credential{ - Sigs: [][crypto.SECP256K1RSigLen]byte{ - [crypto.SECP256K1RSigLen]byte{}, - }, + Sigs: [][crypto.SECP256K1RSigLen]byte{{}}, }) b, err = vm.codec.Marshal(tx) diff --git a/vms/avm/create_asset_tx_test.go b/vms/avm/create_asset_tx_test.go index 324f403..6f1f686 100644 --- a/vms/avm/create_asset_tx_test.go +++ b/vms/avm/create_asset_tx_test.go @@ -8,8 +8,8 @@ import ( "testing" "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/utils/codec" + "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/vms/components/verify" "github.com/ava-labs/gecko/vms/secp256k1fx" ) @@ -93,7 +93,7 @@ func TestCreateAssetTxSerialization(t *testing.T) { 0xbb, 0xbb, 0xbb, 0xbb, 0xaa, 0xaa, 0xaa, 0xaa, 0x99, 0x99, 0x99, 0x99, 0x88, 0x88, 0x88, 0x88, }), - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ ID: ids.NewID([32]byte{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, @@ -122,7 +122,7 @@ func TestCreateAssetTxSerialization(t *testing.T) { }, }, }}, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.NewID([32]byte{ 0xf1, 0xe1, 0xd1, 0xc1, 0xb1, 0xa1, 0x91, 0x81, @@ -152,7 +152,7 @@ func TestCreateAssetTxSerialization(t *testing.T) { Symbol: "VIX", Denomination: 2, States: []*InitialState{ - &InitialState{ + { FxID: 0, Outs: []verify.Verifiable{ &secp256k1fx.TransferOutput{ diff --git a/vms/avm/export_tx_test.go b/vms/avm/export_tx_test.go index fdef399..75b359f 100644 --- a/vms/avm/export_tx_test.go +++ b/vms/avm/export_tx_test.go @@ -12,11 +12,11 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/snow/engine/common" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/utils/crypto" "github.com/ava-labs/gecko/utils/hashing" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) @@ -68,7 +68,7 @@ func TestExportTxSerialization(t *testing.T) { 0xbb, 0xbb, 0xbb, 0xbb, 0xaa, 0xaa, 0xaa, 0xaa, 0x99, 0x99, 0x99, 0x99, 0x88, 0x88, 0x88, 0x88, }), - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{TxID: ids.NewID([32]byte{ 0x0f, 0x2f, 0x4f, 0x6f, 0x8e, 0xae, 0xce, 0xee, 0x0d, 0x2d, 0x4d, 0x6d, 0x8c, 0xac, 0xcc, 0xec, @@ -141,7 +141,7 @@ func TestIssueExportTx(t *testing.T) { memdb.New(), genesisBytes, issuer, - []*common.Fx{&common.Fx{ + []*common.Fx{{ ID: ids.Empty, Fx: &secp256k1fx.Fx{}, }}, @@ -167,7 +167,7 @@ func TestIssueExportTx(t *testing.T) { BaseTx: BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: avaID, OutputIndex: 1, @@ -179,7 +179,7 @@ func TestIssueExportTx(t *testing.T) { }, }}, }, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: avaID}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, @@ -297,7 +297,7 @@ func TestClearForceAcceptedExportTx(t *testing.T) { memdb.New(), genesisBytes, issuer, - []*common.Fx{&common.Fx{ + []*common.Fx{{ ID: ids.Empty, Fx: &secp256k1fx.Fx{}, }}, @@ -323,7 +323,7 @@ func TestClearForceAcceptedExportTx(t *testing.T) { BaseTx: BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: avaID, OutputIndex: 1, @@ -335,7 +335,7 @@ func TestClearForceAcceptedExportTx(t *testing.T) { }, }}, }, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: avaID}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, diff --git a/vms/avm/import_tx.go b/vms/avm/import_tx.go index 1729221..caf0f93 100644 --- a/vms/avm/import_tx.go +++ b/vms/avm/import_tx.go @@ -11,8 +11,8 @@ import ( "github.com/ava-labs/gecko/database/versiondb" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" - "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/utils/codec" + "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/vms/components/verify" ) @@ -33,6 +33,15 @@ func (t *ImportTx) InputUTXOs() []*ava.UTXOID { return utxos } +// ConsumedAssetIDs returns the IDs of the assets this transaction consumes +func (t *ImportTx) ConsumedAssetIDs() ids.Set { + assets := t.BaseTx.AssetIDs() + for _, in := range t.Ins { + assets.Add(in.AssetID()) + } + return assets +} + // AssetIDs returns the IDs of the assets this transaction depends on func (t *ImportTx) AssetIDs() ids.Set { assets := t.BaseTx.AssetIDs() diff --git a/vms/avm/import_tx_test.go b/vms/avm/import_tx_test.go index 696e841..e510aff 100644 --- a/vms/avm/import_tx_test.go +++ b/vms/avm/import_tx_test.go @@ -68,7 +68,7 @@ func TestImportTxSerialization(t *testing.T) { 0x99, 0x99, 0x99, 0x99, 0x88, 0x88, 0x88, 0x88, }), }, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{TxID: ids.NewID([32]byte{ 0x0f, 0x2f, 0x4f, 0x6f, 0x8e, 0xae, 0xce, 0xee, 0x0d, 0x2d, 0x4d, 0x6d, 0x8c, 0xac, 0xcc, 0xec, @@ -130,7 +130,7 @@ func TestIssueImportTx(t *testing.T) { memdb.New(), genesisBytes, issuer, - []*common.Fx{&common.Fx{ + []*common.Fx{{ ID: ids.Empty, Fx: &secp256k1fx.Fx{}, }}, @@ -166,7 +166,7 @@ func TestIssueImportTx(t *testing.T) { NetID: networkID, BCID: chainID, }, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: utxoID, Asset: ava.Asset{ID: avaID}, In: &secp256k1fx.TransferInput{ @@ -288,7 +288,7 @@ func TestForceAcceptImportTx(t *testing.T) { memdb.New(), genesisBytes, issuer, - []*common.Fx{&common.Fx{ + []*common.Fx{{ ID: ids.Empty, Fx: &secp256k1fx.Fx{}, }}, @@ -326,7 +326,7 @@ func TestForceAcceptImportTx(t *testing.T) { NetID: networkID, BCID: chainID, }, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: utxoID, Asset: ava.Asset{ID: genesisTx.ID()}, In: &secp256k1fx.TransferInput{ diff --git a/vms/avm/operation_test.go b/vms/avm/operation_test.go index 8b85901..b3aed54 100644 --- a/vms/avm/operation_test.go +++ b/vms/avm/operation_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/utils/codec" + "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/vms/components/verify" ) @@ -43,11 +43,11 @@ func TestOperationVerifyUTXOIDsNotSorted(t *testing.T) { op := &Operation{ Asset: ava.Asset{ID: ids.Empty}, UTXOIDs: []*ava.UTXOID{ - &ava.UTXOID{ + { TxID: ids.Empty, OutputIndex: 1, }, - &ava.UTXOID{ + { TxID: ids.Empty, OutputIndex: 0, }, @@ -64,7 +64,7 @@ func TestOperationVerify(t *testing.T) { op := &Operation{ Asset: ava.Asset{ID: ids.Empty}, UTXOIDs: []*ava.UTXOID{ - &ava.UTXOID{ + { TxID: ids.Empty, OutputIndex: 1, }, @@ -81,20 +81,20 @@ func TestOperationSorting(t *testing.T) { c.RegisterType(&testOperable{}) ops := []*Operation{ - &Operation{ + { Asset: ava.Asset{ID: ids.Empty}, UTXOIDs: []*ava.UTXOID{ - &ava.UTXOID{ + { TxID: ids.Empty, OutputIndex: 1, }, }, Op: &testOperable{}, }, - &Operation{ + { Asset: ava.Asset{ID: ids.Empty}, UTXOIDs: []*ava.UTXOID{ - &ava.UTXOID{ + { TxID: ids.Empty, OutputIndex: 0, }, @@ -112,7 +112,7 @@ func TestOperationSorting(t *testing.T) { ops = append(ops, &Operation{ Asset: ava.Asset{ID: ids.Empty}, UTXOIDs: []*ava.UTXOID{ - &ava.UTXOID{ + { TxID: ids.Empty, OutputIndex: 1, }, diff --git a/vms/avm/operation_tx.go b/vms/avm/operation_tx.go index ec419c7..c45f0be 100644 --- a/vms/avm/operation_tx.go +++ b/vms/avm/operation_tx.go @@ -8,8 +8,8 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" - "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/utils/codec" + "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/vms/components/verify" ) @@ -39,6 +39,17 @@ func (t *OperationTx) InputUTXOs() []*ava.UTXOID { return utxos } +// ConsumedAssetIDs returns the IDs of the assets this transaction consumes +func (t *OperationTx) ConsumedAssetIDs() ids.Set { + assets := t.BaseTx.AssetIDs() + for _, op := range t.Ops { + if len(op.UTXOIDs) > 0 { + assets.Add(op.AssetID()) + } + } + return assets +} + // AssetIDs returns the IDs of the assets this transaction depends on func (t *OperationTx) AssetIDs() ids.Set { assets := t.BaseTx.AssetIDs() diff --git a/vms/avm/prefixed_state_test.go b/vms/avm/prefixed_state_test.go index 74c4b53..e325142 100644 --- a/vms/avm/prefixed_state_test.go +++ b/vms/avm/prefixed_state_test.go @@ -38,7 +38,7 @@ func TestPrefixedSetsAndGets(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.Empty, OutputIndex: 0, @@ -160,9 +160,7 @@ func TestPrefixedFundingAddresses(t *testing.T) { }, Asset: ava.Asset{ID: ids.Empty}, Out: &ava.TestAddressable{ - Addrs: [][]byte{ - []byte{0}, - }, + Addrs: [][]byte{{0}}, }, } diff --git a/vms/avm/service.go b/vms/avm/service.go index 9b21414..3d33118 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -792,7 +792,7 @@ func (service *Service) Send(r *http.Request, args *SendArgs, reply *SendReply) ava.SortTransferableInputsWithSigners(ins, keys) - outs := []*ava.TransferableOutput{&ava.TransferableOutput{ + outs := []*ava.TransferableOutput{{ Asset: ava.Asset{ID: assetID}, Out: &secp256k1fx.TransferOutput{ Amt: uint64(args.Amount), @@ -946,7 +946,7 @@ func (service *Service) CreateMintTx(r *http.Request, args *CreateMintTxArgs, re BCID: service.vm.ctx.ChainID, }, Ops: []*Operation{ - &Operation{ + { Asset: ava.Asset{ID: assetID}, UTXOIDs: []*ava.UTXOID{ &utxo.UTXOID, @@ -1197,7 +1197,7 @@ func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, reply *I ava.SortTransferableInputsWithSigners(ins, keys) - outs := []*ava.TransferableOutput{&ava.TransferableOutput{ + outs := []*ava.TransferableOutput{{ Asset: ava.Asset{ID: service.vm.ava}, Out: &secp256k1fx.TransferOutput{ Amt: amount, @@ -1352,7 +1352,7 @@ func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, reply *E ava.SortTransferableInputsWithSigners(ins, keys) - exportOuts := []*ava.TransferableOutput{&ava.TransferableOutput{ + exportOuts := []*ava.TransferableOutput{{ Asset: ava.Asset{ID: service.vm.ava}, Out: &secp256k1fx.TransferOutput{ Amt: uint64(args.Amount), diff --git a/vms/avm/service_test.go b/vms/avm/service_test.go index 6e1d387..b8a7d56 100644 --- a/vms/avm/service_test.go +++ b/vms/avm/service_test.go @@ -300,7 +300,7 @@ func TestCreateFixedCapAsset(t *testing.T) { Name: "test asset", Symbol: "test", Denomination: 1, - InitialHolders: []*Holder{&Holder{ + InitialHolders: []*Holder{{ Amount: 123456789, Address: vm.Format(keys[0].PublicKey().Address().Bytes()), }}, @@ -326,7 +326,7 @@ func TestCreateVariableCapAsset(t *testing.T) { Name: "test asset", Symbol: "test", MinterSets: []Owners{ - Owners{ + { Threshold: 1, Minters: []string{ vm.Format(keys[0].PublicKey().Address().Bytes()), @@ -367,7 +367,7 @@ func TestImportAvmKey(t *testing.T) { factory := crypto.FactorySECP256K1R{} skIntf, err := factory.NewPrivateKey() if err != nil { - t.Fatalf("problem generating private key: %w", err) + t.Fatalf("problem generating private key: %s", err) } sk := skIntf.(*crypto.PrivateKeySECP256K1R) @@ -406,7 +406,7 @@ func TestImportAvmKeyNoDuplicates(t *testing.T) { factory := crypto.FactorySECP256K1R{} skIntf, err := factory.NewPrivateKey() if err != nil { - t.Fatalf("problem generating private key: %w", err) + t.Fatalf("problem generating private key: %s", err) } sk := skIntf.(*crypto.PrivateKeySECP256K1R) diff --git a/vms/avm/state_test.go b/vms/avm/state_test.go index 1feabab..3617ddc 100644 --- a/vms/avm/state_test.go +++ b/vms/avm/state_test.go @@ -288,7 +288,7 @@ func TestStateTXs(t *testing.T) { tx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.Empty, OutputIndex: 0, diff --git a/vms/avm/static_service_test.go b/vms/avm/static_service_test.go index fd9acc0..34e8803 100644 --- a/vms/avm/static_service_test.go +++ b/vms/avm/static_service_test.go @@ -11,12 +11,12 @@ func TestBuildGenesis(t *testing.T) { ss := StaticService{} args := BuildGenesisArgs{GenesisData: map[string]AssetDefinition{ - "asset1": AssetDefinition{ + "asset1": { Name: "myFixedCapAsset", Symbol: "MFCA", Denomination: 8, InitialState: map[string][]interface{}{ - "fixedCap": []interface{}{ + "fixedCap": { Holder{ Amount: 100000, Address: "A9bTQjfYGBFK3JPRJqF2eh3JYL7cHocvy", @@ -36,11 +36,11 @@ func TestBuildGenesis(t *testing.T) { }, }, }, - "asset2": AssetDefinition{ + "asset2": { Name: "myVarCapAsset", Symbol: "MVCA", InitialState: map[string][]interface{}{ - "variableCap": []interface{}{ + "variableCap": { Owners{ Threshold: 1, Minters: []string{ @@ -58,10 +58,10 @@ func TestBuildGenesis(t *testing.T) { }, }, }, - "asset3": AssetDefinition{ + "asset3": { Name: "myOtherVarCapAsset", InitialState: map[string][]interface{}{ - "variableCap": []interface{}{ + "variableCap": { Owners{ Threshold: 1, Minters: []string{ diff --git a/vms/avm/tx.go b/vms/avm/tx.go index f1d0b71..6c32153 100644 --- a/vms/avm/tx.go +++ b/vms/avm/tx.go @@ -9,8 +9,8 @@ import ( "github.com/ava-labs/gecko/database" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" - "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/utils/codec" + "github.com/ava-labs/gecko/vms/components/ava" "github.com/ava-labs/gecko/vms/components/verify" ) @@ -24,7 +24,9 @@ type UnsignedTx interface { ID() ids.ID Bytes() []byte + ConsumedAssetIDs() ids.Set AssetIDs() ids.Set + NumCredentials() int InputUTXOs() []*ava.UTXOID UTXOs() []*ava.UTXO diff --git a/vms/avm/tx_test.go b/vms/avm/tx_test.go index 53e20de..f088d9e 100644 --- a/vms/avm/tx_test.go +++ b/vms/avm/tx_test.go @@ -7,9 +7,9 @@ import ( "testing" "github.com/ava-labs/gecko/ids" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/utils/units" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" "github.com/ava-labs/gecko/vms/secp256k1fx" ) @@ -56,7 +56,7 @@ func TestTxInvalidCredential(t *testing.T) { UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: ids.Empty, OutputIndex: 0, @@ -95,7 +95,7 @@ func TestTxInvalidUnsignedTx(t *testing.T) { NetID: networkID, BCID: chainID, Ins: []*ava.TransferableInput{ - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{ TxID: ids.Empty, OutputIndex: 0, @@ -110,7 +110,7 @@ func TestTxInvalidUnsignedTx(t *testing.T) { }, }, }, - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{ TxID: ids.Empty, OutputIndex: 0, @@ -153,7 +153,7 @@ func TestTxInvalidNumberOfCredentials(t *testing.T) { NetID: networkID, BCID: chainID, Ins: []*ava.TransferableInput{ - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{TxID: ids.Empty, OutputIndex: 0}, Asset: ava.Asset{ID: asset}, In: &secp256k1fx.TransferInput{ @@ -165,7 +165,7 @@ func TestTxInvalidNumberOfCredentials(t *testing.T) { }, }, }, - &ava.TransferableInput{ + { UTXOID: ava.UTXOID{TxID: ids.Empty, OutputIndex: 1}, Asset: ava.Asset{ID: asset}, In: &secp256k1fx.TransferInput{ diff --git a/vms/avm/unique_tx.go b/vms/avm/unique_tx.go index 8d3feb2..892b513 100644 --- a/vms/avm/unique_tx.go +++ b/vms/avm/unique_tx.go @@ -206,22 +206,25 @@ func (tx *UniqueTx) Dependencies() []snowstorm.Tx { continue } txID, _ := in.InputSource() - if !txIDs.Contains(txID) { - txIDs.Add(txID) - tx.deps = append(tx.deps, &UniqueTx{ - vm: tx.vm, - txID: txID, - }) + if txIDs.Contains(txID) { + continue } + txIDs.Add(txID) + tx.deps = append(tx.deps, &UniqueTx{ + vm: tx.vm, + txID: txID, + }) } + consumedIDs := tx.Tx.ConsumedAssetIDs() for _, assetID := range tx.Tx.AssetIDs().List() { - if !txIDs.Contains(assetID) { - txIDs.Add(assetID) - tx.deps = append(tx.deps, &UniqueTx{ - vm: tx.vm, - txID: assetID, - }) + if consumedIDs.Contains(assetID) || txIDs.Contains(assetID) { + continue } + txIDs.Add(assetID) + tx.deps = append(tx.deps, &UniqueTx{ + vm: tx.vm, + txID: assetID, + }) } return tx.deps } diff --git a/vms/avm/vm.go b/vms/avm/vm.go index 4c0820d..d89d625 100644 --- a/vms/avm/vm.go +++ b/vms/avm/vm.go @@ -20,12 +20,12 @@ import ( "github.com/ava-labs/gecko/snow/choices" "github.com/ava-labs/gecko/snow/consensus/snowstorm" "github.com/ava-labs/gecko/snow/engine/common" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/utils/timer" "github.com/ava-labs/gecko/utils/wrappers" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/utils/codec" cjson "github.com/ava-labs/gecko/utils/json" ) @@ -248,8 +248,8 @@ func (vm *VM) CreateHandlers() map[string]*common.HTTPHandler { rpcServer.RegisterService(&Service{vm: vm}, "avm") // name this service "avm" return map[string]*common.HTTPHandler{ - "": &common.HTTPHandler{Handler: rpcServer}, - "/pubsub": &common.HTTPHandler{LockOptions: common.NoLock, Handler: vm.pubsub}, + "": {Handler: rpcServer}, + "/pubsub": {LockOptions: common.NoLock, Handler: vm.pubsub}, } } @@ -261,7 +261,7 @@ func (vm *VM) CreateStaticHandlers() map[string]*common.HTTPHandler { newServer.RegisterCodec(codec, "application/json;charset=UTF-8") newServer.RegisterService(&StaticService{}, "avm") // name this service "avm" return map[string]*common.HTTPHandler{ - "": &common.HTTPHandler{LockOptions: common.WriteLock, Handler: newServer}, + "": {LockOptions: common.WriteLock, Handler: newServer}, } } diff --git a/vms/avm/vm_test.go b/vms/avm/vm_test.go index d3a2d73..81a08f8 100644 --- a/vms/avm/vm_test.go +++ b/vms/avm/vm_test.go @@ -84,11 +84,11 @@ func BuildGenesisTest(t *testing.T) []byte { addr2 := keys[2].PublicKey().Address() args := BuildGenesisArgs{GenesisData: map[string]AssetDefinition{ - "asset1": AssetDefinition{ + "asset1": { Name: "myFixedCapAsset", Symbol: "MFCA", InitialState: map[string][]interface{}{ - "fixedCap": []interface{}{ + "fixedCap": { Holder{ Amount: 100000, Address: addr0.String(), @@ -108,11 +108,11 @@ func BuildGenesisTest(t *testing.T) []byte { }, }, }, - "asset2": AssetDefinition{ + "asset2": { Name: "myVarCapAsset", Symbol: "MVCA", InitialState: map[string][]interface{}{ - "variableCap": []interface{}{ + "variableCap": { Owners{ Threshold: 1, Minters: []string{ @@ -131,10 +131,10 @@ func BuildGenesisTest(t *testing.T) []byte { }, }, }, - "asset3": AssetDefinition{ + "asset3": { Name: "myOtherVarCapAsset", InitialState: map[string][]interface{}{ - "variableCap": []interface{}{ + "variableCap": { Owners{ Threshold: 1, Minters: []string{ @@ -168,7 +168,7 @@ func GenesisVM(t *testing.T) ([]byte, chan common.Message, *VM) { memdb.New(), genesisBytes, issuer, - []*common.Fx{&common.Fx{ + []*common.Fx{{ ID: ids.Empty, Fx: &secp256k1fx.Fx{}, }}, @@ -195,7 +195,7 @@ func NewTx(t *testing.T, genesisBytes []byte, vm *VM) *Tx { newTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -357,7 +357,7 @@ func TestTxSerialization(t *testing.T) { Symbol: "symb", Denomination: 0, States: []*InitialState{ - &InitialState{ + { FxID: 0, Outs: []verify.Verifiable{ &secp256k1fx.MintOutput{ @@ -456,7 +456,7 @@ func TestFxInitializationFailure(t *testing.T) { /*db=*/ memdb.New(), /*genesisState=*/ genesisBytes, /*engineMessenger=*/ make(chan common.Message, 1), - /*fxs=*/ []*common.Fx{&common.Fx{ + /*fxs=*/ []*common.Fx{{ ID: ids.Empty, Fx: &testFx{initialize: errUnknownFx}, }}, @@ -537,7 +537,7 @@ func TestIssueDependentTx(t *testing.T) { firstTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: genesisTx.ID(), OutputIndex: 1, @@ -552,7 +552,7 @@ func TestIssueDependentTx(t *testing.T) { }, }, }}, - Outs: []*ava.TransferableOutput{&ava.TransferableOutput{ + Outs: []*ava.TransferableOutput{{ Asset: ava.Asset{ID: genesisTx.ID()}, Out: &secp256k1fx.TransferOutput{ Amt: 50000, @@ -596,7 +596,7 @@ func TestIssueDependentTx(t *testing.T) { secondTx := &Tx{UnsignedTx: &BaseTx{ NetID: networkID, BCID: chainID, - Ins: []*ava.TransferableInput{&ava.TransferableInput{ + Ins: []*ava.TransferableInput{{ UTXOID: ava.UTXOID{ TxID: firstTx.ID(), OutputIndex: 0, @@ -671,11 +671,11 @@ func TestIssueNFT(t *testing.T) { genesisBytes, issuer, []*common.Fx{ - &common.Fx{ + { ID: ids.Empty.Prefix(0), Fx: &secp256k1fx.Fx{}, }, - &common.Fx{ + { ID: ids.Empty.Prefix(1), Fx: &nftfx.Fx{}, }, @@ -704,7 +704,7 @@ func TestIssueNFT(t *testing.T) { Name: "Team Rocket", Symbol: "TR", Denomination: 0, - States: []*InitialState{&InitialState{ + States: []*InitialState{{ FxID: 1, Outs: []verify.Verifiable{ &nftfx.MintOutput{ @@ -740,9 +740,9 @@ func TestIssueNFT(t *testing.T) { NetID: networkID, BCID: chainID, }, - Ops: []*Operation{&Operation{ + Ops: []*Operation{{ Asset: ava.Asset{ID: createAssetTx.ID()}, - UTXOIDs: []*ava.UTXOID{&ava.UTXOID{ + UTXOIDs: []*ava.UTXOID{{ TxID: createAssetTx.ID(), OutputIndex: 0, }}, @@ -752,9 +752,7 @@ func TestIssueNFT(t *testing.T) { }, GroupID: 1, Payload: []byte{'h', 'e', 'l', 'l', 'o'}, - Outputs: []*secp256k1fx.OutputOwners{ - &secp256k1fx.OutputOwners{}, - }, + Outputs: []*secp256k1fx.OutputOwners{{}}, }, }}, }} @@ -793,9 +791,9 @@ func TestIssueNFT(t *testing.T) { NetID: networkID, BCID: chainID, }, - Ops: []*Operation{&Operation{ + Ops: []*Operation{{ Asset: ava.Asset{ID: createAssetTx.ID()}, - UTXOIDs: []*ava.UTXOID{&ava.UTXOID{ + UTXOIDs: []*ava.UTXOID{{ TxID: mintNFTTx.ID(), OutputIndex: 0, }}, @@ -840,15 +838,15 @@ func TestIssueProperty(t *testing.T) { genesisBytes, issuer, []*common.Fx{ - &common.Fx{ + { ID: ids.Empty.Prefix(0), Fx: &secp256k1fx.Fx{}, }, - &common.Fx{ + { ID: ids.Empty.Prefix(1), Fx: &nftfx.Fx{}, }, - &common.Fx{ + { ID: ids.Empty.Prefix(2), Fx: &propertyfx.Fx{}, }, @@ -877,7 +875,7 @@ func TestIssueProperty(t *testing.T) { Name: "Team Rocket", Symbol: "TR", Denomination: 0, - States: []*InitialState{&InitialState{ + States: []*InitialState{{ FxID: 2, Outs: []verify.Verifiable{ &propertyfx.MintOutput{ @@ -905,9 +903,9 @@ func TestIssueProperty(t *testing.T) { NetID: networkID, BCID: chainID, }, - Ops: []*Operation{&Operation{ + Ops: []*Operation{{ Asset: ava.Asset{ID: createAssetTx.ID()}, - UTXOIDs: []*ava.UTXOID{&ava.UTXOID{ + UTXOIDs: []*ava.UTXOID{{ TxID: createAssetTx.ID(), OutputIndex: 0, }}, @@ -960,9 +958,9 @@ func TestIssueProperty(t *testing.T) { NetID: networkID, BCID: chainID, }, - Ops: []*Operation{&Operation{ + Ops: []*Operation{{ Asset: ava.Asset{ID: createAssetTx.ID()}, - UTXOIDs: []*ava.UTXOID{&ava.UTXOID{ + UTXOIDs: []*ava.UTXOID{{ TxID: mintPropertyTx.ID(), OutputIndex: 1, }}, From a3d3ef4787042e4a631a856d0c6086830d3294bf Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Thu, 18 Jun 2020 12:29:17 -0400 Subject: [PATCH 16/80] Change ForceAccepted to process list of already stored vtxs --- snow/engine/avalanche/bootstrapper.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index d5e5e7f..5646195 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -126,11 +126,14 @@ func (b *bootstrapper) fetch(vtxID ids.ID) error { } // Process vertices -func (b *bootstrapper) process(vtx avalanche.Vertex) error { +func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error { toProcess := newMaxVertexHeap() - if _, ok := b.processedCache.Get(vtx.ID()); !ok { // only process if we haven't already - toProcess.Push(vtx) + for _, vtx := range vtxs { + if _, ok := b.processedCache.Get(vtx.ID()); !ok { // only process if we haven't already + toProcess.Push(vtx) + } } + for toProcess.Len() > 0 { vtx := toProcess.Pop() switch vtx.Status() { @@ -217,14 +220,19 @@ func (b *bootstrapper) MultiPut(vdr ids.ShortID, requestID uint32, vtxs [][]byte return b.fetch(neededVtxID) } + processVertices := make([]avalanche.Vertex, 1, len(vtxs)) + processVertices[0] = neededVtx + for _, vtxBytes := range vtxs { // Parse/persist all the vertices - if _, err := b.State.ParseVertex(vtxBytes); err != nil { // Persists the vtx + if vtx, err := b.State.ParseVertex(vtxBytes); err != nil { // Persists the vtx b.BootstrapConfig.Context.Log.Debug("Failed to parse vertex: %w", err) b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes}) + } else { + processVertices = append(processVertices, vtx) } } - return b.process(neededVtx) + return b.process(processVertices...) } // GetAncestorsFailed is called when a GetAncestors message we sent fails @@ -245,15 +253,17 @@ func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error { err) } + storedVtxs := make([]avalanche.Vertex, 0, acceptedContainerIDs.Len()) for _, vtxID := range acceptedContainerIDs.List() { if vtx, err := b.State.GetVertex(vtxID); err == nil { - if err := b.process(vtx); err != nil { - return err - } + storedVtxs = append(storedVtxs, vtx) } else if err := b.fetch(vtxID); err != nil { return err } } + if err := b.process(storedVtxs...); err != nil { + return err + } b.processedStartingAcceptedFrontier = true if numPending := b.outstandingRequests.Len(); numPending == 0 { From 1be5daf5cf352b5cdb13ca467b7328050ccd09a0 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 07:44:28 -0300 Subject: [PATCH 17/80] updating CI --- .ci/run_e2e_tests.sh | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 56d778d..0e43ba0 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,39 +1,39 @@ LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" -docker pull ${LATEST_CONTROLLER_TAG} -docker pull ${LATEST_KURTOSIS_TAG} +#docker pull ${LATEST_CONTROLLER_TAG} SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) SRC_PATH=$(dirname "${SCRIPTS_PATH}") # build docker image we need -bash ${SRC_PATH}/scripts/build_image.sh +bash "${SRC_PATH}"/scripts/build_image.sh # get docker image label GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) -(docker run -v /var/run/docker.sock:/var/run/docker.sock \ ---env DEFAULT_GECKO_IMAGE="${GECKO_IMAGE}" \ ---env TEST_CONTROLLER_IMAGE="${LATEST_CONTROLLER_TAG}" \ -${LATEST_KURTOSIS_TAG}) & +go get -d -t -v github.com/kurtosis-tech/ava-e2e-tests/... -kurtosis_pid=$! +cd "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests -sleep 90 -kill ${kurtosis_pid} +./scripts/full_rebuild_and_run.sh -ACTUAL_EXIT_STATUS=$(docker ps -a --latest --filter ancestor=${LATEST_CONTROLLER_TAG} --format="{{.Status}}") -EXPECTED_EXIT_STATUS="Exited \(0\).*" - -# Clear containers. -echo "Clearing kurtosis testnet containers." -docker rm $(docker stop $(docker ps -a -q --filter ancestor="${GECKO_IMAGE}" --format="{{.ID}}")) >/dev/null - -if [[ ${ACTUAL_EXIT_STATUS} =~ ${EXPECTED_EXIT_STATUS} ]] -then - echo "Kurtosis test succeeded." - exit 0 -else - echo "Kurtosis test failed." - exit 1 -fi +#kurtosis_pid=$! +# +#sleep 90 +#kill ${kurtosis_pid} +# +#ACTUAL_EXIT_STATUS=$(docker ps -a --latest --filter ancestor=${LATEST_CONTROLLER_TAG} --format="{{.Status}}") +#EXPECTED_EXIT_STATUS="Exited \(0\).*" +# +## Clear containers. +#echo "Clearing kurtosis testnet containers." +#docker rm $(docker stop $(docker ps -a -q --filter ancestor="${GECKO_IMAGE}" --format="{{.ID}}")) >/dev/null +# +#if [[ ${ACTUAL_EXIT_STATUS} =~ ${EXPECTED_EXIT_STATUS} ]] +#then +# echo "Kurtosis test succeeded." +# exit 0 +#else +# echo "Kurtosis test failed." +# exit 1 +#fi From 4b22442fd26936e965092881a031397e737fbffb Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 08:01:06 -0300 Subject: [PATCH 18/80] changing path to access ava-e2e-tests --- .ci/run_e2e_tests.sh | 2 +- .travis.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 0e43ba0..9b7dac6 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -13,7 +13,7 @@ GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) go get -d -t -v github.com/kurtosis-tech/ava-e2e-tests/... -cd "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests +cd "${E2E_TEST_HOME}" || exit ./scripts/full_rebuild_and_run.sh diff --git a/.travis.yml b/.travis.yml index bb57fc9..aac0189 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: global: - CODECOV_TOKEN="8c18c993-fc6e-4706-998b-01ddc7987804" - GECKO_HOME=/go/src/github.com/ava-labs/gecko/ + - E2E_TEST_HOME=/go/src/github.com/kurtosis-tech/ava-e2e-tests/ - COMMIT=${TRAVIS_COMMIT::8} - DOCKERHUB_REPO=avaplatform/gecko - secure: "L/A9+re0NEKP6EV6H9RcTGiDhX3WMvsiWrkRKDYKqnviqbjY30RK6EM4vvjrM4Lrw2QwsO3YKgnku3+zioE/TxEZFkpkbjNUXru0nYBrWAg1TKVsDXnYaIZkHUejfryST3E8N7F4Hx6zCtGEO0sEdUeKuT+MNUIuHezHooTgGzDjMogm70EWMFjQHc7VucTJu7dWU1RBPjovWQ0q9qflrtCpbrvXFIiihQQ1PQha1Q2C4wLakKuLbhhSafue90Mnyss0blaPHy/tyewcASJu4vsGTKRBn0DzttlkNTwuD6+nKrbmJY0ohunnkVFzYjrZAw1gyN+DCDb/lPbz4ZDItKPwrIUPEtL5xuUOrxUZPUh+0io3Q2d6rjaqkdGjd1KQXzbnW1mn0BxX3d3b2UpIqhBn9umYYjHBKnMuoRiTK33b7U9+LF3K84+tEvVDCPeHs/mw6Inp5jGRSravnM6yPQ6feGzogs4+3EMzZXxnkngKFKCsnd67Oe9xfV9amOU2aQAx4jaAwlPjEpBEkUa8YKx3lPznvmUk1QsNCUbLjdSl5JBaXojLJoiuPbj29hp4S5AXXgn+3Hvwk3ndcFCxi6/l1W9mjYSOtFqg3EAUdF4EgnA/ykQg9ZokkoKY0+qgOzG2bKOAYuCDWeGr7P1apToh00ccsQXL81nVPiq7uDw=" From 5cc4aa471875c13cb81a078c943818b74381f551 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 08:16:43 -0300 Subject: [PATCH 19/80] adding debugging steps --- .ci/run_e2e_tests.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 9b7dac6..eec6946 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,8 +1,8 @@ -LATEST_KURTOSIS_TAG="kurtosistech/kurtosis:latest" -LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" - +#LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" #docker pull ${LATEST_CONTROLLER_TAG} +set -x + SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) SRC_PATH=$(dirname "${SCRIPTS_PATH}") @@ -13,6 +13,13 @@ GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) go get -d -t -v github.com/kurtosis-tech/ava-e2e-tests/... +ls -ltrh "${GOPATH}" +ls -ltrh "${GOPATH}"/src/ +ls -ltrh "${GOPATH}"/src/github.com +ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech +ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ +cd "${GOPATH}"/src/ || exit + cd "${E2E_TEST_HOME}" || exit ./scripts/full_rebuild_and_run.sh From ea7b77ca9d9e274e8c1f9483d7f0f5a2ea590931 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 08:17:35 -0300 Subject: [PATCH 20/80] adding correct cd --- .ci/run_e2e_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index eec6946..6a96958 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -18,9 +18,7 @@ ls -ltrh "${GOPATH}"/src/ ls -ltrh "${GOPATH}"/src/github.com ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ -cd "${GOPATH}"/src/ || exit - -cd "${E2E_TEST_HOME}" || exit +cd "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ || exit ./scripts/full_rebuild_and_run.sh From 9ac4472a27f4ac76acd0e983b6f74eeb400ff192 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 08:28:39 -0300 Subject: [PATCH 21/80] removing -d --- .ci/run_e2e_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 6a96958..b457c76 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -11,7 +11,7 @@ bash "${SRC_PATH}"/scripts/build_image.sh # get docker image label GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) -go get -d -t -v github.com/kurtosis-tech/ava-e2e-tests/... +go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... ls -ltrh "${GOPATH}" ls -ltrh "${GOPATH}"/src/ From 878056d24a341af689c43cabe5a9ee08dd9acab3 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 08:49:09 -0300 Subject: [PATCH 22/80] pulling in gopath --- .ci/run_e2e_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index b457c76..628e83b 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -11,10 +11,12 @@ bash "${SRC_PATH}"/scripts/build_image.sh # get docker image label GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) +export GOPATH="${GOPATH}" go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... ls -ltrh "${GOPATH}" ls -ltrh "${GOPATH}"/src/ +ls -ltrh "${GOPATH}"/bin/ ls -ltrh "${GOPATH}"/src/github.com ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ From 006ff7514996de983a8737bd2aa29f7599262742 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Fri, 19 Jun 2020 11:02:38 -0400 Subject: [PATCH 23/80] improve leveldb batch usage and pointer releasing in caches --- database/versiondb/db.go | 27 ++++++++++++-------- snow/engine/avalanche/state/unique_vertex.go | 2 ++ vms/avm/unique_tx.go | 6 ++++- vms/avm/vm.go | 2 +- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/database/versiondb/db.go b/database/versiondb/db.go index fb692bf..7223c55 100644 --- a/database/versiondb/db.go +++ b/database/versiondb/db.go @@ -18,9 +18,10 @@ import ( // database, writing changes to the underlying database only when commit is // called. type Database struct { - lock sync.RWMutex - mem map[string]valueDelete - db database.Database + lock sync.RWMutex + mem map[string]valueDelete + db database.Database + batch database.Batch } type valueDelete struct { @@ -31,8 +32,9 @@ type valueDelete struct { // New returns a new prefixed database func New(db database.Database) *Database { return &Database{ - mem: make(map[string]valueDelete, memdb.DefaultSize), - db: db, + mem: make(map[string]valueDelete, memdb.DefaultSize), + db: db, + batch: db.NewBatch(), } } @@ -169,6 +171,7 @@ func (db *Database) SetDatabase(newDB database.Database) error { } db.db = newDB + db.batch = newDB.NewBatch() return nil } @@ -206,7 +209,9 @@ func (db *Database) Abort() { func (db *Database) abort() { db.mem = make(map[string]valueDelete, memdb.DefaultSize) } -// CommitBatch returns a batch that will commit all pending writes to the underlying database +// CommitBatch returns a batch that will commit all pending writes to the +// underlying database. The returned batch should be written before future calls +// to this DB unless the batch will never be written. func (db *Database) CommitBatch() (database.Batch, error) { db.lock.Lock() defer db.lock.Unlock() @@ -219,21 +224,21 @@ func (db *Database) commitBatch() (database.Batch, error) { return nil, database.ErrClosed } - batch := db.db.NewBatch() + db.batch.Reset() for key, value := range db.mem { if value.delete { - if err := batch.Delete([]byte(key)); err != nil { + if err := db.batch.Delete([]byte(key)); err != nil { return nil, err } - } else if err := batch.Put([]byte(key), value.value); err != nil { + } else if err := db.batch.Put([]byte(key), value.value); err != nil { return nil, err } } - if err := batch.Write(); err != nil { + if err := db.batch.Write(); err != nil { return nil, err } - return batch, nil + return db.batch, nil } // Close implements the database.Database interface diff --git a/snow/engine/avalanche/state/unique_vertex.go b/snow/engine/avalanche/state/unique_vertex.go index f7c6927..75c5893 100644 --- a/snow/engine/avalanche/state/unique_vertex.go +++ b/snow/engine/avalanche/state/unique_vertex.go @@ -54,6 +54,8 @@ func (vtx *uniqueVertex) refresh() { func (vtx *uniqueVertex) Evict() { if vtx.v != nil { vtx.v.unique = false + // make sure the parents are able to be garbage collected + vtx.v.parents = nil } } diff --git a/vms/avm/unique_tx.go b/vms/avm/unique_tx.go index 892b513..576702e 100644 --- a/vms/avm/unique_tx.go +++ b/vms/avm/unique_tx.go @@ -85,7 +85,11 @@ func (tx *UniqueTx) refresh() { // Evict is called when this UniqueTx will no longer be returned from a cache // lookup -func (tx *UniqueTx) Evict() { tx.unique = false } // Lock is already held here +func (tx *UniqueTx) Evict() { + // Lock is already held here + tx.unique = false + tx.deps = nil +} func (tx *UniqueTx) setStatus(status choices.Status) error { tx.refresh() diff --git a/vms/avm/vm.go b/vms/avm/vm.go index d89d625..715ce95 100644 --- a/vms/avm/vm.go +++ b/vms/avm/vm.go @@ -35,7 +35,7 @@ const ( batchSize = 30 stateCacheSize = 10000 idCacheSize = 10000 - txCacheSize = 100000 + txCacheSize = 10000 addressSep = "-" ) From 8b75abdee03ef1b1ab1b2b92d2420201dbacf418 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 12:18:20 -0400 Subject: [PATCH 24/80] cache only contains vertices at height 5000, 10000, etc. --- snow/engine/avalanche/bootstrapper.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index 5646195..cdfb77b 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -17,7 +17,14 @@ import ( ) const ( - cacheSize = 100000 + // We cache processed vertices where height = c * stripeDistance for c = {1,2,3...} + // This forms a "stripe" of cached DAG vertices at height stripeDistance, 2*stripeDistance, etc. + // This helps to limit the number of repeated DAG traversals performed + // + // With stripeDistance == 2500, average DAG width == 25 and processedCache size == 100,000 + // the graph can have depth up to 10,000,000 and hold every stripe in cache (100,000 / 25 == x / 2,500) + stripeDistance = 2500 + cacheSize = 100000 ) // BootstrapConfig ... @@ -175,6 +182,9 @@ func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error { toProcess.Push(parent) } } + if vtx.Height()%stripeDistance == 0 { + b.processedCache.Put(vtx.ID(), nil) + } b.processedCache.Put(vtx.ID(), nil) } } From 28a313becf19ca1648ab9ba0a23743b6ff0e870d Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 19 Jun 2020 13:02:28 -0400 Subject: [PATCH 25/80] Fix param attribute access --- snow/engine/avalanche/transitive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snow/engine/avalanche/transitive.go b/snow/engine/avalanche/transitive.go index 74e2ed2..565267b 100644 --- a/snow/engine/avalanche/transitive.go +++ b/snow/engine/avalanche/transitive.go @@ -57,7 +57,7 @@ func (t *Transitive) Initialize(config Config) error { t.onFinished = t.finishBootstrapping - t.polls = newPolls(int(config.Alpha), config.Context.Log, t.numPolls) + t.polls = newPolls(int(config.Params.Alpha), config.Context.Log, t.numPolls) return t.bootstrapper.Initialize(config.BootstrapConfig) } From 25478846d3f97d23170d56ab412673802a56f176 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 19 Jun 2020 13:34:47 -0400 Subject: [PATCH 26/80] Remove extra string formatter in chain router QueryFailed message --- snow/networking/router/chain_router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snow/networking/router/chain_router.go b/snow/networking/router/chain_router.go index 8cf708a..bbb345d 100644 --- a/snow/networking/router/chain_router.go +++ b/snow/networking/router/chain_router.go @@ -320,7 +320,7 @@ func (sr *ChainRouter) QueryFailed(validatorID ids.ShortID, chainID ids.ID, requ if chain, exists := sr.chains[chainID.Key()]; exists { chain.QueryFailed(validatorID, requestID) } else { - sr.log.Error("QueryFailed(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID) + sr.log.Error("QueryFailed(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } } From 27bdba477608bf43279ff66d3511411d28fdf5dd Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 13:44:57 -0400 Subject: [PATCH 27/80] GET to /ext/health returns a 200 and no body, per Fabio's request --- api/health/service.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/health/service.go b/api/health/service.go index 1989ab3..51a7c82 100644 --- a/api/health/service.go +++ b/api/health/service.go @@ -38,7 +38,14 @@ func (h *Health) Handler() *common.HTTPHandler { newServer.RegisterCodec(codec, "application/json") newServer.RegisterCodec(codec, "application/json;charset=UTF-8") newServer.RegisterService(h, "health") - return &common.HTTPHandler{LockOptions: common.NoLock, Handler: newServer} + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { // GET request --> reply with 200 + w.WriteHeader(http.StatusOK) + } else { + newServer.ServeHTTP(w, r) // Other request --> use JSON RPC + } + }) + return &common.HTTPHandler{LockOptions: common.NoLock, Handler: handler} } // RegisterHeartbeat adds a check with default options and a CheckFn that checks From 979f4e27597824a8d089bd83abfcb17babb28863 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 14:05:11 -0400 Subject: [PATCH 28/80] GET to ext/health returns 200 if heathy, else 500 --- api/health/service.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/health/service.go b/api/health/service.go index 51a7c82..27a15f7 100644 --- a/api/health/service.go +++ b/api/health/service.go @@ -39,8 +39,12 @@ func (h *Health) Handler() *common.HTTPHandler { newServer.RegisterCodec(codec, "application/json;charset=UTF-8") newServer.RegisterService(h, "health") handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodGet { // GET request --> reply with 200 - w.WriteHeader(http.StatusOK) + if r.Method == http.MethodGet { // GET request --> return 200 if getLiveness returns true, else 500 + if _, healthy := h.health.Results(); healthy { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusInternalServerError) + } } else { newServer.ServeHTTP(w, r) // Other request --> use JSON RPC } From 0627c7c28e01885bdeb907acf0bbdee629249872 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Fri, 19 Jun 2020 15:50:09 -0300 Subject: [PATCH 29/80] turning GO111MODULE off to pull --- .ci/run_e2e_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 628e83b..67bd2a8 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -12,7 +12,7 @@ bash "${SRC_PATH}"/scripts/build_image.sh GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) export GOPATH="${GOPATH}" -go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... +GO111MODULE=off go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... ls -ltrh "${GOPATH}" ls -ltrh "${GOPATH}"/src/ From 975198bb5e8c16aeb42c7e99047df13bd3f4439b Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Fri, 19 Jun 2020 15:13:34 -0400 Subject: [PATCH 30/80] report error returned by the snowstorm RecordPoll --- snow/consensus/avalanche/topological.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/snow/consensus/avalanche/topological.go b/snow/consensus/avalanche/topological.go index f99cff1..d786a0f 100644 --- a/snow/consensus/avalanche/topological.go +++ b/snow/consensus/avalanche/topological.go @@ -141,7 +141,9 @@ func (ta *Topological) RecordPoll(responses ids.UniqueBag) error { votes := ta.pushVotes(kahns, leaves) // Update the conflict graph: O(|Transactions|) ta.ctx.Log.Verbo("Updating consumer confidences based on:\n%s", &votes) - ta.cg.RecordPoll(votes) + if err := ta.cg.RecordPoll(votes); err != nil { + return err + } // Update the dag: O(|Live Set|) return ta.updateFrontiers() } From 245f13ea6542384de34af7813914ac6baab81ff7 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 16:29:27 -0400 Subject: [PATCH 31/80] add stripeDistance and stripeWidth --- snow/engine/avalanche/bootstrapper.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index cdfb77b..08ca7df 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -20,10 +20,8 @@ const ( // We cache processed vertices where height = c * stripeDistance for c = {1,2,3...} // This forms a "stripe" of cached DAG vertices at height stripeDistance, 2*stripeDistance, etc. // This helps to limit the number of repeated DAG traversals performed - // - // With stripeDistance == 2500, average DAG width == 25 and processedCache size == 100,000 - // the graph can have depth up to 10,000,000 and hold every stripe in cache (100,000 / 25 == x / 2,500) - stripeDistance = 2500 + stripeDistance = 2000 + stripeWidth = 5 cacheSize = 100000 ) @@ -182,10 +180,9 @@ func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error { toProcess.Push(parent) } } - if vtx.Height()%stripeDistance == 0 { + if vtx.Height()%stripeDistance < stripeWidth { b.processedCache.Put(vtx.ID(), nil) } - b.processedCache.Put(vtx.ID(), nil) } } From 7c05353d3d048d3a01eea52597c91611664bd095 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 16:33:33 -0400 Subject: [PATCH 32/80] skip unneccessary parse --- snow/engine/avalanche/bootstrapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index 08ca7df..3ed58c7 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -230,7 +230,7 @@ func (b *bootstrapper) MultiPut(vdr ids.ShortID, requestID uint32, vtxs [][]byte processVertices := make([]avalanche.Vertex, 1, len(vtxs)) processVertices[0] = neededVtx - for _, vtxBytes := range vtxs { // Parse/persist all the vertices + for _, vtxBytes := range vtxs[1:] { // Parse/persist all the vertices if vtx, err := b.State.ParseVertex(vtxBytes); err != nil { // Persists the vtx b.BootstrapConfig.Context.Log.Debug("Failed to parse vertex: %w", err) b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes}) From 6c34fd79eb0cfb080be256178582a56d2480275d Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Fri, 19 Jun 2020 17:56:35 -0400 Subject: [PATCH 33/80] version bump --- node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node.go b/node/node.go index 752c78d..d003767 100644 --- a/node/node.go +++ b/node/node.go @@ -57,7 +57,7 @@ var ( genesisHashKey = []byte("genesisID") // Version is the version of this code - Version = version.NewDefaultVersion("avalanche", 0, 5, 5) + Version = version.NewDefaultVersion("avalanche", 0, 5, 6) versionParser = version.NewDefaultParser() ) From 4d4a073d604d9d37948ff144dc01c9fb0e2d69dc Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 17:57:47 -0400 Subject: [PATCH 34/80] only have 15 outstanding GetAncestors at a time during bootstrapping to not flood the network --- snow/engine/avalanche/bootstrapper.go | 33 +++++++++++++++++++++++++-- snow/engine/common/bootstrapper.go | 3 +++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index 3ed58c7..749694e 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -51,6 +51,11 @@ type bootstrapper struct { // tracks which validators were asked for which containers in which requests outstandingRequests common.Requests + // IDs of vertices that we will send a GetAncestors request for once we are not at the + // max number of outstanding requests + // Invariant: The intersection of needToFetch and outstandingRequests is empty + needToFetch ids.Set + // Contains IDs of vertices that have recently been processed processedCache *cache.LRU @@ -103,11 +108,21 @@ func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set { return acceptedVtxIDs } -// Get vertex [vtxID] and its ancestors +// Calls fetch for a pending vertex if there are any +func (b *bootstrapper) fetchANeededVtx() error { + if b.needToFetch.Len() > 0 { + return b.fetch(b.needToFetch.List()[0]) + } + return nil +} + +// Get vertex [vtxID] and its ancestors. +// If [vtxID] has already been requested or is already fetched, and there are +// unrequested vertices, requests one such vertex instead of [vtxID] func (b *bootstrapper) fetch(vtxID ids.ID) error { // Make sure we haven't already requested this block if b.outstandingRequests.Contains(vtxID) { - return nil + return b.fetchANeededVtx() } // Make sure we don't already have this vertex @@ -115,6 +130,13 @@ func (b *bootstrapper) fetch(vtxID ids.ID) error { if numPending := b.outstandingRequests.Len(); numPending == 0 && b.processedStartingAcceptedFrontier { return b.finish() } + b.needToFetch.Remove(vtxID) // we have this vertex. no need to request it. + return b.fetchANeededVtx() + } + + // If we're already at maximum number of outstanding requests, queue for later + if b.outstandingRequests.Len() >= common.MaxOutstandingRequests { + b.needToFetch.Add(vtxID) return nil } @@ -126,6 +148,7 @@ func (b *bootstrapper) fetch(vtxID ids.ID) error { b.RequestID++ b.outstandingRequests.Add(validatorID, b.RequestID, vtxID) + b.needToFetch.Remove(vtxID) // maintains invariant that intersection with outstandingRequests is empty b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, vtxID) // request vertex and ancestors return nil } @@ -236,9 +259,15 @@ func (b *bootstrapper) MultiPut(vdr ids.ShortID, requestID uint32, vtxs [][]byte b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes}) } else { processVertices = append(processVertices, vtx) + b.needToFetch.Remove(vtx.ID()) // No need to fetch this vertex since we have it now } } + // Now there is one less outstanding request; send another if needed + if err := b.fetchANeededVtx(); err != nil { + return err + } + return b.process(processVertices...) } diff --git a/snow/engine/common/bootstrapper.go b/snow/engine/common/bootstrapper.go index 49f4051..963a4fb 100644 --- a/snow/engine/common/bootstrapper.go +++ b/snow/engine/common/bootstrapper.go @@ -17,6 +17,9 @@ const ( // StatusUpdateFrequency ... bootstrapper logs "processed X blocks/vertices" every [statusUpdateFrequency] blocks/vertices StatusUpdateFrequency = 2500 + + // MaxOutstandingRequests is the maximum number of GetAncestors sent but not responsded to/failed + MaxOutstandingRequests = 15 ) var ( From 67d92815010bc617c01094ccedd4d307ba57234a Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 19 Jun 2020 18:06:04 -0400 Subject: [PATCH 35/80] change maximum # outstanding to 8 to reduce load on nodes --- snow/engine/common/bootstrapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snow/engine/common/bootstrapper.go b/snow/engine/common/bootstrapper.go index 963a4fb..8c9c745 100644 --- a/snow/engine/common/bootstrapper.go +++ b/snow/engine/common/bootstrapper.go @@ -19,7 +19,7 @@ const ( StatusUpdateFrequency = 2500 // MaxOutstandingRequests is the maximum number of GetAncestors sent but not responsded to/failed - MaxOutstandingRequests = 15 + MaxOutstandingRequests = 8 ) var ( From 32812e5375d763d33d67ea0657d22f9dfbdb1680 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Fri, 19 Jun 2020 18:36:45 -0400 Subject: [PATCH 36/80] re-added the admin API calls to be backwards compatible --- api/admin/service.go | 96 +++++++++++++++++++++++++++++++++++++++++++- node/node.go | 2 +- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/api/admin/service.go b/api/admin/service.go index 3d61730..0718dfd 100644 --- a/api/admin/service.go +++ b/api/admin/service.go @@ -10,35 +10,129 @@ import ( "github.com/ava-labs/gecko/api" "github.com/ava-labs/gecko/chains" + "github.com/ava-labs/gecko/genesis" + "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/network" "github.com/ava-labs/gecko/snow/engine/common" "github.com/ava-labs/gecko/utils/logging" + "github.com/ava-labs/gecko/version" cjson "github.com/ava-labs/gecko/utils/json" ) // Admin is the API service for node admin management type Admin struct { + version version.Version + nodeID ids.ShortID + networkID uint32 log logging.Logger + networking network.Network performance Performance chainManager chains.Manager httpServer *api.Server } // NewService returns a new admin API service -func NewService(log logging.Logger, chainManager chains.Manager, peers network.Network, httpServer *api.Server) *common.HTTPHandler { +func NewService(version version.Version, nodeID ids.ShortID, networkID uint32, log logging.Logger, chainManager chains.Manager, peers network.Network, httpServer *api.Server) *common.HTTPHandler { newServer := rpc.NewServer() codec := cjson.NewCodec() newServer.RegisterCodec(codec, "application/json") newServer.RegisterCodec(codec, "application/json;charset=UTF-8") newServer.RegisterService(&Admin{ + version: version, + nodeID: nodeID, + networkID: networkID, log: log, chainManager: chainManager, + networking: peers, httpServer: httpServer, }, "admin") return &common.HTTPHandler{Handler: newServer} } +// GetNodeVersionReply are the results from calling GetNodeVersion +type GetNodeVersionReply struct { + Version string `json:"version"` +} + +// GetNodeVersion returns the version this node is running +func (service *Admin) GetNodeVersion(_ *http.Request, _ *struct{}, reply *GetNodeVersionReply) error { + service.log.Info("Admin: GetNodeVersion called") + + reply.Version = service.version.String() + return nil +} + +// GetNodeIDReply are the results from calling GetNodeID +type GetNodeIDReply struct { + NodeID ids.ShortID `json:"nodeID"` +} + +// GetNodeID returns the node ID of this node +func (service *Admin) GetNodeID(_ *http.Request, _ *struct{}, reply *GetNodeIDReply) error { + service.log.Info("Admin: GetNodeID called") + + reply.NodeID = service.nodeID + return nil +} + +// GetNetworkIDReply are the results from calling GetNetworkID +type GetNetworkIDReply struct { + NetworkID cjson.Uint32 `json:"networkID"` +} + +// GetNetworkID returns the network ID this node is running on +func (service *Admin) GetNetworkID(_ *http.Request, _ *struct{}, reply *GetNetworkIDReply) error { + service.log.Info("Admin: GetNetworkID called") + + reply.NetworkID = cjson.Uint32(service.networkID) + return nil +} + +// GetNetworkNameReply is the result from calling GetNetworkName +type GetNetworkNameReply struct { + NetworkName string `json:"networkName"` +} + +// GetNetworkName returns the network name this node is running on +func (service *Admin) GetNetworkName(_ *http.Request, _ *struct{}, reply *GetNetworkNameReply) error { + service.log.Info("Admin: GetNetworkName called") + + reply.NetworkName = genesis.NetworkName(service.networkID) + return nil +} + +// GetBlockchainIDArgs are the arguments for calling GetBlockchainID +type GetBlockchainIDArgs struct { + Alias string `json:"alias"` +} + +// GetBlockchainIDReply are the results from calling GetBlockchainID +type GetBlockchainIDReply struct { + BlockchainID string `json:"blockchainID"` +} + +// GetBlockchainID returns the blockchain ID that resolves the alias that was supplied +func (service *Admin) GetBlockchainID(_ *http.Request, args *GetBlockchainIDArgs, reply *GetBlockchainIDReply) error { + service.log.Info("Admin: GetBlockchainID called") + + bID, err := service.chainManager.Lookup(args.Alias) + reply.BlockchainID = bID.String() + return err +} + +// PeersReply are the results from calling Peers +type PeersReply struct { + Peers []network.PeerID `json:"peers"` +} + +// Peers returns the list of current validators +func (service *Admin) Peers(_ *http.Request, _ *struct{}, reply *PeersReply) error { + service.log.Info("Admin: Peers called") + reply.Peers = service.networking.Peers() + return nil +} + // StartCPUProfilerArgs are the arguments for calling StartCPUProfiler type StartCPUProfilerArgs struct { Filename string `json:"filename"` diff --git a/node/node.go b/node/node.go index d003767..dbc58a8 100644 --- a/node/node.go +++ b/node/node.go @@ -462,7 +462,7 @@ func (n *Node) initMetricsAPI() { func (n *Node) initAdminAPI() { if n.Config.AdminAPIEnabled { n.Log.Info("initializing Admin API") - service := admin.NewService(n.Log, n.chainManager, n.Net, &n.APIServer) + service := admin.NewService(Version, n.ID, n.Config.NetworkID, n.Log, n.chainManager, n.Net, &n.APIServer) n.APIServer.AddRoute(service, &sync.RWMutex{}, "admin", "", n.HTTPLog) } } From 2e16f2087a58f39b74a64e6f0046feaca70d06b5 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 10:22:55 -0300 Subject: [PATCH 37/80] cleaning up CI script --- .ci/run_e2e_tests.sh | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 67bd2a8..6aef7a3 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,46 +1,10 @@ -#LATEST_CONTROLLER_TAG="kurtosistech/ava-test-controller:latest" -#docker pull ${LATEST_CONTROLLER_TAG} - -set -x - SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) SRC_PATH=$(dirname "${SCRIPTS_PATH}") - -# build docker image we need +# Build the runnable Gecko docker image bash "${SRC_PATH}"/scripts/build_image.sh -# get docker image label GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) -export GOPATH="${GOPATH}" GO111MODULE=off go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... -ls -ltrh "${GOPATH}" -ls -ltrh "${GOPATH}"/src/ -ls -ltrh "${GOPATH}"/bin/ -ls -ltrh "${GOPATH}"/src/github.com -ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech -ls -ltrh "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ cd "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ || exit - ./scripts/full_rebuild_and_run.sh - -#kurtosis_pid=$! -# -#sleep 90 -#kill ${kurtosis_pid} -# -#ACTUAL_EXIT_STATUS=$(docker ps -a --latest --filter ancestor=${LATEST_CONTROLLER_TAG} --format="{{.Status}}") -#EXPECTED_EXIT_STATUS="Exited \(0\).*" -# -## Clear containers. -#echo "Clearing kurtosis testnet containers." -#docker rm $(docker stop $(docker ps -a -q --filter ancestor="${GECKO_IMAGE}" --format="{{.ID}}")) >/dev/null -# -#if [[ ${ACTUAL_EXIT_STATUS} =~ ${EXPECTED_EXIT_STATUS} ]] -#then -# echo "Kurtosis test succeeded." -# exit 0 -#else -# echo "Kurtosis test failed." -# exit 1 -#fi From a523fb184aba68b5e2c6991a6db4e8bf17e0bda6 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 10:24:02 -0300 Subject: [PATCH 38/80] cleaning CI run script --- .ci/run_e2e_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 6aef7a3..119fcf2 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -4,7 +4,7 @@ SRC_PATH=$(dirname "${SCRIPTS_PATH}") bash "${SRC_PATH}"/scripts/build_image.sh GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) +# Turn off GO111MODULE to pull e2e test source code in order to get run script. GO111MODULE=off go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... - cd "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ || exit ./scripts/full_rebuild_and_run.sh From 4cacb56cf53aaa6ed435d8a9cd3f3182261da33c Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 12:12:53 -0300 Subject: [PATCH 39/80] commenting controller image label from script --- .ci/run_e2e_tests.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 119fcf2..fa5bb8d 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -7,4 +7,10 @@ GECKO_IMAGE=$(docker image ls --format="{{.Repository}}" | head -n 1) # Turn off GO111MODULE to pull e2e test source code in order to get run script. GO111MODULE=off go get -t -v github.com/kurtosis-tech/ava-e2e-tests/... cd "${GOPATH}"/src/github.com/kurtosis-tech/ava-e2e-tests/ || exit -./scripts/full_rebuild_and_run.sh + +bash "./scripts/rebuild_initializer_binary.sh" +bash "./scripts/rebuild_controller_image.sh" +# TODO: Make the controller image label a parameter to rebuild_controller_image script +# Standard controller image label used by above scripts. +CONTROLLER_IMAGE="kurtosistech/ava-e2e-tests_controller:latest" +bash "./build/ava-e2e-tests --gecko-image-name=${GECKO_IMAGE} --test-controller-image-name=${CONTROLLER_IMAGE}" From 551e16fe368a4406a59bf63fdab23edaa1e583ab Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 12:26:36 -0300 Subject: [PATCH 40/80] checking build directory --- .ci/run_e2e_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index fa5bb8d..03c2582 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,3 +1,5 @@ +set -x + SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) SRC_PATH=$(dirname "${SCRIPTS_PATH}") # Build the runnable Gecko docker image @@ -13,4 +15,6 @@ bash "./scripts/rebuild_controller_image.sh" # TODO: Make the controller image label a parameter to rebuild_controller_image script # Standard controller image label used by above scripts. CONTROLLER_IMAGE="kurtosistech/ava-e2e-tests_controller:latest" +ls -ltrh ./ +ls -ltrh ./build/ bash "./build/ava-e2e-tests --gecko-image-name=${GECKO_IMAGE} --test-controller-image-name=${CONTROLLER_IMAGE}" From 0f0439ff159f3210e97ce0ec2ca4f07891e23bd0 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 12:42:23 -0300 Subject: [PATCH 41/80] calling executable raw --- .ci/run_e2e_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index 03c2582..df6b1c2 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -17,4 +17,4 @@ bash "./scripts/rebuild_controller_image.sh" CONTROLLER_IMAGE="kurtosistech/ava-e2e-tests_controller:latest" ls -ltrh ./ ls -ltrh ./build/ -bash "./build/ava-e2e-tests --gecko-image-name=${GECKO_IMAGE} --test-controller-image-name=${CONTROLLER_IMAGE}" +./build/ava-e2e-tests --gecko-image-name="${GECKO_IMAGE}" --test-controller-image-name="${CONTROLLER_IMAGE}" From 50fba7520babce27ba1fa865b34c175e58c429fa Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 12:54:02 -0300 Subject: [PATCH 42/80] defining just two tests --- .ci/run_e2e_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index df6b1c2..a4f0b81 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -15,6 +15,4 @@ bash "./scripts/rebuild_controller_image.sh" # TODO: Make the controller image label a parameter to rebuild_controller_image script # Standard controller image label used by above scripts. CONTROLLER_IMAGE="kurtosistech/ava-e2e-tests_controller:latest" -ls -ltrh ./ -ls -ltrh ./build/ -./build/ava-e2e-tests --gecko-image-name="${GECKO_IMAGE}" --test-controller-image-name="${CONTROLLER_IMAGE}" +./build/ava-e2e-tests --gecko-image-name="${GECKO_IMAGE}" --test-controller-image-name="${CONTROLLER_IMAGE}" --test="fiveStakingNodeGetValidatorsTest,fiveStakingNodeFullyConnectedTest" From f4a428351d6dd5253a99a679428f141c9b17fcfa Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sat, 20 Jun 2020 13:02:04 -0300 Subject: [PATCH 43/80] corrected test-names arg --- .ci/run_e2e_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index a4f0b81..fee3861 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -15,4 +15,4 @@ bash "./scripts/rebuild_controller_image.sh" # TODO: Make the controller image label a parameter to rebuild_controller_image script # Standard controller image label used by above scripts. CONTROLLER_IMAGE="kurtosistech/ava-e2e-tests_controller:latest" -./build/ava-e2e-tests --gecko-image-name="${GECKO_IMAGE}" --test-controller-image-name="${CONTROLLER_IMAGE}" --test="fiveStakingNodeGetValidatorsTest,fiveStakingNodeFullyConnectedTest" +./build/ava-e2e-tests --gecko-image-name="${GECKO_IMAGE}" --test-controller-image-name="${CONTROLLER_IMAGE}" --test-names="fiveStakingNodeGetValidatorsTest,fiveStakingNodeFullyConnectedTest" From 6a37d268bcce86eef0c24c77b98c77b77f840911 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 21 Jun 2020 16:12:53 -0300 Subject: [PATCH 44/80] replacing whitespace --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aac0189..559bed8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then .ci/before_install_linux.sh; fi install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then DOCKER_BUILDKIT=1 docker build --progress plain -t $DOCKERHUB_REPO:$COMMIT .; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then DOCKER_BUILDKIT=1 docker build --progress plain -t $DOCKERHUB_REPO:$COMMIT . ; fi script: - if [ "$TRAVIS_OS_NAME" = "osx" ]; then .ci/runscript_osx.sh; fi From e2aea232147da693608bf67844eb8a9fe332a87f Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 21 Jun 2020 16:13:59 -0300 Subject: [PATCH 45/80] re-enabling osx --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 559bed8..b06f3c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ jobs: include: - os: linux dist: bionic - #- os: osx - # osx_image: xcode11.4 + - os: osx + osx_image: xcode11.4 services: - docker env: From 62340e4f29c2ac4978dea280d6461e4d159c6421 Mon Sep 17 00:00:00 2001 From: galenmarchetti Date: Sun, 21 Jun 2020 17:09:12 -0300 Subject: [PATCH 46/80] removing set x --- .ci/run_e2e_tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh index fee3861..82126d1 100755 --- a/.ci/run_e2e_tests.sh +++ b/.ci/run_e2e_tests.sh @@ -1,5 +1,3 @@ -set -x - SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) SRC_PATH=$(dirname "${SCRIPTS_PATH}") # Build the runnable Gecko docker image From b0ad887a42701724d3d059b01513766f40a54fb9 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 19:38:24 -0400 Subject: [PATCH 47/80] refactored polls into their own sub package --- .../avalanche/poll/early_term_no_traversal.go | 85 +++++++ .../poll/early_term_no_traversal_test.go | 207 ++++++++++++++++++ snow/engine/avalanche/poll/interfaces.go | 33 +++ snow/engine/avalanche/poll/no_early_term.go | 52 +++++ .../avalanche/poll/no_early_term_test.go | 91 ++++++++ snow/engine/avalanche/poll/set.go | 107 +++++++++ snow/engine/avalanche/poll/set_test.go | 94 ++++++++ snow/engine/avalanche/polls.go | 137 ------------ snow/engine/avalanche/polls_test.go | 99 --------- snow/engine/avalanche/transitive.go | 14 +- .../snowman/poll/early_term_no_traversal.go | 73 ++++++ .../poll/early_term_no_traversal_test.go | 205 +++++++++++++++++ snow/engine/snowman/poll/interfaces.go | 35 +++ snow/engine/snowman/poll/no_early_term.go | 55 +++++ .../engine/snowman/poll/no_early_term_test.go | 92 ++++++++ snow/engine/snowman/poll/set.go | 134 ++++++++++++ snow/engine/snowman/poll/set_test.go | 132 +++++++++++ snow/engine/snowman/polls.go | 115 ---------- snow/engine/snowman/transitive.go | 15 +- snow/engine/snowman/transitive_test.go | 4 +- snow/engine/snowman/voter.go | 2 +- 21 files changed, 1417 insertions(+), 364 deletions(-) create mode 100644 snow/engine/avalanche/poll/early_term_no_traversal.go create mode 100644 snow/engine/avalanche/poll/early_term_no_traversal_test.go create mode 100644 snow/engine/avalanche/poll/interfaces.go create mode 100644 snow/engine/avalanche/poll/no_early_term.go create mode 100644 snow/engine/avalanche/poll/no_early_term_test.go create mode 100644 snow/engine/avalanche/poll/set.go create mode 100644 snow/engine/avalanche/poll/set_test.go delete mode 100644 snow/engine/avalanche/polls.go delete mode 100644 snow/engine/avalanche/polls_test.go create mode 100644 snow/engine/snowman/poll/early_term_no_traversal.go create mode 100644 snow/engine/snowman/poll/early_term_no_traversal_test.go create mode 100644 snow/engine/snowman/poll/interfaces.go create mode 100644 snow/engine/snowman/poll/no_early_term.go create mode 100644 snow/engine/snowman/poll/no_early_term_test.go create mode 100644 snow/engine/snowman/poll/set.go create mode 100644 snow/engine/snowman/poll/set_test.go delete mode 100644 snow/engine/snowman/polls.go diff --git a/snow/engine/avalanche/poll/early_term_no_traversal.go b/snow/engine/avalanche/poll/early_term_no_traversal.go new file mode 100644 index 0000000..52fdae3 --- /dev/null +++ b/snow/engine/avalanche/poll/early_term_no_traversal.go @@ -0,0 +1,85 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + + "github.com/ava-labs/gecko/ids" +) + +type earlyTermNoTraversalFactory struct { + alpha int +} + +// NewEarlyTermNoTraversalFactory returns a factory that returns polls with +// early termination, without doing DAG traversals +func NewEarlyTermNoTraversalFactory(alpha int) Factory { + return &earlyTermNoTraversalFactory{alpha: alpha} +} + +func (f *earlyTermNoTraversalFactory) New(vdrs ids.ShortSet) Poll { + return &earlyTermNoTraversalPoll{ + polled: vdrs, + alpha: f.alpha, + } +} + +// earlyTermNoTraversalPoll finishes when any remaining validators can't change +// the result of the poll. However, does not terminate tightly with this bound. +// It terminates as quickly as it can without performing any DAG traversals. +type earlyTermNoTraversalPoll struct { + votes ids.UniqueBag + polled ids.ShortSet + alpha int +} + +// Vote registers a response for this poll +func (p *earlyTermNoTraversalPoll) Vote(vdr ids.ShortID, votes []ids.ID) { + if !p.polled.Contains(vdr) { + // if the validator wasn't polled or already responded to this poll, we + // should just drop the vote + return + } + + // make sure that a validator can't respond multiple times + p.polled.Remove(vdr) + + // track the votes the validator responded with + p.votes.Add(uint(p.polled.Len()), votes...) +} + +// Finished returns true when all validators have voted +func (p *earlyTermNoTraversalPoll) Finished() bool { + // If there are no outstanding queries, the poll is finished + numPending := p.polled.Len() + if numPending == 0 { + return true + } + // If there are still enough pending responses to include another vertex, + // then the poll must wait for more responses + if numPending > p.alpha { + return false + } + + // Ignore any vertex that has already received alpha votes. To safely skip + // DAG traversal, assume that all votes for vertices with less than alpha + // votes will be applied to a single shared ancestor. In this case, the poll + // can terminate early, iff there are not enough pending votes for this + // ancestor to receive alpha votes. + partialVotes := ids.BitSet(0) + for _, vote := range p.votes.List() { + if voters := p.votes.GetSet(vote); voters.Len() < p.alpha { + partialVotes.Union(voters) + } + } + return partialVotes.Len()+numPending < p.alpha +} + +// Result returns the result of this poll +func (p *earlyTermNoTraversalPoll) Result() ids.UniqueBag { return p.votes } + +func (p *earlyTermNoTraversalPoll) String() string { + return fmt.Sprintf("waiting on %s", p.polled) +} diff --git a/snow/engine/avalanche/poll/early_term_no_traversal_test.go b/snow/engine/avalanche/poll/early_term_no_traversal_test.go new file mode 100644 index 0000000..ba2d81a --- /dev/null +++ b/snow/engine/avalanche/poll/early_term_no_traversal_test.go @@ -0,0 +1,207 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "testing" + + "github.com/ava-labs/gecko/ids" +) + +func TestEarlyTermNoTraversalResults(t *testing.T) { + alpha := 1 + + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) // k = 1 + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } + + result := poll.Result() + if list := result.List(); len(list) != 1 { + t.Fatalf("Wrong number of vertices returned") + } else if retVtxID := list[0]; !retVtxID.Equals(vtxID) { + t.Fatalf("Wrong vertex returned") + } else if set := result.GetSet(vtxID); set.Len() != 1 { + t.Fatalf("Wrong number of votes returned") + } +} + +func TestEarlyTermNoTraversalString(t *testing.T) { + alpha := 2 + + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + + expected := "waiting on {BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp}" + if result := poll.String(); expected != result { + t.Fatalf("Poll should have returned %s but returned %s", expected, result) + } +} + +func TestEarlyTermNoTraversalDropsDuplicatedVotes(t *testing.T) { + alpha := 2 + + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr1, votes) + if poll.Finished() { + t.Fatalf("Poll finished after getting a duplicated vote") + } + poll.Vote(vdr2, votes) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } +} + +func TestEarlyTermNoTraversalTerminatesEarly(t *testing.T) { + alpha := 3 + + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) + vdr3 := ids.NewShortID([20]byte{3}) + vdr4 := ids.NewShortID([20]byte{4}) + vdr5 := ids.NewShortID([20]byte{5}) // k = 5 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + vdr3, + vdr4, + vdr5, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr2, votes) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr3, votes) + if !poll.Finished() { + t.Fatalf("Poll did not terminate early after receiving alpha votes for one vertex and none for other vertices") + } +} + +func TestEarlyTermNoTraversalForSharedAncestor(t *testing.T) { + alpha := 4 + + vtxA := ids.NewID([32]byte{1}) + vtxB := ids.NewID([32]byte{2}) + vtxC := ids.NewID([32]byte{3}) + vtxD := ids.NewID([32]byte{4}) + + // If validators 1-3 vote for frontier vertices + // B, C, and D respectively, which all share the common ancestor + // A, then we cannot terminate early with alpha = k = 4 + // If the final vote is cast for any of A, B, C, or D, then + // vertex A will have transitively received alpha = 4 votes + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) + vdr3 := ids.NewShortID([20]byte{3}) + vdr4 := ids.NewShortID([20]byte{4}) + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + vdrs.Add(vdr2) + vdrs.Add(vdr3) + vdrs.Add(vdr4) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, []ids.ID{vtxB}) + if poll.Finished() { + t.Fatalf("Poll finished early after receiving one vote") + } + poll.Vote(vdr2, []ids.ID{vtxC}) + if poll.Finished() { + t.Fatalf("Poll finished early after receiving two votes") + } + poll.Vote(vdr3, []ids.ID{vtxD}) + if poll.Finished() { + t.Fatalf("Poll terminated early, when a shared ancestor could have received alpha votes") + } + poll.Vote(vdr4, []ids.ID{vtxA}) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving all outstanding votes") + } +} + +func TestEarlyTermNoTraversalWithFastDrops(t *testing.T) { + alpha := 2 + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) + vdr3 := ids.NewShortID([20]byte{3}) // k = 3 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + vdr3, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, nil) + if poll.Finished() { + t.Fatalf("Poll finished early after dropping one vote") + } + poll.Vote(vdr2, nil) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after dropping two votes") + } +} diff --git a/snow/engine/avalanche/poll/interfaces.go b/snow/engine/avalanche/poll/interfaces.go new file mode 100644 index 0000000..05234a3 --- /dev/null +++ b/snow/engine/avalanche/poll/interfaces.go @@ -0,0 +1,33 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + + "github.com/ava-labs/gecko/ids" +) + +// Set is a collection of polls +type Set interface { + fmt.Stringer + + Add(requestID uint32, vdrs ids.ShortSet) bool + Vote(requestID uint32, vdr ids.ShortID, votes []ids.ID) (ids.UniqueBag, bool) + Len() int +} + +// Poll is an outstanding poll +type Poll interface { + fmt.Stringer + + Vote(vdr ids.ShortID, votes []ids.ID) + Finished() bool + Result() ids.UniqueBag +} + +// Factory creates a new Poll +type Factory interface { + New(vdrs ids.ShortSet) Poll +} diff --git a/snow/engine/avalanche/poll/no_early_term.go b/snow/engine/avalanche/poll/no_early_term.go new file mode 100644 index 0000000..9a06649 --- /dev/null +++ b/snow/engine/avalanche/poll/no_early_term.go @@ -0,0 +1,52 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + + "github.com/ava-labs/gecko/ids" +) + +type noEarlyTermFactory struct{} + +// NewNoEarlyTermFactory returns a factory that returns polls with no early +// termination +func NewNoEarlyTermFactory() Factory { return noEarlyTermFactory{} } + +func (noEarlyTermFactory) New(vdrs ids.ShortSet) Poll { + return &noEarlyTermPoll{polled: vdrs} +} + +// noEarlyTermPoll finishes when all polled validators either respond to the +// query or a timeout occurs +type noEarlyTermPoll struct { + votes ids.UniqueBag + polled ids.ShortSet +} + +// Vote registers a response for this poll +func (p *noEarlyTermPoll) Vote(vdr ids.ShortID, votes []ids.ID) { + if !p.polled.Contains(vdr) { + // if the validator wasn't polled or already responded to this poll, we + // should just drop the vote + return + } + + // make sure that a validator can't respond multiple times + p.polled.Remove(vdr) + + // track the votes the validator responded with + p.votes.Add(uint(p.polled.Len()), votes...) +} + +// Finished returns true when all validators have voted +func (p *noEarlyTermPoll) Finished() bool { return p.polled.Len() == 0 } + +// Result returns the result of this poll +func (p *noEarlyTermPoll) Result() ids.UniqueBag { return p.votes } + +func (p *noEarlyTermPoll) String() string { + return fmt.Sprintf("waiting on %s", p.polled) +} diff --git a/snow/engine/avalanche/poll/no_early_term_test.go b/snow/engine/avalanche/poll/no_early_term_test.go new file mode 100644 index 0000000..f877416 --- /dev/null +++ b/snow/engine/avalanche/poll/no_early_term_test.go @@ -0,0 +1,91 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "testing" + + "github.com/ava-labs/gecko/ids" +) + +func TestNoEarlyTermResults(t *testing.T) { + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) // k = 1 + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + + factory := NewNoEarlyTermFactory() + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } + + result := poll.Result() + if list := result.List(); len(list) != 1 { + t.Fatalf("Wrong number of vertices returned") + } else if retVtxID := list[0]; !retVtxID.Equals(vtxID) { + t.Fatalf("Wrong vertex returned") + } else if set := result.GetSet(vtxID); set.Len() != 1 { + t.Fatalf("Wrong number of votes returned") + } +} + +func TestNoEarlyTermString(t *testing.T) { + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewNoEarlyTermFactory() + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + + expected := "waiting on {BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp}" + if result := poll.String(); expected != result { + t.Fatalf("Poll should have returned %s but returned %s", expected, result) + } +} + +func TestNoEarlyTermDropsDuplicatedVotes(t *testing.T) { + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewNoEarlyTermFactory() + poll := factory.New(vdrs) + + poll.Vote(vdr1, votes) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr1, votes) + if poll.Finished() { + t.Fatalf("Poll finished after getting a duplicated vote") + } + poll.Vote(vdr2, votes) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } +} diff --git a/snow/engine/avalanche/poll/set.go b/snow/engine/avalanche/poll/set.go new file mode 100644 index 0000000..34a8a1a --- /dev/null +++ b/snow/engine/avalanche/poll/set.go @@ -0,0 +1,107 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/ava-labs/gecko/ids" + "github.com/ava-labs/gecko/utils/logging" +) + +type set struct { + log logging.Logger + numPolls prometheus.Gauge + factory Factory + polls map[uint32]Poll +} + +// NewSet returns a new empty set of polls +func NewSet( + factory Factory, + log logging.Logger, + namespace string, + registerer prometheus.Registerer, +) Set { + numPolls := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "polls", + Help: "Number of pending network polls", + }) + if err := registerer.Register(numPolls); err != nil { + log.Error("failed to register av_polls statistics due to %s", err) + } + + return &set{ + log: log, + numPolls: numPolls, + factory: factory, + polls: make(map[uint32]Poll), + } +} + +// Add to the current set of polls +// Returns true if the poll was registered correctly and the network sample +// should be made. +func (s *set) Add(requestID uint32, vdrs ids.ShortSet) bool { + if _, exists := s.polls[requestID]; exists { + s.log.Debug("dropping poll due to duplicated requestID: %d", requestID) + return false + } + + s.log.Verbo("creating poll with requestID %d and validators %s", + requestID, + vdrs) + + s.polls[requestID] = s.factory.New(vdrs) // create the new poll + s.numPolls.Inc() // increase the metrics + return true +} + +// Vote registers the connections response to a query for [id]. If there was no +// query, or the response has already be registered, nothing is performed. +func (s *set) Vote( + requestID uint32, + vdr ids.ShortID, + votes []ids.ID, +) (ids.UniqueBag, bool) { + poll, exists := s.polls[requestID] + if !exists { + s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d", + vdr, + requestID) + return nil, false + } + + s.log.Verbo("processing vote from %s in the poll with requestID: %d with the votes %v", + vdr, + requestID, + votes) + + poll.Vote(vdr, votes) + if !poll.Finished() { + return nil, false + } + + s.log.Verbo("poll with requestID %d finished as %s", requestID, poll) + + delete(s.polls, requestID) // remove the poll from the current set + s.numPolls.Dec() // decrease the metrics + return poll.Result(), true +} + +// Len returns the number of outstanding polls +func (s *set) Len() int { return len(s.polls) } + +func (s *set) String() string { + sb := strings.Builder{} + sb.WriteString(fmt.Sprintf("current polls: (Size = %d)", len(s.polls))) + for requestID, poll := range s.polls { + sb.WriteString(fmt.Sprintf("\n %d: %s", requestID, poll)) + } + return sb.String() +} diff --git a/snow/engine/avalanche/poll/set_test.go b/snow/engine/avalanche/poll/set_test.go new file mode 100644 index 0000000..496f993 --- /dev/null +++ b/snow/engine/avalanche/poll/set_test.go @@ -0,0 +1,94 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "testing" + + "github.com/ava-labs/gecko/ids" + "github.com/ava-labs/gecko/utils/logging" + "github.com/prometheus/client_golang/prometheus" +) + +func TestNewSetErrorOnMetrics(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + + registerer.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "polls", + })) + + _ = NewSet(factory, log, namespace, registerer) +} + +func TestCreateAndFinishPoll(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + s := NewSet(factory, log, namespace, registerer) + + vtxID := ids.NewID([32]byte{1}) + votes := []ids.ID{vtxID} + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + if s.Len() != 0 { + t.Fatalf("Shouldn't have any active polls yet") + } else if !s.Add(0, vdrs) { + t.Fatalf("Should have been able to add a new poll") + } else if s.Len() != 1 { + t.Fatalf("Should only have one active poll") + } else if s.Add(0, vdrs) { + t.Fatalf("Shouldn't have been able to add a duplicated poll") + } else if s.Len() != 1 { + t.Fatalf("Should only have one active poll") + } else if _, finished := s.Vote(1, vdr1, votes); finished { + t.Fatalf("Shouldn't have been able to finish a non-existant poll") + } else if _, finished := s.Vote(0, vdr1, votes); finished { + t.Fatalf("Shouldn't have been able to finish an ongoing poll") + } else if _, finished := s.Vote(0, vdr1, votes); finished { + t.Fatalf("Should have dropped a duplicated poll") + } else if result, finished := s.Vote(0, vdr2, votes); !finished { + t.Fatalf("Should have finished the") + } else if list := result.List(); len(list) != 1 { + t.Fatalf("Wrong number of vertices returned") + } else if retVtxID := list[0]; !retVtxID.Equals(vtxID) { + t.Fatalf("Wrong vertex returned") + } else if set := result.GetSet(vtxID); set.Len() != 2 { + t.Fatalf("Wrong number of votes returned") + } +} + +func TestSetString(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + s := NewSet(factory, log, namespace, registerer) + + vdr1 := ids.NewShortID([20]byte{1}) // k = 1 + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + + expected := "current polls: (Size = 1)\n" + + " 0: waiting on {6HgC8KRBEhXYbF4riJyJFLSHt37UNuRt}" + if !s.Add(0, vdrs) { + t.Fatalf("Should have been able to add a new poll") + } else if str := s.String(); expected != str { + t.Fatalf("Set return wrong string, Expected:\n%s\nReturned:\n%s", + expected, + str) + } +} diff --git a/snow/engine/avalanche/polls.go b/snow/engine/avalanche/polls.go deleted file mode 100644 index ac3fe5c..0000000 --- a/snow/engine/avalanche/polls.go +++ /dev/null @@ -1,137 +0,0 @@ -// (c) 2019-2020, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package avalanche - -import ( - "fmt" - "strings" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/utils/logging" -) - -// TODO: There is a conservative early termination case that doesn't require dag -// traversals we may want to implement. The algorithm would go as follows: -// Keep track of the number of response that reference an ID. If an ID gets >= -// alpha responses, then remove it from all responses and place it into a chit -// list. Remove all empty responses. If the number of responses + the number of -// pending responses is less than alpha, terminate the poll. -// In the synchronous + virtuous case, when everyone returns the same hash, the -// poll now terminates after receiving alpha responses. -// In the rogue case, it is possible that the poll doesn't terminate as quickly -// as possible, because IDs may have the alpha threshold but only when counting -// transitive votes. In this case, we may wait even if it is no longer possible -// for another ID to earn alpha votes. -// Because alpha is typically set close to k, this may not be performance -// critical. However, early termination may be performance critical with crashed -// nodes. - -type polls struct { - log logging.Logger - numPolls prometheus.Gauge - alpha int - m map[uint32]poll -} - -func newPolls(alpha int, log logging.Logger, numPolls prometheus.Gauge) polls { - return polls{ - log: log, - numPolls: numPolls, - alpha: alpha, - m: make(map[uint32]poll), - } -} - -// Add to the current set of polls -// Returns true if the poll was registered correctly and the network sample -// should be made. -func (p *polls) Add(requestID uint32, vdrs ids.ShortSet) bool { - poll, exists := p.m[requestID] - if !exists { - poll.polled = vdrs - poll.alpha = p.alpha - p.m[requestID] = poll - - p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics - } - return !exists -} - -// Vote registers the connections response to a query for [id]. If there was no -// query, or the response has already be registered, nothing is performed. -func (p *polls) Vote(requestID uint32, vdr ids.ShortID, votes []ids.ID) (ids.UniqueBag, bool) { - p.log.Verbo("Vote. requestID: %d. validatorID: %s.", requestID, vdr) - poll, exists := p.m[requestID] - p.log.Verbo("Poll: %+v", poll) - if !exists { - return nil, false - } - - poll.Vote(votes, vdr) - if poll.Finished() { - p.log.Verbo("Poll is finished") - delete(p.m, requestID) - p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics - return poll.votes, true - } - p.m[requestID] = poll - return nil, false -} - -func (p *polls) String() string { - sb := strings.Builder{} - - sb.WriteString(fmt.Sprintf("Current polls: (Size = %d)", len(p.m))) - for requestID, poll := range p.m { - sb.WriteString(fmt.Sprintf("\n %d: %s", requestID, poll)) - } - - return sb.String() -} - -// poll represents the current state of a network poll for a vertex -type poll struct { - votes ids.UniqueBag - polled ids.ShortSet - alpha int -} - -// Vote registers a vote for this poll -func (p *poll) Vote(votes []ids.ID, vdr ids.ShortID) { - if p.polled.Contains(vdr) { - p.polled.Remove(vdr) - p.votes.Add(uint(p.polled.Len()), votes...) - } -} - -// Finished returns true if the poll has completed, with no more required -// responses -func (p poll) Finished() bool { - // If there are no outstanding queries, the poll is finished - numPending := p.polled.Len() - if numPending == 0 { - return true - } - // If there are still enough pending responses to include another vertex, - // then the poll must wait for more responses - if numPending > p.alpha { - return false - } - - // Ignore any vertex that has already received alpha votes. To safely skip - // DAG traversal, assume that all votes for vertices with less than alpha - // votes will be applied to a single shared ancestor. In this case, the poll - // can terminate early, iff there are not enough pending votes for this - // ancestor to receive alpha votes. - partialVotes := ids.BitSet(0) - for _, vote := range p.votes.List() { - if voters := p.votes.GetSet(vote); voters.Len() < p.alpha { - partialVotes.Union(voters) - } - } - return partialVotes.Len()+numPending < p.alpha -} -func (p poll) String() string { return fmt.Sprintf("Waiting on %d chits", p.polled.Len()) } diff --git a/snow/engine/avalanche/polls_test.go b/snow/engine/avalanche/polls_test.go deleted file mode 100644 index cbb1ea4..0000000 --- a/snow/engine/avalanche/polls_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package avalanche - -import ( - "testing" - - "github.com/ava-labs/gecko/ids" -) - -func TestPollTerminatesEarlyVirtuousCase(t *testing.T) { - alpha := 3 - - vtxID := GenerateID() - votes := []ids.ID{vtxID} - - vdr1 := ids.NewShortID([20]byte{1}) - vdr2 := ids.NewShortID([20]byte{2}) - vdr3 := ids.NewShortID([20]byte{3}) - vdr4 := ids.NewShortID([20]byte{4}) - vdr5 := ids.NewShortID([20]byte{5}) // k = 5 - - vdrs := ids.ShortSet{} - vdrs.Add(vdr1) - vdrs.Add(vdr2) - vdrs.Add(vdr3) - vdrs.Add(vdr4) - vdrs.Add(vdr5) - - poll := poll{ - votes: make(ids.UniqueBag), - polled: vdrs, - alpha: alpha, - } - - poll.Vote(votes, vdr1) - if poll.Finished() { - t.Fatalf("Poll finished after less than alpha votes") - } - poll.Vote(votes, vdr2) - if poll.Finished() { - t.Fatalf("Poll finished after less than alpha votes") - } - poll.Vote(votes, vdr3) - if !poll.Finished() { - t.Fatalf("Poll did not terminate early after receiving alpha votes for one vertex and none for other vertices") - } -} - -func TestPollAccountsForSharedAncestor(t *testing.T) { - alpha := 4 - - vtxA := GenerateID() - vtxB := GenerateID() - vtxC := GenerateID() - vtxD := GenerateID() - - // If validators 1-3 vote for frontier vertices - // B, C, and D respectively, which all share the common ancestor - // A, then we cannot terminate early with alpha = k = 4 - // If the final vote is cast for any of A, B, C, or D, then - // vertex A will have transitively received alpha = 4 votes - vdr1 := ids.NewShortID([20]byte{1}) - vdr2 := ids.NewShortID([20]byte{2}) - vdr3 := ids.NewShortID([20]byte{3}) - vdr4 := ids.NewShortID([20]byte{4}) - - vdrs := ids.ShortSet{} - vdrs.Add(vdr1) - vdrs.Add(vdr2) - vdrs.Add(vdr3) - vdrs.Add(vdr4) - - poll := poll{ - votes: make(ids.UniqueBag), - polled: vdrs, - alpha: alpha, - } - - votes1 := []ids.ID{vtxB} - poll.Vote(votes1, vdr1) - if poll.Finished() { - t.Fatalf("Poll finished early after receiving one vote") - } - votes2 := []ids.ID{vtxC} - poll.Vote(votes2, vdr2) - if poll.Finished() { - t.Fatalf("Poll finished early after receiving two votes") - } - votes3 := []ids.ID{vtxD} - poll.Vote(votes3, vdr3) - if poll.Finished() { - t.Fatalf("Poll terminated early, when a shared ancestor could have received alpha votes") - } - - votes4 := []ids.ID{vtxA} - poll.Vote(votes4, vdr4) - if !poll.Finished() { - t.Fatalf("Poll did not terminate after receiving all outstanding votes") - } -} diff --git a/snow/engine/avalanche/transitive.go b/snow/engine/avalanche/transitive.go index 565267b..7412276 100644 --- a/snow/engine/avalanche/transitive.go +++ b/snow/engine/avalanche/transitive.go @@ -12,6 +12,7 @@ import ( "github.com/ava-labs/gecko/snow/choices" "github.com/ava-labs/gecko/snow/consensus/avalanche" "github.com/ava-labs/gecko/snow/consensus/snowstorm" + "github.com/ava-labs/gecko/snow/engine/avalanche/poll" "github.com/ava-labs/gecko/snow/engine/common" "github.com/ava-labs/gecko/snow/events" "github.com/ava-labs/gecko/utils/formatting" @@ -31,7 +32,7 @@ type Transitive struct { Config bootstrapper - polls polls // track people I have asked for their preference + polls poll.Set // track people I have asked for their preference // vtxReqs prevents asking validators for the same vertex vtxReqs common.Requests @@ -57,7 +58,12 @@ func (t *Transitive) Initialize(config Config) error { t.onFinished = t.finishBootstrapping - t.polls = newPolls(int(config.Params.Alpha), config.Context.Log, t.numPolls) + factory := poll.NewEarlyTermNoTraversalFactory(int(config.Params.Alpha)) + t.polls = poll.NewSet(factory, + config.Context.Log, + config.Params.Namespace, + config.Params.Metrics, + ) return t.bootstrapper.Initialize(config.BootstrapConfig) } @@ -309,7 +315,7 @@ func (t *Transitive) Notify(msg common.Message) error { } func (t *Transitive) repoll() error { - if len(t.polls.m) >= t.Params.ConcurrentRepolls || t.errs.Errored() { + if t.polls.Len() >= t.Params.ConcurrentRepolls || t.errs.Errored() { return nil } @@ -318,7 +324,7 @@ func (t *Transitive) repoll() error { return err } - for i := len(t.polls.m); i < t.Params.ConcurrentRepolls; i++ { + for i := t.polls.Len(); i < t.Params.ConcurrentRepolls; i++ { if err := t.batch(nil, false /*=force*/, true /*=empty*/); err != nil { return err } diff --git a/snow/engine/snowman/poll/early_term_no_traversal.go b/snow/engine/snowman/poll/early_term_no_traversal.go new file mode 100644 index 0000000..8042b27 --- /dev/null +++ b/snow/engine/snowman/poll/early_term_no_traversal.go @@ -0,0 +1,73 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + + "github.com/ava-labs/gecko/ids" +) + +type earlyTermNoTraversalFactory struct { + alpha int +} + +// NewEarlyTermNoTraversalFactory returns a factory that returns polls with +// early termination, without doing DAG traversals +func NewEarlyTermNoTraversalFactory(alpha int) Factory { + return &earlyTermNoTraversalFactory{alpha: alpha} +} + +func (f *earlyTermNoTraversalFactory) New(vdrs ids.ShortSet) Poll { + return &earlyTermNoTraversalPoll{ + polled: vdrs, + alpha: f.alpha, + } +} + +// earlyTermNoTraversalPoll finishes when any remaining validators can't change +// the result of the poll. However, does not terminate tightly with this bound. +// It terminates as quickly as it can without performing any DAG traversals. +type earlyTermNoTraversalPoll struct { + votes ids.Bag + polled ids.ShortSet + alpha int +} + +// Vote registers a response for this poll +func (p *earlyTermNoTraversalPoll) Vote(vdr ids.ShortID, vote ids.ID) { + if !p.polled.Contains(vdr) { + // if the validator wasn't polled or already responded to this poll, we + // should just drop the vote + return + } + + // make sure that a validator can't respond multiple times + p.polled.Remove(vdr) + + // track the votes the validator responded with + p.votes.Add(vote) +} + +// Drop any future response for this poll +func (p *earlyTermNoTraversalPoll) Drop(vdr ids.ShortID) { + p.polled.Remove(vdr) +} + +// Finished returns true when all validators have voted +func (p *earlyTermNoTraversalPoll) Finished() bool { + remaining := p.polled.Len() + received := p.votes.Len() + _, freq := p.votes.Mode() + return remaining == 0 || // All k nodes responded + freq >= p.alpha || // An alpha majority has returned + received+remaining < p.alpha // An alpha majority can never return +} + +// Result returns the result of this poll +func (p *earlyTermNoTraversalPoll) Result() ids.Bag { return p.votes } + +func (p *earlyTermNoTraversalPoll) String() string { + return fmt.Sprintf("waiting on %s", p.polled) +} diff --git a/snow/engine/snowman/poll/early_term_no_traversal_test.go b/snow/engine/snowman/poll/early_term_no_traversal_test.go new file mode 100644 index 0000000..dd444e9 --- /dev/null +++ b/snow/engine/snowman/poll/early_term_no_traversal_test.go @@ -0,0 +1,205 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "testing" + + "github.com/ava-labs/gecko/ids" +) + +func TestEarlyTermNoTraversalResults(t *testing.T) { + alpha := 1 + + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) // k = 1 + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } + + result := poll.Result() + if list := result.List(); len(list) != 1 { + t.Fatalf("Wrong number of vertices returned") + } else if retVtxID := list[0]; !retVtxID.Equals(vtxID) { + t.Fatalf("Wrong vertex returned") + } else if result.Count(vtxID) != 1 { + t.Fatalf("Wrong number of votes returned") + } +} + +func TestEarlyTermNoTraversalString(t *testing.T) { + alpha := 2 + + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + + expected := "waiting on {BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp}" + if result := poll.String(); expected != result { + t.Fatalf("Poll should have returned %s but returned %s", expected, result) + } +} + +func TestEarlyTermNoTraversalDropsDuplicatedVotes(t *testing.T) { + alpha := 2 + + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr1, vtxID) + if poll.Finished() { + t.Fatalf("Poll finished after getting a duplicated vote") + } + poll.Vote(vdr2, vtxID) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } +} + +func TestEarlyTermNoTraversalTerminatesEarly(t *testing.T) { + alpha := 3 + + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) + vdr3 := ids.NewShortID([20]byte{3}) + vdr4 := ids.NewShortID([20]byte{4}) + vdr5 := ids.NewShortID([20]byte{5}) // k = 5 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + vdr3, + vdr4, + vdr5, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr2, vtxID) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr3, vtxID) + if !poll.Finished() { + t.Fatalf("Poll did not terminate early after receiving alpha votes for one vertex and none for other vertices") + } +} + +func TestEarlyTermNoTraversalForSharedAncestor(t *testing.T) { + alpha := 4 + + vtxA := ids.NewID([32]byte{1}) + vtxB := ids.NewID([32]byte{2}) + vtxC := ids.NewID([32]byte{3}) + vtxD := ids.NewID([32]byte{4}) + + // If validators 1-3 vote for frontier vertices + // B, C, and D respectively, which all share the common ancestor + // A, then we cannot terminate early with alpha = k = 4 + // If the final vote is cast for any of A, B, C, or D, then + // vertex A will have transitively received alpha = 4 votes + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) + vdr3 := ids.NewShortID([20]byte{3}) + vdr4 := ids.NewShortID([20]byte{4}) + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + vdr3, + vdr4, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxB) + if poll.Finished() { + t.Fatalf("Poll finished early after receiving one vote") + } + poll.Vote(vdr2, vtxC) + if poll.Finished() { + t.Fatalf("Poll finished early after receiving two votes") + } + poll.Vote(vdr3, vtxD) + if poll.Finished() { + t.Fatalf("Poll terminated early, when a shared ancestor could have received alpha votes") + } + poll.Vote(vdr4, vtxA) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving all outstanding votes") + } +} + +func TestEarlyTermNoTraversalWithFastDrops(t *testing.T) { + alpha := 2 + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) + vdr3 := ids.NewShortID([20]byte{3}) // k = 3 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + vdr3, + ) + + factory := NewEarlyTermNoTraversalFactory(alpha) + poll := factory.New(vdrs) + + poll.Drop(vdr1) + if poll.Finished() { + t.Fatalf("Poll finished early after dropping one vote") + } + poll.Drop(vdr2) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after dropping two votes") + } +} diff --git a/snow/engine/snowman/poll/interfaces.go b/snow/engine/snowman/poll/interfaces.go new file mode 100644 index 0000000..33731ad --- /dev/null +++ b/snow/engine/snowman/poll/interfaces.go @@ -0,0 +1,35 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + + "github.com/ava-labs/gecko/ids" +) + +// Set is a collection of polls +type Set interface { + fmt.Stringer + + Add(requestID uint32, vdrs ids.ShortSet) bool + Vote(requestID uint32, vdr ids.ShortID, vote ids.ID) (ids.Bag, bool) + Drop(requestID uint32, vdr ids.ShortID) (ids.Bag, bool) + Len() int +} + +// Poll is an outstanding poll +type Poll interface { + fmt.Stringer + + Vote(vdr ids.ShortID, vote ids.ID) + Drop(vdr ids.ShortID) + Finished() bool + Result() ids.Bag +} + +// Factory creates a new Poll +type Factory interface { + New(vdrs ids.ShortSet) Poll +} diff --git a/snow/engine/snowman/poll/no_early_term.go b/snow/engine/snowman/poll/no_early_term.go new file mode 100644 index 0000000..3bcaf38 --- /dev/null +++ b/snow/engine/snowman/poll/no_early_term.go @@ -0,0 +1,55 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + + "github.com/ava-labs/gecko/ids" +) + +type noEarlyTermFactory struct{} + +// NewNoEarlyTermFactory returns a factory that returns polls with no early +// termination +func NewNoEarlyTermFactory() Factory { return noEarlyTermFactory{} } + +func (noEarlyTermFactory) New(vdrs ids.ShortSet) Poll { + return &noEarlyTermPoll{polled: vdrs} +} + +// noEarlyTermPoll finishes when all polled validators either respond to the +// query or a timeout occurs +type noEarlyTermPoll struct { + votes ids.Bag + polled ids.ShortSet +} + +// Vote registers a response for this poll +func (p *noEarlyTermPoll) Vote(vdr ids.ShortID, vote ids.ID) { + if !p.polled.Contains(vdr) { + // if the validator wasn't polled or already responded to this poll, we + // should just drop the vote + return + } + + // make sure that a validator can't respond multiple times + p.polled.Remove(vdr) + + // track the votes the validator responded with + p.votes.Add(vote) +} + +// Drop any future response for this poll +func (p *noEarlyTermPoll) Drop(vdr ids.ShortID) { p.polled.Remove(vdr) } + +// Finished returns true when all validators have voted +func (p *noEarlyTermPoll) Finished() bool { return p.polled.Len() == 0 } + +// Result returns the result of this poll +func (p *noEarlyTermPoll) Result() ids.Bag { return p.votes } + +func (p *noEarlyTermPoll) String() string { + return fmt.Sprintf("waiting on %s", p.polled) +} diff --git a/snow/engine/snowman/poll/no_early_term_test.go b/snow/engine/snowman/poll/no_early_term_test.go new file mode 100644 index 0000000..a366b5e --- /dev/null +++ b/snow/engine/snowman/poll/no_early_term_test.go @@ -0,0 +1,92 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "testing" + + "github.com/ava-labs/gecko/ids" +) + +func TestNoEarlyTermResults(t *testing.T) { + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) // k = 1 + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + + factory := NewNoEarlyTermFactory() + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } + + result := poll.Result() + if list := result.List(); len(list) != 1 { + t.Fatalf("Wrong number of vertices returned") + } else if retVtxID := list[0]; !retVtxID.Equals(vtxID) { + t.Fatalf("Wrong vertex returned") + } else if result.Count(vtxID) != 1 { + t.Fatalf("Wrong number of votes returned") + } +} + +func TestNoEarlyTermString(t *testing.T) { + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewNoEarlyTermFactory() + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + + expected := "waiting on {BaMPFdqMUQ46BV8iRcwbVfsam55kMqcp}" + if result := poll.String(); expected != result { + t.Fatalf("Poll should have returned %s but returned %s", expected, result) + } +} + +func TestNoEarlyTermDropsDuplicatedVotes(t *testing.T) { + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + factory := NewNoEarlyTermFactory() + poll := factory.New(vdrs) + + poll.Vote(vdr1, vtxID) + if poll.Finished() { + t.Fatalf("Poll finished after less than alpha votes") + } + poll.Vote(vdr1, vtxID) + if poll.Finished() { + t.Fatalf("Poll finished after getting a duplicated vote") + } + poll.Drop(vdr1) + if poll.Finished() { + t.Fatalf("Poll finished after getting a duplicated vote") + } + poll.Vote(vdr2, vtxID) + if !poll.Finished() { + t.Fatalf("Poll did not terminate after receiving k votes") + } +} diff --git a/snow/engine/snowman/poll/set.go b/snow/engine/snowman/poll/set.go new file mode 100644 index 0000000..cded824 --- /dev/null +++ b/snow/engine/snowman/poll/set.go @@ -0,0 +1,134 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/ava-labs/gecko/ids" + "github.com/ava-labs/gecko/utils/logging" +) + +type set struct { + log logging.Logger + numPolls prometheus.Gauge + factory Factory + polls map[uint32]Poll +} + +// NewSet returns a new empty set of polls +func NewSet( + factory Factory, + log logging.Logger, + namespace string, + registerer prometheus.Registerer, +) Set { + numPolls := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "polls", + Help: "Number of pending network polls", + }) + if err := registerer.Register(numPolls); err != nil { + log.Error("failed to register av_polls statistics due to %s", err) + } + + return &set{ + log: log, + numPolls: numPolls, + factory: factory, + polls: make(map[uint32]Poll), + } +} + +// Add to the current set of polls +// Returns true if the poll was registered correctly and the network sample +// should be made. +func (s *set) Add(requestID uint32, vdrs ids.ShortSet) bool { + if _, exists := s.polls[requestID]; exists { + s.log.Debug("dropping poll due to duplicated requestID: %d", requestID) + return false + } + + s.log.Verbo("creating poll with requestID %d and validators %s", + requestID, + vdrs) + + s.polls[requestID] = s.factory.New(vdrs) // create the new poll + s.numPolls.Inc() // increase the metrics + return true +} + +// Vote registers the connections response to a query for [id]. If there was no +// query, or the response has already be registered, nothing is performed. +func (s *set) Vote( + requestID uint32, + vdr ids.ShortID, + vote ids.ID, +) (ids.Bag, bool) { + poll, exists := s.polls[requestID] + if !exists { + s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d", + vdr, + requestID) + return ids.Bag{}, false + } + + s.log.Verbo("processing vote from %s in the poll with requestID: %d with the vote %s", + vdr, + requestID, + vote) + + poll.Vote(vdr, vote) + if !poll.Finished() { + return ids.Bag{}, false + } + + s.log.Verbo("poll with requestID %d finished as %s", requestID, poll) + + delete(s.polls, requestID) // remove the poll from the current set + s.numPolls.Dec() // decrease the metrics + return poll.Result(), true +} + +// Drop registers the connections response to a query for [id]. If there was no +// query, or the response has already be registered, nothing is performed. +func (s *set) Drop(requestID uint32, vdr ids.ShortID) (ids.Bag, bool) { + poll, exists := s.polls[requestID] + if !exists { + s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d", + vdr, + requestID) + return ids.Bag{}, false + } + + s.log.Verbo("processing dropped vote from %s in the poll with requestID: %d", + vdr, + requestID) + + poll.Drop(vdr) + if !poll.Finished() { + return ids.Bag{}, false + } + + s.log.Verbo("poll with requestID %d finished as %s", requestID, poll) + + delete(s.polls, requestID) // remove the poll from the current set + s.numPolls.Dec() // decrease the metrics + return poll.Result(), true +} + +// Len returns the number of outstanding polls +func (s *set) Len() int { return len(s.polls) } + +func (s *set) String() string { + sb := strings.Builder{} + sb.WriteString(fmt.Sprintf("current polls: (Size = %d)", len(s.polls))) + for requestID, poll := range s.polls { + sb.WriteString(fmt.Sprintf("\n %d: %s", requestID, poll)) + } + return sb.String() +} diff --git a/snow/engine/snowman/poll/set_test.go b/snow/engine/snowman/poll/set_test.go new file mode 100644 index 0000000..316aaf6 --- /dev/null +++ b/snow/engine/snowman/poll/set_test.go @@ -0,0 +1,132 @@ +// (c) 2019-2020, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package poll + +import ( + "testing" + + "github.com/ava-labs/gecko/ids" + "github.com/ava-labs/gecko/utils/logging" + "github.com/prometheus/client_golang/prometheus" +) + +func TestNewSetErrorOnMetrics(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + + registerer.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "polls", + })) + + _ = NewSet(factory, log, namespace, registerer) +} + +func TestCreateAndFinishSuccessfulPoll(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + s := NewSet(factory, log, namespace, registerer) + + vtxID := ids.NewID([32]byte{1}) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + if s.Len() != 0 { + t.Fatalf("Shouldn't have any active polls yet") + } else if !s.Add(0, vdrs) { + t.Fatalf("Should have been able to add a new poll") + } else if s.Len() != 1 { + t.Fatalf("Should only have one active poll") + } else if s.Add(0, vdrs) { + t.Fatalf("Shouldn't have been able to add a duplicated poll") + } else if s.Len() != 1 { + t.Fatalf("Should only have one active poll") + } else if _, finished := s.Vote(1, vdr1, vtxID); finished { + t.Fatalf("Shouldn't have been able to finish a non-existant poll") + } else if _, finished := s.Vote(0, vdr1, vtxID); finished { + t.Fatalf("Shouldn't have been able to finish an ongoing poll") + } else if _, finished := s.Vote(0, vdr1, vtxID); finished { + t.Fatalf("Should have dropped a duplicated poll") + } else if result, finished := s.Vote(0, vdr2, vtxID); !finished { + t.Fatalf("Should have finished the") + } else if list := result.List(); len(list) != 1 { + t.Fatalf("Wrong number of vertices returned") + } else if retVtxID := list[0]; !retVtxID.Equals(vtxID) { + t.Fatalf("Wrong vertex returned") + } else if result.Count(vtxID) != 2 { + t.Fatalf("Wrong number of votes returned") + } +} + +func TestCreateAndFinishFailedPoll(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + s := NewSet(factory, log, namespace, registerer) + + vdr1 := ids.NewShortID([20]byte{1}) + vdr2 := ids.NewShortID([20]byte{2}) // k = 2 + + vdrs := ids.ShortSet{} + vdrs.Add( + vdr1, + vdr2, + ) + + if s.Len() != 0 { + t.Fatalf("Shouldn't have any active polls yet") + } else if !s.Add(0, vdrs) { + t.Fatalf("Should have been able to add a new poll") + } else if s.Len() != 1 { + t.Fatalf("Should only have one active poll") + } else if s.Add(0, vdrs) { + t.Fatalf("Shouldn't have been able to add a duplicated poll") + } else if s.Len() != 1 { + t.Fatalf("Should only have one active poll") + } else if _, finished := s.Drop(1, vdr1); finished { + t.Fatalf("Shouldn't have been able to finish a non-existant poll") + } else if _, finished := s.Drop(0, vdr1); finished { + t.Fatalf("Shouldn't have been able to finish an ongoing poll") + } else if _, finished := s.Drop(0, vdr1); finished { + t.Fatalf("Should have dropped a duplicated poll") + } else if result, finished := s.Drop(0, vdr2); !finished { + t.Fatalf("Should have finished the") + } else if list := result.List(); len(list) != 0 { + t.Fatalf("Wrong number of vertices returned") + } +} + +func TestSetString(t *testing.T) { + factory := NewNoEarlyTermFactory() + log := logging.NoLog{} + namespace := "" + registerer := prometheus.NewRegistry() + s := NewSet(factory, log, namespace, registerer) + + vdr1 := ids.NewShortID([20]byte{1}) // k = 1 + + vdrs := ids.ShortSet{} + vdrs.Add(vdr1) + + expected := "current polls: (Size = 1)\n" + + " 0: waiting on {6HgC8KRBEhXYbF4riJyJFLSHt37UNuRt}" + if !s.Add(0, vdrs) { + t.Fatalf("Should have been able to add a new poll") + } else if str := s.String(); expected != str { + t.Fatalf("Set return wrong string, Expected:\n%s\nReturned:\n%s", + expected, + str) + } +} diff --git a/snow/engine/snowman/polls.go b/snow/engine/snowman/polls.go deleted file mode 100644 index 6765ff7..0000000 --- a/snow/engine/snowman/polls.go +++ /dev/null @@ -1,115 +0,0 @@ -// (c) 2019-2020, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package snowman - -import ( - "fmt" - "strings" - - "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/utils/logging" - "github.com/prometheus/client_golang/prometheus" -) - -type polls struct { - log logging.Logger - numPolls prometheus.Gauge - alpha int - m map[uint32]poll -} - -// Add to the current set of polls -// Returns true if the poll was registered correctly and the network sample -// should be made. -func (p *polls) Add(requestID uint32, vdrs ids.ShortSet) bool { - poll, exists := p.m[requestID] - if !exists { - poll.alpha = p.alpha - poll.polled = vdrs - p.m[requestID] = poll - - p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics - } - return !exists -} - -// Vote registers the connections response to a query for [id]. If there was no -// query, or the response has already be registered, nothing is performed. -func (p *polls) Vote(requestID uint32, vdr ids.ShortID, vote ids.ID) (ids.Bag, bool) { - p.log.Verbo("[polls.Vote] Vote: requestID: %d. validatorID: %s. Vote: %s", requestID, vdr, vote) - poll, exists := p.m[requestID] - if !exists { - return ids.Bag{}, false - } - poll.Vote(vote, vdr) - if poll.Finished() { - delete(p.m, requestID) - p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics - return poll.votes, true - } - p.m[requestID] = poll - return ids.Bag{}, false -} - -// CancelVote registers the connections failure to respond to a query for [id]. -func (p *polls) CancelVote(requestID uint32, vdr ids.ShortID) (ids.Bag, bool) { - p.log.Verbo("CancelVote received. requestID: %d. validatorID: %s. Vote: %s", requestID, vdr) - poll, exists := p.m[requestID] - if !exists { - return ids.Bag{}, false - } - - poll.CancelVote(vdr) - if poll.Finished() { - delete(p.m, requestID) - p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics - return poll.votes, true - } - p.m[requestID] = poll - return ids.Bag{}, false -} - -func (p *polls) String() string { - sb := strings.Builder{} - - sb.WriteString(fmt.Sprintf("Current polls: (Size = %d)", len(p.m))) - for requestID, poll := range p.m { - sb.WriteString(fmt.Sprintf("\n %d: %s", requestID, poll)) - } - - return sb.String() -} - -// poll represents the current state of a network poll for a block -type poll struct { - alpha int - votes ids.Bag - polled ids.ShortSet -} - -// Vote registers a vote for this poll -func (p *poll) CancelVote(vdr ids.ShortID) { p.polled.Remove(vdr) } - -// Vote registers a vote for this poll -func (p *poll) Vote(vote ids.ID, vdr ids.ShortID) { - if p.polled.Contains(vdr) { - p.polled.Remove(vdr) - p.votes.Add(vote) - } -} - -// Finished returns true if the poll has completed, with no more required -// responses -func (p poll) Finished() bool { - remaining := p.polled.Len() - received := p.votes.Len() - _, freq := p.votes.Mode() - return remaining == 0 || // All k nodes responded - freq >= p.alpha || // An alpha majority has returned - received+remaining < p.alpha // An alpha majority can never return -} - -func (p poll) String() string { - return fmt.Sprintf("Waiting on %d chits from %s", p.polled.Len(), p.polled) -} diff --git a/snow/engine/snowman/transitive.go b/snow/engine/snowman/transitive.go index 0a89dc4..ab4a881 100644 --- a/snow/engine/snowman/transitive.go +++ b/snow/engine/snowman/transitive.go @@ -12,6 +12,7 @@ import ( "github.com/ava-labs/gecko/snow/choices" "github.com/ava-labs/gecko/snow/consensus/snowman" "github.com/ava-labs/gecko/snow/engine/common" + "github.com/ava-labs/gecko/snow/engine/snowman/poll" "github.com/ava-labs/gecko/snow/events" "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/utils/wrappers" @@ -30,7 +31,7 @@ type Transitive struct { bootstrapper // track outstanding preference requests - polls polls + polls poll.Set // blocks that have outstanding get requests blkReqs common.Requests @@ -64,10 +65,12 @@ func (t *Transitive) Initialize(config Config) error { t.onFinished = t.finishBootstrapping - t.polls.log = config.Context.Log - t.polls.numPolls = t.numPolls - t.polls.alpha = t.Params.Alpha - t.polls.m = make(map[uint32]poll) + factory := poll.NewEarlyTermNoTraversalFactory(int(config.Params.Alpha)) + t.polls = poll.NewSet(factory, + config.Context.Log, + config.Params.Namespace, + config.Params.Metrics, + ) return t.bootstrapper.Initialize(config.BootstrapConfig) } @@ -409,7 +412,7 @@ func (t *Transitive) repoll() { // propagate the most likely branch as quickly as possible prefID := t.Consensus.Preference() - for i := len(t.polls.m); i < t.Params.ConcurrentRepolls; i++ { + for i := t.polls.Len(); i < t.Params.ConcurrentRepolls; i++ { t.pullSample(prefID) } } diff --git a/snow/engine/snowman/transitive_test.go b/snow/engine/snowman/transitive_test.go index 8c5f9d1..e9571af 100644 --- a/snow/engine/snowman/transitive_test.go +++ b/snow/engine/snowman/transitive_test.go @@ -810,13 +810,13 @@ func TestVoteCanceling(t *testing.T) { te.insert(blk) - if len(te.polls.m) != 1 { + if te.polls.Len() != 1 { t.Fatalf("Shouldn't have finished blocking issue") } te.QueryFailed(vdr0.ID(), *queryRequestID) - if len(te.polls.m) != 1 { + if te.polls.Len() != 1 { t.Fatalf("Shouldn't have finished blocking issue") } diff --git a/snow/engine/snowman/voter.go b/snow/engine/snowman/voter.go index bd15831..5a3ca87 100644 --- a/snow/engine/snowman/voter.go +++ b/snow/engine/snowman/voter.go @@ -32,7 +32,7 @@ func (v *voter) Update() { results := ids.Bag{} finished := false if v.response.IsZero() { - results, finished = v.t.polls.CancelVote(v.requestID, v.vdr) + results, finished = v.t.polls.Drop(v.requestID, v.vdr) } else { results, finished = v.t.polls.Vote(v.requestID, v.vdr, v.response) } From 7a2a7f0add5921065e4fac8f8631699513efc21c Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 19:44:35 -0400 Subject: [PATCH 48/80] fixed polls metrics --- snow/engine/avalanche/metrics.go | 11 +---------- snow/engine/avalanche/poll/set.go | 2 +- snow/engine/snowman/metrics.go | 11 +---------- snow/engine/snowman/poll/set.go | 2 +- 4 files changed, 4 insertions(+), 22 deletions(-) diff --git a/snow/engine/avalanche/metrics.go b/snow/engine/avalanche/metrics.go index 021fe38..2cd3671 100644 --- a/snow/engine/avalanche/metrics.go +++ b/snow/engine/avalanche/metrics.go @@ -14,7 +14,7 @@ type metrics struct { numBSVtx, numBSDroppedVtx, numBSTx, numBSDroppedTx prometheus.Counter - numPolls, numVtxRequests, numTxRequests, numPendingVtx prometheus.Gauge + numVtxRequests, numTxRequests, numPendingVtx prometheus.Gauge } // Initialize implements the Engine interface @@ -61,12 +61,6 @@ func (m *metrics) Initialize(log logging.Logger, namespace string, registerer pr Name: "av_bs_dropped_txs", Help: "Number of dropped txs", }) - m.numPolls = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "av_polls", - Help: "Number of pending network polls", - }) m.numVtxRequests = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: namespace, @@ -107,9 +101,6 @@ func (m *metrics) Initialize(log logging.Logger, namespace string, registerer pr if err := registerer.Register(m.numBSDroppedTx); err != nil { log.Error("Failed to register av_bs_dropped_txs statistics due to %s", err) } - if err := registerer.Register(m.numPolls); err != nil { - log.Error("Failed to register av_polls statistics due to %s", err) - } if err := registerer.Register(m.numVtxRequests); err != nil { log.Error("Failed to register av_vtx_requests statistics due to %s", err) } diff --git a/snow/engine/avalanche/poll/set.go b/snow/engine/avalanche/poll/set.go index 34a8a1a..4107a38 100644 --- a/snow/engine/avalanche/poll/set.go +++ b/snow/engine/avalanche/poll/set.go @@ -33,7 +33,7 @@ func NewSet( Help: "Number of pending network polls", }) if err := registerer.Register(numPolls); err != nil { - log.Error("failed to register av_polls statistics due to %s", err) + log.Error("failed to register polls statistics due to %s", err) } return &set{ diff --git a/snow/engine/snowman/metrics.go b/snow/engine/snowman/metrics.go index f17d360..d71697a 100644 --- a/snow/engine/snowman/metrics.go +++ b/snow/engine/snowman/metrics.go @@ -13,7 +13,7 @@ type metrics struct { numPendingRequests, numBlocked prometheus.Gauge numBootstrapped, numDropped prometheus.Counter - numPolls, numBlkRequests, numBlockedBlk prometheus.Gauge + numBlkRequests, numBlockedBlk prometheus.Gauge } // Initialize implements the Engine interface @@ -42,12 +42,6 @@ func (m *metrics) Initialize(log logging.Logger, namespace string, registerer pr Name: "sm_bs_dropped", Help: "Number of dropped bootstrap blocks", }) - m.numPolls = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "sm_polls", - Help: "Number of pending network polls", - }) m.numBlkRequests = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: namespace, @@ -73,9 +67,6 @@ func (m *metrics) Initialize(log logging.Logger, namespace string, registerer pr if err := registerer.Register(m.numDropped); err != nil { log.Error("Failed to register sm_bs_dropped statistics due to %s", err) } - if err := registerer.Register(m.numPolls); err != nil { - log.Error("Failed to register sm_polls statistics due to %s", err) - } if err := registerer.Register(m.numBlkRequests); err != nil { log.Error("Failed to register sm_blk_requests statistics due to %s", err) } diff --git a/snow/engine/snowman/poll/set.go b/snow/engine/snowman/poll/set.go index cded824..069d153 100644 --- a/snow/engine/snowman/poll/set.go +++ b/snow/engine/snowman/poll/set.go @@ -33,7 +33,7 @@ func NewSet( Help: "Number of pending network polls", }) if err := registerer.Register(numPolls); err != nil { - log.Error("failed to register av_polls statistics due to %s", err) + log.Error("failed to register polls statistics due to %s", err) } return &set{ From d2573be25fc2239b513fd108ea7a72d8501a623f Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 20:00:54 -0400 Subject: [PATCH 49/80] Added duration metrics --- snow/engine/avalanche/poll/set.go | 33 +++++++++++++++++++---- snow/engine/avalanche/poll/set_test.go | 3 +++ snow/engine/snowman/poll/set.go | 36 +++++++++++++++++++++----- snow/engine/snowman/poll/set_test.go | 3 +++ 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/snow/engine/avalanche/poll/set.go b/snow/engine/avalanche/poll/set.go index 4107a38..24c93fb 100644 --- a/snow/engine/avalanche/poll/set.go +++ b/snow/engine/avalanche/poll/set.go @@ -6,18 +6,26 @@ package poll import ( "fmt" "strings" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/logging" + "github.com/ava-labs/gecko/utils/timer" ) +type poll struct { + Poll + start time.Time +} + type set struct { log logging.Logger numPolls prometheus.Gauge + durPolls prometheus.Histogram factory Factory - polls map[uint32]Poll + polls map[uint32]poll } // NewSet returns a new empty set of polls @@ -36,11 +44,22 @@ func NewSet( log.Error("failed to register polls statistics due to %s", err) } + durPolls := prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "poll_duration", + Help: "Length of time the poll existed in milliseconds", + Buckets: timer.MillisecondsBuckets, + }) + if err := registerer.Register(durPolls); err != nil { + log.Error("failed to register poll_duration statistics due to %s", err) + } + return &set{ log: log, numPolls: numPolls, + durPolls: durPolls, factory: factory, - polls: make(map[uint32]Poll), + polls: make(map[uint32]poll), } } @@ -57,8 +76,11 @@ func (s *set) Add(requestID uint32, vdrs ids.ShortSet) bool { requestID, vdrs) - s.polls[requestID] = s.factory.New(vdrs) // create the new poll - s.numPolls.Inc() // increase the metrics + s.polls[requestID] = poll{ + Poll: s.factory.New(vdrs), // create the new poll + start: time.Now(), + } + s.numPolls.Inc() // increase the metrics return true } @@ -90,7 +112,8 @@ func (s *set) Vote( s.log.Verbo("poll with requestID %d finished as %s", requestID, poll) delete(s.polls, requestID) // remove the poll from the current set - s.numPolls.Dec() // decrease the metrics + s.durPolls.Observe(float64(time.Now().Sub(poll.start).Milliseconds())) + s.numPolls.Dec() // decrease the metrics return poll.Result(), true } diff --git a/snow/engine/avalanche/poll/set_test.go b/snow/engine/avalanche/poll/set_test.go index 496f993..0f0e38e 100644 --- a/snow/engine/avalanche/poll/set_test.go +++ b/snow/engine/avalanche/poll/set_test.go @@ -20,6 +20,9 @@ func TestNewSetErrorOnMetrics(t *testing.T) { registerer.Register(prometheus.NewCounter(prometheus.CounterOpts{ Name: "polls", })) + registerer.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "poll_duration", + })) _ = NewSet(factory, log, namespace, registerer) } diff --git a/snow/engine/snowman/poll/set.go b/snow/engine/snowman/poll/set.go index 069d153..25e0e68 100644 --- a/snow/engine/snowman/poll/set.go +++ b/snow/engine/snowman/poll/set.go @@ -6,18 +6,26 @@ package poll import ( "fmt" "strings" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/logging" + "github.com/ava-labs/gecko/utils/timer" ) +type poll struct { + Poll + start time.Time +} + type set struct { log logging.Logger numPolls prometheus.Gauge + durPolls prometheus.Histogram factory Factory - polls map[uint32]Poll + polls map[uint32]poll } // NewSet returns a new empty set of polls @@ -36,11 +44,22 @@ func NewSet( log.Error("failed to register polls statistics due to %s", err) } + durPolls := prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "poll_duration", + Help: "Length of time the poll existed in milliseconds", + Buckets: timer.MillisecondsBuckets, + }) + if err := registerer.Register(durPolls); err != nil { + log.Error("failed to register poll_duration statistics due to %s", err) + } + return &set{ log: log, numPolls: numPolls, + durPolls: durPolls, factory: factory, - polls: make(map[uint32]Poll), + polls: make(map[uint32]poll), } } @@ -57,8 +76,11 @@ func (s *set) Add(requestID uint32, vdrs ids.ShortSet) bool { requestID, vdrs) - s.polls[requestID] = s.factory.New(vdrs) // create the new poll - s.numPolls.Inc() // increase the metrics + s.polls[requestID] = poll{ + Poll: s.factory.New(vdrs), // create the new poll + start: time.Now(), + } + s.numPolls.Inc() // increase the metrics return true } @@ -90,7 +112,8 @@ func (s *set) Vote( s.log.Verbo("poll with requestID %d finished as %s", requestID, poll) delete(s.polls, requestID) // remove the poll from the current set - s.numPolls.Dec() // decrease the metrics + s.durPolls.Observe(float64(time.Now().Sub(poll.start).Milliseconds())) + s.numPolls.Dec() // decrease the metrics return poll.Result(), true } @@ -117,7 +140,8 @@ func (s *set) Drop(requestID uint32, vdr ids.ShortID) (ids.Bag, bool) { s.log.Verbo("poll with requestID %d finished as %s", requestID, poll) delete(s.polls, requestID) // remove the poll from the current set - s.numPolls.Dec() // decrease the metrics + s.durPolls.Observe(float64(time.Now().Sub(poll.start).Milliseconds())) + s.numPolls.Dec() // decrease the metrics return poll.Result(), true } diff --git a/snow/engine/snowman/poll/set_test.go b/snow/engine/snowman/poll/set_test.go index 316aaf6..2ccf0f3 100644 --- a/snow/engine/snowman/poll/set_test.go +++ b/snow/engine/snowman/poll/set_test.go @@ -20,6 +20,9 @@ func TestNewSetErrorOnMetrics(t *testing.T) { registerer.Register(prometheus.NewCounter(prometheus.CounterOpts{ Name: "polls", })) + registerer.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "poll_duration", + })) _ = NewSet(factory, log, namespace, registerer) } From 8865eabec73aa152c2fefb75cd5a13c7c1ff0f33 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 21:26:50 -0400 Subject: [PATCH 50/80] Added tests for error checking in snowman --- snow/consensus/snowman/block_test.go | 7 +- snow/consensus/snowman/consensus_test.go | 411 +++++++++++++++++------ 2 files changed, 313 insertions(+), 105 deletions(-) diff --git a/snow/consensus/snowman/block_test.go b/snow/consensus/snowman/block_test.go index e03e6c2..d7db116 100644 --- a/snow/consensus/snowman/block_test.go +++ b/snow/consensus/snowman/block_test.go @@ -17,6 +17,7 @@ type TestBlock struct { height int status choices.Status bytes []byte + err error } func (b *TestBlock) Parent() Block { return b.parent } @@ -27,16 +28,16 @@ func (b *TestBlock) Accept() error { return errors.New("Dis-agreement") } b.status = choices.Accepted - return nil + return b.err } func (b *TestBlock) Reject() error { if b.status.Decided() && b.status != choices.Rejected { return errors.New("Dis-agreement") } b.status = choices.Rejected - return nil + return b.err } -func (b *TestBlock) Verify() error { return nil } +func (b *TestBlock) Verify() error { return b.err } func (b *TestBlock) Bytes() []byte { return b.bytes } type sortBlocks []*TestBlock diff --git a/snow/consensus/snowman/consensus_test.go b/snow/consensus/snowman/consensus_test.go index f64b4d8..aa8836c 100644 --- a/snow/consensus/snowman/consensus_test.go +++ b/snow/consensus/snowman/consensus_test.go @@ -4,6 +4,7 @@ package snowman import ( + "errors" "math/rand" "testing" @@ -42,6 +43,10 @@ var ( MetricsProcessingErrorTest, MetricsAcceptedErrorTest, MetricsRejectedErrorTest, + ErrorOnInitialRejectionTest, + ErrorOnAcceptTest, + ErrorOnRejectSiblingTest, + ErrorOnTransitiveRejectionTest, RandomizedConsistencyTest, } ) @@ -101,7 +106,9 @@ func AddToTailTest(t *testing.T, factory Factory) { } // Adding to the previous preference will update the preference - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } if pref := sm.Preference(); !pref.Equals(block.id) { t.Fatalf("Wrong preference. Expected %s, got %s", block.id, pref) @@ -133,7 +140,9 @@ func AddToNonTailTest(t *testing.T, factory Factory) { } // Adding to the previous preference will update the preference - sm.Add(firstBlock) + if err := sm.Add(firstBlock); err != nil { + t.Fatal(err) + } if pref := sm.Preference(); !pref.Equals(firstBlock.id) { t.Fatalf("Wrong preference. Expected %s, got %s", firstBlock.id, pref) @@ -141,7 +150,9 @@ func AddToNonTailTest(t *testing.T, factory Factory) { // Adding to something other than the previous preference won't update the // preference - sm.Add(secondBlock) + if err := sm.Add(secondBlock); err != nil { + t.Fatal(err) + } if pref := sm.Preference(); !pref.Equals(firstBlock.id) { t.Fatalf("Wrong preference. Expected %s, got %s", firstBlock.id, pref) @@ -171,7 +182,9 @@ func AddToUnknownTest(t *testing.T, factory Factory) { // Adding a block with an unknown parent means the parent must have already // been rejected. Therefore the block should be immediately rejected - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } if pref := sm.Preference(); !pref.Equals(GenesisID) { t.Fatalf("Wrong preference. Expected %s, got %s", GenesisID, pref) @@ -269,7 +282,9 @@ func IssuedIssuedTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } if !sm.Issued(block) { t.Fatalf("Should have marked a pending block as having been issued") @@ -296,12 +311,15 @@ func RecordPollAcceptSingleBlockTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } votes := ids.Bag{} votes.Add(block.id) - - sm.RecordPoll(votes) + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } if pref := sm.Preference(); !pref.Equals(block.id) { t.Fatalf("Preference returned the wrong block") @@ -309,11 +327,9 @@ func RecordPollAcceptSingleBlockTest(t *testing.T, factory Factory) { t.Fatalf("Snowman instance finalized too soon") } else if status := block.Status(); status != choices.Processing { t.Fatalf("Block's status changed unexpectedly") - } - - sm.RecordPoll(votes) - - if pref := sm.Preference(); !pref.Equals(block.id) { + } else if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if pref := sm.Preference(); !pref.Equals(block.id) { t.Fatalf("Preference returned the wrong block") } else if !sm.Finalized() { t.Fatalf("Snowman instance didn't finalize") @@ -347,15 +363,19 @@ func RecordPollAcceptAndRejectTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(firstBlock) - sm.Add(secondBlock) + if err := sm.Add(firstBlock); err != nil { + t.Fatal(err) + } + if err := sm.Add(secondBlock); err != nil { + t.Fatal(err) + } votes := ids.Bag{} votes.Add(firstBlock.id) - sm.RecordPoll(votes) - - if pref := sm.Preference(); !pref.Equals(firstBlock.id) { + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if pref := sm.Preference(); !pref.Equals(firstBlock.id) { t.Fatalf("Preference returned the wrong block") } else if sm.Finalized() { t.Fatalf("Snowman instance finalized too soon") @@ -363,11 +383,9 @@ func RecordPollAcceptAndRejectTest(t *testing.T, factory Factory) { t.Fatalf("Block's status changed unexpectedly") } else if status := secondBlock.Status(); status != choices.Processing { t.Fatalf("Block's status changed unexpectedly") - } - - sm.RecordPoll(votes) - - if pref := sm.Preference(); !pref.Equals(firstBlock.id) { + } else if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if pref := sm.Preference(); !pref.Equals(firstBlock.id) { t.Fatalf("Preference returned the wrong block") } else if !sm.Finalized() { t.Fatalf("Snowman instance didn't finalize") @@ -394,9 +412,9 @@ func RecordPollWhenFinalizedTest(t *testing.T, factory Factory) { votes := ids.Bag{} votes.Add(GenesisID) - sm.RecordPoll(votes) - - if !sm.Finalized() { + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if !sm.Finalized() { t.Fatalf("Consensus should still be finalized") } else if pref := sm.Preference(); !GenesisID.Equals(pref) { t.Fatalf("Wrong preference listed") @@ -433,9 +451,15 @@ func RecordPollRejectTransitivelyTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block0) - sm.Add(block1) - sm.Add(block2) + if err := sm.Add(block0); err != nil { + t.Fatal(err) + } + if err := sm.Add(block1); err != nil { + t.Fatal(err) + } + if err := sm.Add(block2); err != nil { + t.Fatal(err) + } // Current graph structure: // G @@ -447,7 +471,9 @@ func RecordPollRejectTransitivelyTest(t *testing.T, factory Factory) { votes := ids.Bag{} votes.Add(block0.id) - sm.RecordPoll(votes) + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } // Current graph structure: // 0 @@ -457,9 +483,7 @@ func RecordPollRejectTransitivelyTest(t *testing.T, factory Factory) { t.Fatalf("Finalized too late") } else if pref := sm.Preference(); !block0.id.Equals(pref) { t.Fatalf("Wrong preference listed") - } - - if status := block0.Status(); status != choices.Accepted { + } else if status := block0.Status(); status != choices.Accepted { t.Fatalf("Wrong status returned") } else if status := block1.Status(); status != choices.Rejected { t.Fatalf("Wrong status returned") @@ -503,10 +527,18 @@ func RecordPollTransitivelyResetConfidenceTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block0) - sm.Add(block1) - sm.Add(block2) - sm.Add(block3) + if err := sm.Add(block0); err != nil { + t.Fatal(err) + } + if err := sm.Add(block1); err != nil { + t.Fatal(err) + } + if err := sm.Add(block2); err != nil { + t.Fatal(err) + } + if err := sm.Add(block3); err != nil { + t.Fatal(err) + } // Current graph structure: // G @@ -517,26 +549,24 @@ func RecordPollTransitivelyResetConfidenceTest(t *testing.T, factory Factory) { votesFor2 := ids.Bag{} votesFor2.Add(block2.id) - sm.RecordPoll(votesFor2) - - if sm.Finalized() { + if err := sm.RecordPoll(votesFor2); err != nil { + t.Fatal(err) + } else if sm.Finalized() { t.Fatalf("Finalized too early") } else if pref := sm.Preference(); !block2.id.Equals(pref) { t.Fatalf("Wrong preference listed") } emptyVotes := ids.Bag{} - sm.RecordPoll(emptyVotes) - - if sm.Finalized() { + if err := sm.RecordPoll(emptyVotes); err != nil { + t.Fatal(err) + } else if sm.Finalized() { t.Fatalf("Finalized too early") } else if pref := sm.Preference(); !block2.id.Equals(pref) { t.Fatalf("Wrong preference listed") - } - - sm.RecordPoll(votesFor2) - - if sm.Finalized() { + } else if err := sm.RecordPoll(votesFor2); err != nil { + t.Fatal(err) + } else if sm.Finalized() { t.Fatalf("Finalized too early") } else if pref := sm.Preference(); !block2.id.Equals(pref) { t.Fatalf("Wrong preference listed") @@ -544,23 +574,19 @@ func RecordPollTransitivelyResetConfidenceTest(t *testing.T, factory Factory) { votesFor3 := ids.Bag{} votesFor3.Add(block3.id) - sm.RecordPoll(votesFor3) - - if sm.Finalized() { + if err := sm.RecordPoll(votesFor3); err != nil { + t.Fatal(err) + } else if sm.Finalized() { t.Fatalf("Finalized too early") } else if pref := sm.Preference(); !block2.id.Equals(pref) { t.Fatalf("Wrong preference listed") - } - - sm.RecordPoll(votesFor3) - - if !sm.Finalized() { + } else if err := sm.RecordPoll(votesFor3); err != nil { + t.Fatal(err) + } else if !sm.Finalized() { t.Fatalf("Finalized too late") } else if pref := sm.Preference(); !block3.id.Equals(pref) { t.Fatalf("Wrong preference listed") - } - - if status := block0.Status(); status != choices.Rejected { + } else if status := block0.Status(); status != choices.Rejected { t.Fatalf("Wrong status returned") } else if status := block1.Status(); status != choices.Accepted { t.Fatalf("Wrong status returned") @@ -592,19 +618,23 @@ func RecordPollInvalidVoteTest(t *testing.T, factory Factory) { } unknownBlockID := ids.Empty.Prefix(2) - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } validVotes := ids.Bag{} validVotes.Add(block.id) - sm.RecordPoll(validVotes) + if err := sm.RecordPoll(validVotes); err != nil { + t.Fatal(err) + } invalidVotes := ids.Bag{} invalidVotes.Add(unknownBlockID) - sm.RecordPoll(invalidVotes) - - sm.RecordPoll(validVotes) - - if sm.Finalized() { + if err := sm.RecordPoll(invalidVotes); err != nil { + t.Fatal(err) + } else if err := sm.RecordPoll(validVotes); err != nil { + t.Fatal(err) + } else if sm.Finalized() { t.Fatalf("Finalized too early") } else if pref := sm.Preference(); !block.id.Equals(pref) { t.Fatalf("Wrong preference listed") @@ -651,11 +681,21 @@ func RecordPollTransitiveVotingTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block0) - sm.Add(block1) - sm.Add(block2) - sm.Add(block3) - sm.Add(block4) + if err := sm.Add(block0); err != nil { + t.Fatal(err) + } + if err := sm.Add(block1); err != nil { + t.Fatal(err) + } + if err := sm.Add(block2); err != nil { + t.Fatal(err) + } + if err := sm.Add(block3); err != nil { + t.Fatal(err) + } + if err := sm.Add(block4); err != nil { + t.Fatal(err) + } // Current graph structure: // G @@ -668,10 +708,14 @@ func RecordPollTransitiveVotingTest(t *testing.T, factory Factory) { // Tail = 2 votes0_2_4 := ids.Bag{} - votes0_2_4.Add(block0.id) - votes0_2_4.Add(block2.id) - votes0_2_4.Add(block4.id) - sm.RecordPoll(votes0_2_4) + votes0_2_4.Add( + block0.id, + block2.id, + block4.id, + ) + if err := sm.RecordPoll(votes0_2_4); err != nil { + t.Fatal(err) + } // Current graph structure: // 0 @@ -699,7 +743,9 @@ func RecordPollTransitiveVotingTest(t *testing.T, factory Factory) { dep2_2_2 := ids.Bag{} dep2_2_2.AddCount(block2.id, 3) - sm.RecordPoll(dep2_2_2) + if err := sm.RecordPoll(dep2_2_2); err != nil { + t.Fatal(err) + } // Current graph structure: // 2 @@ -757,20 +803,26 @@ func RecordPollDivergedVotingTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block0) - sm.Add(block1) + if err := sm.Add(block0); err != nil { + t.Fatal(err) + } + if err := sm.Add(block1); err != nil { + t.Fatal(err) + } votes0 := ids.Bag{} votes0.Add(block0.id) - sm.RecordPoll(votes0) - - sm.Add(block2) + if err := sm.RecordPoll(votes0); err != nil { + t.Fatal(err) + } else if err := sm.Add(block2); err != nil { + t.Fatal(err) + } // dep2 is already rejected. - sm.Add(block3) - - if status := block0.Status(); status == choices.Accepted { + if err := sm.Add(block3); err != nil { + t.Fatal(err) + } else if status := block0.Status(); status == choices.Accepted { t.Fatalf("Shouldn't be accepted yet") } @@ -778,9 +830,9 @@ func RecordPollDivergedVotingTest(t *testing.T, factory Factory) { // dep0. Because dep2 is already rejected, this will accept dep0. votes3 := ids.Bag{} votes3.Add(block3.id) - sm.RecordPoll(votes3) - - if !sm.Finalized() { + if err := sm.RecordPoll(votes3); err != nil { + t.Fatal(err) + } else if !sm.Finalized() { t.Fatalf("Finalized too late") } else if status := block0.Status(); status != choices.Accepted { t.Fatalf("Should be accepted") @@ -818,14 +870,15 @@ func MetricsProcessingErrorTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } votes := ids.Bag{} votes.Add(block.id) - - sm.RecordPoll(votes) - - if !sm.Finalized() { + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if !sm.Finalized() { t.Fatalf("Snowman instance didn't finalize") } } @@ -861,14 +914,15 @@ func MetricsAcceptedErrorTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } votes := ids.Bag{} votes.Add(block.id) - - sm.RecordPoll(votes) - - if !sm.Finalized() { + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if !sm.Finalized() { t.Fatalf("Snowman instance didn't finalize") } } @@ -904,18 +958,171 @@ func MetricsRejectedErrorTest(t *testing.T, factory Factory) { status: choices.Processing, } - sm.Add(block) + if err := sm.Add(block); err != nil { + t.Fatal(err) + } votes := ids.Bag{} votes.Add(block.id) - - sm.RecordPoll(votes) - - if !sm.Finalized() { + if err := sm.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if !sm.Finalized() { t.Fatalf("Snowman instance didn't finalize") } } +func ErrorOnInitialRejectionTest(t *testing.T, factory Factory) { + sm := factory.New() + + ctx := snow.DefaultContextTest() + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + } + + sm.Initialize(ctx, params, GenesisID) + + rejectedBlock := &TestBlock{ + id: ids.Empty.Prefix(1), + status: choices.Rejected, + } + + block := &TestBlock{ + parent: rejectedBlock, + id: ids.Empty.Prefix(2), + status: choices.Processing, + err: errors.New(""), + } + + if err := sm.Add(block); err == nil { + t.Fatalf("Should have errored on rejecting the rejectable block") + } +} + +func ErrorOnAcceptTest(t *testing.T, factory Factory) { + sm := factory.New() + + ctx := snow.DefaultContextTest() + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + } + + sm.Initialize(ctx, params, GenesisID) + + block := &TestBlock{ + parent: Genesis, + id: ids.Empty.Prefix(1), + status: choices.Processing, + err: errors.New(""), + } + + if err := sm.Add(block); err != nil { + t.Fatal(err) + } + + votes := ids.Bag{} + votes.Add(block.id) + if err := sm.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on accepted the block") + } +} + +func ErrorOnRejectSiblingTest(t *testing.T, factory Factory) { + sm := factory.New() + + ctx := snow.DefaultContextTest() + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + } + + sm.Initialize(ctx, params, GenesisID) + + block0 := &TestBlock{ + parent: Genesis, + id: ids.Empty.Prefix(1), + status: choices.Processing, + } + block1 := &TestBlock{ + parent: Genesis, + id: ids.Empty.Prefix(2), + status: choices.Processing, + err: errors.New(""), + } + + if err := sm.Add(block0); err != nil { + t.Fatal(err) + } else if err := sm.Add(block1); err != nil { + t.Fatal(err) + } + + votes := ids.Bag{} + votes.Add(block0.id) + if err := sm.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on rejecting the block's sibling") + } +} + +func ErrorOnTransitiveRejectionTest(t *testing.T, factory Factory) { + sm := factory.New() + + ctx := snow.DefaultContextTest() + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + } + + sm.Initialize(ctx, params, GenesisID) + + block0 := &TestBlock{ + parent: Genesis, + id: ids.Empty.Prefix(1), + status: choices.Processing, + } + block1 := &TestBlock{ + parent: Genesis, + id: ids.Empty.Prefix(2), + status: choices.Processing, + } + block2 := &TestBlock{ + parent: block1, + id: ids.Empty.Prefix(3), + status: choices.Processing, + err: errors.New(""), + } + + if err := sm.Add(block0); err != nil { + t.Fatal(err) + } else if err := sm.Add(block1); err != nil { + t.Fatal(err) + } else if err := sm.Add(block2); err != nil { + t.Fatal(err) + } + + votes := ids.Bag{} + votes.Add(block0.id) + if err := sm.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on transitively rejecting the block") + } +} + func RandomizedConsistencyTest(t *testing.T, factory Factory) { numColors := 50 numNodes := 100 From d923d5c0f9710e6a1f8dd621fab33f13b68f7f45 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 21:30:29 -0400 Subject: [PATCH 51/80] Cleaned up tests --- snow/consensus/snowman/consensus_test.go | 63 +++++++----------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/snow/consensus/snowman/consensus_test.go b/snow/consensus/snowman/consensus_test.go index aa8836c..3d3fe7f 100644 --- a/snow/consensus/snowman/consensus_test.go +++ b/snow/consensus/snowman/consensus_test.go @@ -76,11 +76,9 @@ func InitializeTest(t *testing.T, factory Factory) { if p := sm.Parameters(); p != params { t.Fatalf("Wrong returned parameters") - } - if pref := sm.Preference(); !pref.Equals(GenesisID) { + } else if pref := sm.Preference(); !pref.Equals(GenesisID) { t.Fatalf("Wrong preference returned") - } - if !sm.Finalized() { + } else if !sm.Finalized() { t.Fatalf("Wrong should have marked the instance as being finalized") } } @@ -108,9 +106,7 @@ func AddToTailTest(t *testing.T, factory Factory) { // Adding to the previous preference will update the preference if err := sm.Add(block); err != nil { t.Fatal(err) - } - - if pref := sm.Preference(); !pref.Equals(block.id) { + } else if pref := sm.Preference(); !pref.Equals(block.id) { t.Fatalf("Wrong preference. Expected %s, got %s", block.id, pref) } } @@ -142,9 +138,7 @@ func AddToNonTailTest(t *testing.T, factory Factory) { // Adding to the previous preference will update the preference if err := sm.Add(firstBlock); err != nil { t.Fatal(err) - } - - if pref := sm.Preference(); !pref.Equals(firstBlock.id) { + } else if pref := sm.Preference(); !pref.Equals(firstBlock.id) { t.Fatalf("Wrong preference. Expected %s, got %s", firstBlock.id, pref) } @@ -152,9 +146,7 @@ func AddToNonTailTest(t *testing.T, factory Factory) { // preference if err := sm.Add(secondBlock); err != nil { t.Fatal(err) - } - - if pref := sm.Preference(); !pref.Equals(firstBlock.id) { + } else if pref := sm.Preference(); !pref.Equals(firstBlock.id) { t.Fatalf("Wrong preference. Expected %s, got %s", firstBlock.id, pref) } } @@ -184,9 +176,7 @@ func AddToUnknownTest(t *testing.T, factory Factory) { // been rejected. Therefore the block should be immediately rejected if err := sm.Add(block); err != nil { t.Fatal(err) - } - - if pref := sm.Preference(); !pref.Equals(GenesisID) { + } else if pref := sm.Preference(); !pref.Equals(GenesisID) { t.Fatalf("Wrong preference. Expected %s, got %s", GenesisID, pref) } else if status := block.Status(); status != choices.Rejected { t.Fatalf("Should have rejected the block") @@ -284,9 +274,7 @@ func IssuedIssuedTest(t *testing.T, factory Factory) { if err := sm.Add(block); err != nil { t.Fatal(err) - } - - if !sm.Issued(block) { + } else if !sm.Issued(block) { t.Fatalf("Should have marked a pending block as having been issued") } } @@ -319,9 +307,7 @@ func RecordPollAcceptSingleBlockTest(t *testing.T, factory Factory) { votes.Add(block.id) if err := sm.RecordPoll(votes); err != nil { t.Fatal(err) - } - - if pref := sm.Preference(); !pref.Equals(block.id) { + } else if pref := sm.Preference(); !pref.Equals(block.id) { t.Fatalf("Preference returned the wrong block") } else if sm.Finalized() { t.Fatalf("Snowman instance finalized too soon") @@ -365,8 +351,7 @@ func RecordPollAcceptAndRejectTest(t *testing.T, factory Factory) { if err := sm.Add(firstBlock); err != nil { t.Fatal(err) - } - if err := sm.Add(secondBlock); err != nil { + } else if err := sm.Add(secondBlock); err != nil { t.Fatal(err) } @@ -453,11 +438,9 @@ func RecordPollRejectTransitivelyTest(t *testing.T, factory Factory) { if err := sm.Add(block0); err != nil { t.Fatal(err) - } - if err := sm.Add(block1); err != nil { + } else if err := sm.Add(block1); err != nil { t.Fatal(err) - } - if err := sm.Add(block2); err != nil { + } else if err := sm.Add(block2); err != nil { t.Fatal(err) } @@ -529,14 +512,11 @@ func RecordPollTransitivelyResetConfidenceTest(t *testing.T, factory Factory) { if err := sm.Add(block0); err != nil { t.Fatal(err) - } - if err := sm.Add(block1); err != nil { + } else if err := sm.Add(block1); err != nil { t.Fatal(err) - } - if err := sm.Add(block2); err != nil { + } else if err := sm.Add(block2); err != nil { t.Fatal(err) - } - if err := sm.Add(block3); err != nil { + } else if err := sm.Add(block3); err != nil { t.Fatal(err) } @@ -683,17 +663,13 @@ func RecordPollTransitiveVotingTest(t *testing.T, factory Factory) { if err := sm.Add(block0); err != nil { t.Fatal(err) - } - if err := sm.Add(block1); err != nil { + } else if err := sm.Add(block1); err != nil { t.Fatal(err) - } - if err := sm.Add(block2); err != nil { + } else if err := sm.Add(block2); err != nil { t.Fatal(err) - } - if err := sm.Add(block3); err != nil { + } else if err := sm.Add(block3); err != nil { t.Fatal(err) - } - if err := sm.Add(block4); err != nil { + } else if err := sm.Add(block4); err != nil { t.Fatal(err) } @@ -805,8 +781,7 @@ func RecordPollDivergedVotingTest(t *testing.T, factory Factory) { if err := sm.Add(block0); err != nil { t.Fatal(err) - } - if err := sm.Add(block1); err != nil { + } else if err := sm.Add(block1); err != nil { t.Fatal(err) } From fb7e4910001c51da11cfe15f026265bc1a7fb014 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 22:38:53 -0400 Subject: [PATCH 52/80] Set up tests for error handling in snowstorm --- snow/consensus/snowstorm/consensus_test.go | 496 ++++++++++++++------- snow/consensus/snowstorm/directed_test.go | 32 +- snow/consensus/snowstorm/input_test.go | 30 +- snow/consensus/snowstorm/test_tx.go | 4 +- 4 files changed, 351 insertions(+), 211 deletions(-) diff --git a/snow/consensus/snowstorm/consensus_test.go b/snow/consensus/snowstorm/consensus_test.go index 270292c..5ff46ab 100644 --- a/snow/consensus/snowstorm/consensus_test.go +++ b/snow/consensus/snowstorm/consensus_test.go @@ -4,6 +4,7 @@ package snowstorm import ( + "errors" "testing" "github.com/prometheus/client_golang/prometheus" @@ -19,6 +20,28 @@ var ( Green = &TestTx{Identifier: ids.Empty.Prefix(1)} Blue = &TestTx{Identifier: ids.Empty.Prefix(2)} Alpha = &TestTx{Identifier: ids.Empty.Prefix(3)} + + Tests = []func(*testing.T, Factory){ + MetricsTest, + ParamsTest, + IssuedTest, + LeftoverInputTest, + LowerConfidenceTest, + MiddleConfidenceTest, + IndependentTest, + VirtuousTest, + IsVirtuousTest, + QuiesceTest, + AcceptingDependencyTest, + RejectingDependencyTest, + VacuouslyAcceptedTest, + ConflictsTest, + VirtuousDependsOnRogueTest, + ErrorOnVacuouslyAcceptedTest, + ErrorOnAcceptedTest, + ErrorOnRejectingLowerConfidenceConflictTest, + ErrorOnRejectingHigherConfidenceConflictTest, + } ) // R - G - B - A @@ -46,6 +69,52 @@ func Setup() { Alpha.Reset() } +// Execute all tests against a consensus implementation +func ConsensusTest(t *testing.T, factory Factory, prefix string) { + for _, test := range Tests { + test(t, factory) + } + StringTest(t, factory, prefix) +} + +func MetricsTest(t *testing.T, factory Factory) { + Setup() + + { + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 2, + } + params.Metrics.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "tx_processing", + })) + graph := factory.New() + graph.Initialize(snow.DefaultContextTest(), params) + } + { + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 2, + } + params.Metrics.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "tx_accepted", + })) + graph := factory.New() + graph.Initialize(snow.DefaultContextTest(), params) + } + { + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 2, + } + params.Metrics.Register(prometheus.NewCounter(prometheus.CounterOpts{ + Name: "tx_rejected", + })) + graph := factory.New() + graph.Initialize(snow.DefaultContextTest(), params) + } +} + func ParamsTest(t *testing.T, factory Factory) { Setup() @@ -81,15 +150,13 @@ func IssuedTest(t *testing.T, factory Factory) { if issued := graph.Issued(Red); issued { t.Fatalf("Haven't issued anything yet.") - } - - graph.Add(Red) - - if issued := graph.Issued(Red); !issued { + } else if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if issued := graph.Issued(Red); !issued { t.Fatalf("Have already issued.") } - Blue.Accept() + _ = Blue.Accept() if issued := graph.Issued(Blue); !issued { t.Fatalf("Have already accepted.") @@ -106,10 +173,12 @@ func LeftoverInputTest(t *testing.T, factory Factory) { K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 1, } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Green) - if prefs := graph.Preferences(); prefs.Len() != 1 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 1 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s got %s", Red.ID(), prefs.List()[0]) @@ -120,15 +189,13 @@ func LeftoverInputTest(t *testing.T, factory Factory) { r := ids.Bag{} r.SetThreshold(2) r.AddCount(Red.ID(), 2) - graph.RecordPoll(r) - - if prefs := graph.Preferences(); prefs.Len() != 0 { + if err := graph.RecordPoll(r); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 0 { t.Fatalf("Wrong number of preferences.") } else if !graph.Finalized() { t.Fatalf("Finalized too late") - } - - if Red.Status() != choices.Accepted { + } else if Red.Status() != choices.Accepted { t.Fatalf("%s should have been accepted", Red.ID()) } else if Green.Status() != choices.Rejected { t.Fatalf("%s should have been rejected", Green.ID()) @@ -145,11 +212,14 @@ func LowerConfidenceTest(t *testing.T, factory Factory) { K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 1, } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Green) - graph.Add(Blue) - if prefs := graph.Preferences(); prefs.Len() != 1 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if err := graph.Add(Blue); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 1 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s got %s", Red.ID(), prefs.List()[0]) @@ -160,9 +230,9 @@ func LowerConfidenceTest(t *testing.T, factory Factory) { r := ids.Bag{} r.SetThreshold(2) r.AddCount(Red.ID(), 2) - graph.RecordPoll(r) - - if prefs := graph.Preferences(); prefs.Len() != 1 { + if err := graph.RecordPoll(r); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 1 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Blue.ID()) { t.Fatalf("Wrong preference. Expected %s", Blue.ID()) @@ -181,12 +251,16 @@ func MiddleConfidenceTest(t *testing.T, factory Factory) { K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 1, } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Green) - graph.Add(Alpha) - graph.Add(Blue) - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if err := graph.Add(Alpha); err != nil { + t.Fatal(err) + } else if err := graph.Add(Blue); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s", Red.ID()) @@ -199,9 +273,9 @@ func MiddleConfidenceTest(t *testing.T, factory Factory) { r := ids.Bag{} r.SetThreshold(2) r.AddCount(Red.ID(), 2) - graph.RecordPoll(r) - - if prefs := graph.Preferences(); prefs.Len() != 1 { + if err := graph.RecordPoll(r); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 1 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Alpha.ID()) { t.Fatalf("Wrong preference. Expected %s", Alpha.ID()) @@ -209,6 +283,7 @@ func MiddleConfidenceTest(t *testing.T, factory Factory) { t.Fatalf("Finalized too early") } } + func IndependentTest(t *testing.T, factory Factory) { Setup() @@ -219,10 +294,12 @@ func IndependentTest(t *testing.T, factory Factory) { K: 2, Alpha: 2, BetaVirtuous: 2, BetaRogue: 2, } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Alpha) - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Alpha); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s", Red.ID()) @@ -236,9 +313,9 @@ func IndependentTest(t *testing.T, factory Factory) { ra.SetThreshold(2) ra.AddCount(Red.ID(), 2) ra.AddCount(Alpha.ID(), 2) - graph.RecordPoll(ra) - - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.RecordPoll(ra); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s", Red.ID()) @@ -246,11 +323,9 @@ func IndependentTest(t *testing.T, factory Factory) { t.Fatalf("Wrong preference. Expected %s", Alpha.ID()) } else if graph.Finalized() { t.Fatalf("Finalized too early") - } - - graph.RecordPoll(ra) - - if prefs := graph.Preferences(); prefs.Len() != 0 { + } else if err := graph.RecordPoll(ra); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 0 { t.Fatalf("Wrong number of preferences.") } else if !graph.Finalized() { t.Fatalf("Finalized too late") @@ -267,35 +342,30 @@ func VirtuousTest(t *testing.T, factory Factory) { K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 1, } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - if virtuous := graph.Virtuous(); virtuous.Len() != 1 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if virtuous := graph.Virtuous(); virtuous.Len() != 1 { t.Fatalf("Wrong number of virtuous.") } else if !virtuous.Contains(Red.ID()) { t.Fatalf("Wrong virtuous. Expected %s", Red.ID()) - } - - graph.Add(Alpha) - - if virtuous := graph.Virtuous(); virtuous.Len() != 2 { + } else if err := graph.Add(Alpha); err != nil { + t.Fatal(err) + } else if virtuous := graph.Virtuous(); virtuous.Len() != 2 { t.Fatalf("Wrong number of virtuous.") } else if !virtuous.Contains(Red.ID()) { t.Fatalf("Wrong virtuous. Expected %s", Red.ID()) } else if !virtuous.Contains(Alpha.ID()) { t.Fatalf("Wrong virtuous. Expected %s", Alpha.ID()) - } - - graph.Add(Green) - - if virtuous := graph.Virtuous(); virtuous.Len() != 1 { + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if virtuous := graph.Virtuous(); virtuous.Len() != 1 { t.Fatalf("Wrong number of virtuous.") } else if !virtuous.Contains(Alpha.ID()) { t.Fatalf("Wrong virtuous. Expected %s", Alpha.ID()) - } - - graph.Add(Blue) - - if virtuous := graph.Virtuous(); virtuous.Len() != 0 { + } else if err := graph.Add(Blue); err != nil { + t.Fatal(err) + } else if virtuous := graph.Virtuous(); virtuous.Len() != 0 { t.Fatalf("Wrong number of virtuous.") } } @@ -319,11 +389,9 @@ func IsVirtuousTest(t *testing.T, factory Factory) { t.Fatalf("Should be virtuous") } else if !graph.IsVirtuous(Alpha) { t.Fatalf("Should be virtuous") - } - - graph.Add(Red) - - if !graph.IsVirtuous(Red) { + } else if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if !graph.IsVirtuous(Red) { t.Fatalf("Should be virtuous") } else if graph.IsVirtuous(Green) { t.Fatalf("Should not be virtuous") @@ -331,11 +399,9 @@ func IsVirtuousTest(t *testing.T, factory Factory) { t.Fatalf("Should be virtuous") } else if !graph.IsVirtuous(Alpha) { t.Fatalf("Should be virtuous") - } - - graph.Add(Green) - - if graph.IsVirtuous(Red) { + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if graph.IsVirtuous(Red) { t.Fatalf("Should not be virtuous") } else if graph.IsVirtuous(Green) { t.Fatalf("Should not be virtuous") @@ -357,17 +423,13 @@ func QuiesceTest(t *testing.T, factory Factory) { if !graph.Quiesce() { t.Fatalf("Should quiesce") - } - - graph.Add(Red) - - if graph.Quiesce() { + } else if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if graph.Quiesce() { t.Fatalf("Shouldn't quiesce") - } - - graph.Add(Green) - - if !graph.Quiesce() { + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if !graph.Quiesce() { t.Fatalf("Should quiesce") } } @@ -390,11 +452,13 @@ func AcceptingDependencyTest(t *testing.T, factory Factory) { } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Green) - graph.Add(purple) - - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if err := graph.Add(purple); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s", Red.ID()) @@ -410,10 +474,9 @@ func AcceptingDependencyTest(t *testing.T, factory Factory) { g := ids.Bag{} g.Add(Green.ID()) - - graph.RecordPoll(g) - - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.RecordPoll(g); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Green.ID()) { t.Fatalf("Wrong preference. Expected %s", Green.ID()) @@ -429,10 +492,9 @@ func AcceptingDependencyTest(t *testing.T, factory Factory) { rp := ids.Bag{} rp.Add(Red.ID(), purple.ID()) - - graph.RecordPoll(rp) - - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.RecordPoll(rp); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Green.ID()) { t.Fatalf("Wrong preference. Expected %s", Green.ID()) @@ -448,10 +510,9 @@ func AcceptingDependencyTest(t *testing.T, factory Factory) { r := ids.Bag{} r.Add(Red.ID()) - - graph.RecordPoll(r) - - if prefs := graph.Preferences(); prefs.Len() != 0 { + if err := graph.RecordPoll(r); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 0 { t.Fatalf("Wrong number of preferences.") } else if Red.Status() != choices.Accepted { t.Fatalf("Wrong status. %s should be %s", Red.ID(), choices.Accepted) @@ -480,12 +541,15 @@ func RejectingDependencyTest(t *testing.T, factory Factory) { } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Green) - graph.Add(Blue) - graph.Add(purple) - - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if err := graph.Add(Blue); err != nil { + t.Fatal(err) + } else if err := graph.Add(purple); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s", Red.ID()) @@ -503,10 +567,9 @@ func RejectingDependencyTest(t *testing.T, factory Factory) { gp := ids.Bag{} gp.Add(Green.ID(), purple.ID()) - - graph.RecordPoll(gp) - - if prefs := graph.Preferences(); prefs.Len() != 2 { + if err := graph.RecordPoll(gp); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 2 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Green.ID()) { t.Fatalf("Wrong preference. Expected %s", Green.ID()) @@ -520,11 +583,9 @@ func RejectingDependencyTest(t *testing.T, factory Factory) { t.Fatalf("Wrong status. %s should be %s", Blue.ID(), choices.Processing) } else if purple.Status() != choices.Processing { t.Fatalf("Wrong status. %s should be %s", purple.ID(), choices.Processing) - } - - graph.RecordPoll(gp) - - if prefs := graph.Preferences(); prefs.Len() != 0 { + } else if err := graph.RecordPoll(gp); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 0 { t.Fatalf("Wrong number of preferences.") } else if Red.Status() != choices.Rejected { t.Fatalf("Wrong status. %s should be %s", Red.ID(), choices.Rejected) @@ -553,9 +614,9 @@ func VacuouslyAcceptedTest(t *testing.T, factory Factory) { } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(purple) - - if prefs := graph.Preferences(); prefs.Len() != 0 { + if err := graph.Add(purple); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 0 { t.Fatalf("Wrong number of preferences.") } else if status := purple.Status(); status != choices.Accepted { t.Fatalf("Wrong status. %s should be %s", purple.ID(), choices.Accepted) @@ -593,17 +654,15 @@ func ConflictsTest(t *testing.T, factory Factory) { Ins: insPurple, } - graph.Add(purple) - - if orangeConflicts := graph.Conflicts(orange); orangeConflicts.Len() != 1 { + if err := graph.Add(purple); err != nil { + t.Fatal(err) + } else if orangeConflicts := graph.Conflicts(orange); orangeConflicts.Len() != 1 { t.Fatalf("Wrong number of conflicts") } else if !orangeConflicts.Contains(purple.Identifier) { t.Fatalf("Conflicts does not contain the right transaction") - } - - graph.Add(orange) - - if orangeConflicts := graph.Conflicts(orange); orangeConflicts.Len() != 1 { + } else if err := graph.Add(orange); err != nil { + t.Fatal(err) + } else if orangeConflicts := graph.Conflicts(orange); orangeConflicts.Len() != 1 { t.Fatalf("Wrong number of conflicts") } else if !orangeConflicts.Contains(purple.Identifier) { t.Fatalf("Conflicts does not contain the right transaction") @@ -643,17 +702,20 @@ func VirtuousDependsOnRogueTest(t *testing.T, factory Factory) { virtuous.Ins.Add(input2) - graph.Add(rogue1) - graph.Add(rogue2) - graph.Add(virtuous) + if err := graph.Add(rogue1); err != nil { + t.Fatal(err) + } else if err := graph.Add(rogue2); err != nil { + t.Fatal(err) + } else if err := graph.Add(virtuous); err != nil { + t.Fatal(err) + } votes := ids.Bag{} votes.Add(rogue1.ID()) votes.Add(virtuous.ID()) - - graph.RecordPoll(votes) - - if status := rogue1.Status(); status != choices.Processing { + if err := graph.RecordPoll(votes); err != nil { + t.Fatal(err) + } else if status := rogue1.Status(); status != choices.Processing { t.Fatalf("Rogue Tx is %s expected %s", status, choices.Processing) } else if status := rogue2.Status(); status != choices.Processing { t.Fatalf("Rogue Tx is %s expected %s", status, choices.Processing) @@ -664,6 +726,135 @@ func VirtuousDependsOnRogueTest(t *testing.T, factory Factory) { } } +func ErrorOnVacuouslyAcceptedTest(t *testing.T, factory Factory) { + Setup() + + graph := factory.New() + + purple := &TestTx{ + Identifier: ids.Empty.Prefix(7), + Stat: choices.Processing, + Validity: errors.New(""), + } + + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, Alpha: 1, BetaVirtuous: 1, BetaRogue: 2, + } + graph.Initialize(snow.DefaultContextTest(), params) + + if err := graph.Add(purple); err == nil { + t.Fatalf("Should have errored on acceptance") + } +} + +func ErrorOnAcceptedTest(t *testing.T, factory Factory) { + Setup() + + graph := factory.New() + + purple := &TestTx{ + Identifier: ids.Empty.Prefix(7), + Stat: choices.Processing, + Validity: errors.New(""), + } + purple.Ins.Add(ids.Empty.Prefix(4)) + + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, Alpha: 1, BetaVirtuous: 1, BetaRogue: 2, + } + graph.Initialize(snow.DefaultContextTest(), params) + + if err := graph.Add(purple); err != nil { + t.Fatal(err) + } + + votes := ids.Bag{} + votes.Add(purple.ID()) + if err := graph.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on accepting an invalid tx") + } +} + +func ErrorOnRejectingLowerConfidenceConflictTest(t *testing.T, factory Factory) { + Setup() + + graph := factory.New() + + X := ids.Empty.Prefix(4) + + purple := &TestTx{ + Identifier: ids.Empty.Prefix(7), + Stat: choices.Processing, + } + purple.Ins.Add(X) + + pink := &TestTx{ + Identifier: ids.Empty.Prefix(8), + Stat: choices.Processing, + Validity: errors.New(""), + } + pink.Ins.Add(X) + + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, Alpha: 1, BetaVirtuous: 1, BetaRogue: 1, + } + graph.Initialize(snow.DefaultContextTest(), params) + + if err := graph.Add(purple); err != nil { + t.Fatal(err) + } else if err := graph.Add(pink); err != nil { + t.Fatal(err) + } + + votes := ids.Bag{} + votes.Add(purple.ID()) + if err := graph.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on rejecting an invalid tx") + } +} + +func ErrorOnRejectingHigherConfidenceConflictTest(t *testing.T, factory Factory) { + Setup() + + graph := factory.New() + + X := ids.Empty.Prefix(4) + + purple := &TestTx{ + Identifier: ids.Empty.Prefix(7), + Stat: choices.Processing, + } + purple.Ins.Add(X) + + pink := &TestTx{ + Identifier: ids.Empty.Prefix(8), + Stat: choices.Processing, + Validity: errors.New(""), + } + pink.Ins.Add(X) + + params := snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, Alpha: 1, BetaVirtuous: 1, BetaRogue: 1, + } + graph.Initialize(snow.DefaultContextTest(), params) + + if err := graph.Add(pink); err != nil { + t.Fatal(err) + } else if err := graph.Add(purple); err != nil { + t.Fatal(err) + } + + votes := ids.Bag{} + votes.Add(purple.ID()) + if err := graph.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on rejecting an invalid tx") + } +} + func StringTest(t *testing.T, factory Factory, prefix string) { Setup() @@ -674,12 +865,16 @@ func StringTest(t *testing.T, factory Factory, prefix string) { K: 2, Alpha: 2, BetaVirtuous: 1, BetaRogue: 2, } graph.Initialize(snow.DefaultContextTest(), params) - graph.Add(Red) - graph.Add(Green) - graph.Add(Blue) - graph.Add(Alpha) - if prefs := graph.Preferences(); prefs.Len() != 1 { + if err := graph.Add(Red); err != nil { + t.Fatal(err) + } else if err := graph.Add(Green); err != nil { + t.Fatal(err) + } else if err := graph.Add(Blue); err != nil { + t.Fatal(err) + } else if err := graph.Add(Alpha); err != nil { + t.Fatal(err) + } else if prefs := graph.Preferences(); prefs.Len() != 1 { t.Fatalf("Wrong number of preferences.") } else if !prefs.Contains(Red.ID()) { t.Fatalf("Wrong preference. Expected %s got %s", Red.ID(), prefs.List()[0]) @@ -691,8 +886,11 @@ func StringTest(t *testing.T, factory Factory, prefix string) { rb.SetThreshold(2) rb.AddCount(Red.ID(), 2) rb.AddCount(Blue.ID(), 2) - graph.RecordPoll(rb) - graph.Add(Blue) + if err := graph.RecordPoll(rb); err != nil { + t.Fatal(err) + } else if err := graph.Add(Blue); err != nil { + t.Fatal(err) + } { expected := prefix + "(\n" + @@ -720,7 +918,9 @@ func StringTest(t *testing.T, factory Factory, prefix string) { ga.SetThreshold(2) ga.AddCount(Green.ID(), 2) ga.AddCount(Alpha.ID(), 2) - graph.RecordPoll(ga) + if err := graph.RecordPoll(ga); err != nil { + t.Fatal(err) + } { expected := prefix + "(\n" + @@ -745,7 +945,9 @@ func StringTest(t *testing.T, factory Factory, prefix string) { } empty := ids.Bag{} - graph.RecordPoll(empty) + if err := graph.RecordPoll(empty); err != nil { + t.Fatal(err) + } { expected := prefix + "(\n" + @@ -767,10 +969,10 @@ func StringTest(t *testing.T, factory Factory, prefix string) { t.Fatalf("Wrong preference. Expected %s", Blue.ID()) } else if graph.Finalized() { t.Fatalf("Finalized too early") + } else if err := graph.RecordPoll(ga); err != nil { + t.Fatal(err) } - graph.RecordPoll(ga) - { expected := prefix + "(\n" + " Choice[0] = ID: LUC1cmcxnfNR9LdkACS2ccGKLEK7SYqB4gLLTycQfg1koyfSq Confidence: 0 Bias: 1\n" + @@ -791,10 +993,10 @@ func StringTest(t *testing.T, factory Factory, prefix string) { t.Fatalf("Wrong preference. Expected %s", Alpha.ID()) } else if graph.Finalized() { t.Fatalf("Finalized too early") + } else if err := graph.RecordPoll(ga); err != nil { + t.Fatal(err) } - graph.RecordPoll(ga) - { expected := prefix + "()" if str := graph.String(); str != expected { @@ -806,9 +1008,7 @@ func StringTest(t *testing.T, factory Factory, prefix string) { t.Fatalf("Wrong number of preferences.") } else if !graph.Finalized() { t.Fatalf("Finalized too late") - } - - if Green.Status() != choices.Accepted { + } else if Green.Status() != choices.Accepted { t.Fatalf("%s should have been accepted", Green.ID()) } else if Alpha.Status() != choices.Accepted { t.Fatalf("%s should have been accepted", Alpha.ID()) @@ -816,10 +1016,10 @@ func StringTest(t *testing.T, factory Factory, prefix string) { t.Fatalf("%s should have been rejected", Red.ID()) } else if Blue.Status() != choices.Rejected { t.Fatalf("%s should have been rejected", Blue.ID()) + } else if err := graph.RecordPoll(rb); err != nil { + t.Fatal(err) } - graph.RecordPoll(rb) - { expected := prefix + "()" if str := graph.String(); str != expected { @@ -831,9 +1031,7 @@ func StringTest(t *testing.T, factory Factory, prefix string) { t.Fatalf("Wrong number of preferences.") } else if !graph.Finalized() { t.Fatalf("Finalized too late") - } - - if Green.Status() != choices.Accepted { + } else if Green.Status() != choices.Accepted { t.Fatalf("%s should have been accepted", Green.ID()) } else if Alpha.Status() != choices.Accepted { t.Fatalf("%s should have been accepted", Alpha.ID()) diff --git a/snow/consensus/snowstorm/directed_test.go b/snow/consensus/snowstorm/directed_test.go index 39bc5bf..f61d53c 100644 --- a/snow/consensus/snowstorm/directed_test.go +++ b/snow/consensus/snowstorm/directed_test.go @@ -7,34 +7,4 @@ import ( "testing" ) -func TestDirectedParams(t *testing.T) { ParamsTest(t, DirectedFactory{}) } - -func TestDirectedIssued(t *testing.T) { IssuedTest(t, DirectedFactory{}) } - -func TestDirectedLeftoverInput(t *testing.T) { LeftoverInputTest(t, DirectedFactory{}) } - -func TestDirectedLowerConfidence(t *testing.T) { LowerConfidenceTest(t, DirectedFactory{}) } - -func TestDirectedMiddleConfidence(t *testing.T) { MiddleConfidenceTest(t, DirectedFactory{}) } - -func TestDirectedIndependent(t *testing.T) { IndependentTest(t, DirectedFactory{}) } - -func TestDirectedVirtuous(t *testing.T) { VirtuousTest(t, DirectedFactory{}) } - -func TestDirectedIsVirtuous(t *testing.T) { IsVirtuousTest(t, DirectedFactory{}) } - -func TestDirectedConflicts(t *testing.T) { ConflictsTest(t, DirectedFactory{}) } - -func TestDirectedQuiesce(t *testing.T) { QuiesceTest(t, DirectedFactory{}) } - -func TestDirectedAcceptingDependency(t *testing.T) { AcceptingDependencyTest(t, DirectedFactory{}) } - -func TestDirectedRejectingDependency(t *testing.T) { RejectingDependencyTest(t, DirectedFactory{}) } - -func TestDirectedVacuouslyAccepted(t *testing.T) { VacuouslyAcceptedTest(t, DirectedFactory{}) } - -func TestDirectedVirtuousDependsOnRogue(t *testing.T) { - VirtuousDependsOnRogueTest(t, DirectedFactory{}) -} - -func TestDirectedString(t *testing.T) { StringTest(t, DirectedFactory{}, "DG") } +func TestDirectedConsensus(t *testing.T) { ConsensusTest(t, DirectedFactory{}, "DG") } diff --git a/snow/consensus/snowstorm/input_test.go b/snow/consensus/snowstorm/input_test.go index 46a0033..9cae5e7 100644 --- a/snow/consensus/snowstorm/input_test.go +++ b/snow/consensus/snowstorm/input_test.go @@ -7,32 +7,4 @@ import ( "testing" ) -func TestInputParams(t *testing.T) { ParamsTest(t, InputFactory{}) } - -func TestInputIssued(t *testing.T) { IssuedTest(t, InputFactory{}) } - -func TestInputLeftoverInput(t *testing.T) { LeftoverInputTest(t, InputFactory{}) } - -func TestInputLowerConfidence(t *testing.T) { LowerConfidenceTest(t, InputFactory{}) } - -func TestInputMiddleConfidence(t *testing.T) { MiddleConfidenceTest(t, InputFactory{}) } - -func TestInputIndependent(t *testing.T) { IndependentTest(t, InputFactory{}) } - -func TestInputVirtuous(t *testing.T) { VirtuousTest(t, InputFactory{}) } - -func TestInputIsVirtuous(t *testing.T) { IsVirtuousTest(t, InputFactory{}) } - -func TestInputConflicts(t *testing.T) { ConflictsTest(t, InputFactory{}) } - -func TestInputQuiesce(t *testing.T) { QuiesceTest(t, InputFactory{}) } - -func TestInputAcceptingDependency(t *testing.T) { AcceptingDependencyTest(t, InputFactory{}) } - -func TestInputRejectingDependency(t *testing.T) { RejectingDependencyTest(t, InputFactory{}) } - -func TestInputVacuouslyAccepted(t *testing.T) { VacuouslyAcceptedTest(t, InputFactory{}) } - -func TestInputVirtuousDependsOnRogue(t *testing.T) { VirtuousDependsOnRogueTest(t, InputFactory{}) } - -func TestInputString(t *testing.T) { StringTest(t, InputFactory{}, "IG") } +func TestInputConsensus(t *testing.T) { ConsensusTest(t, InputFactory{}, "IG") } diff --git a/snow/consensus/snowstorm/test_tx.go b/snow/consensus/snowstorm/test_tx.go index 18f1465..e9fb8f7 100644 --- a/snow/consensus/snowstorm/test_tx.go +++ b/snow/consensus/snowstorm/test_tx.go @@ -31,10 +31,10 @@ func (tx *TestTx) InputIDs() ids.Set { return tx.Ins } func (tx *TestTx) Status() choices.Status { return tx.Stat } // Accept implements the Consumer interface -func (tx *TestTx) Accept() error { tx.Stat = choices.Accepted; return nil } +func (tx *TestTx) Accept() error { tx.Stat = choices.Accepted; return tx.Validity } // Reject implements the Consumer interface -func (tx *TestTx) Reject() error { tx.Stat = choices.Rejected; return nil } +func (tx *TestTx) Reject() error { tx.Stat = choices.Rejected; return tx.Validity } // Reset sets the status to pending func (tx *TestTx) Reset() { tx.Stat = choices.Processing } From 3211546b5a796a47703b9a48b54564cd74d5399e Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sun, 21 Jun 2020 23:56:08 -0400 Subject: [PATCH 53/80] Set up tests for error handling in avalanche --- snow/consensus/avalanche/consensus_test.go | 1327 +++++++++++++++++- snow/consensus/avalanche/topological_test.go | 825 +---------- snow/consensus/avalanche/vertex_test.go | 8 +- 3 files changed, 1288 insertions(+), 872 deletions(-) diff --git a/snow/consensus/avalanche/consensus_test.go b/snow/consensus/avalanche/consensus_test.go index 3135ce2..e47fed0 100644 --- a/snow/consensus/avalanche/consensus_test.go +++ b/snow/consensus/avalanche/consensus_test.go @@ -4,7 +4,9 @@ package avalanche import ( + "errors" "fmt" + "math" "testing" "github.com/prometheus/client_golang/prometheus" @@ -24,8 +26,102 @@ func GenerateID() ids.ID { var ( Genesis = GenerateID() offset = uint64(0) + + Tests = []func(*testing.T, Factory){ + MetricsTest, + ParamsTest, + AddTest, + VertexIssuedTest, + TxIssuedTest, + VirtuousTest, + VotingTest, + IgnoreInvalidVotingTest, + TransitiveVotingTest, + SplitVotingTest, + TransitiveRejectionTest, + IsVirtuousTest, + QuiesceTest, + OrphansTest, + ErrorOnVacuousAcceptTest, + ErrorOnTxAcceptTest, + ErrorOnVtxAcceptTest, + ErrorOnVtxRejectTest, + ErrorOnParentVtxRejectTest, + ErrorOnTransitiveVtxRejectTest, + } ) +func ConsensusTest(t *testing.T, factory Factory) { + for _, test := range Tests { + test(t, factory) + } +} + +func MetricsTest(t *testing.T, factory Factory) { + ctx := snow.DefaultContextTest() + + { + avl := factory.New() + params := Parameters{ + Parameters: snowball.Parameters{ + Namespace: fmt.Sprintf("gecko_%s", ctx.ChainID.String()), + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + }, + Parents: 2, + BatchSize: 1, + } + params.Metrics.Register(prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: params.Namespace, + Name: "vtx_processing", + })) + avl.Initialize(ctx, params, nil) + } + { + avl := factory.New() + params := Parameters{ + Parameters: snowball.Parameters{ + Namespace: fmt.Sprintf("gecko_%s", ctx.ChainID.String()), + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + }, + Parents: 2, + BatchSize: 1, + } + params.Metrics.Register(prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: params.Namespace, + Name: "vtx_accepted", + })) + avl.Initialize(ctx, params, nil) + } + { + avl := factory.New() + params := Parameters{ + Parameters: snowball.Parameters{ + Namespace: fmt.Sprintf("gecko_%s", ctx.ChainID.String()), + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + }, + Parents: 2, + BatchSize: 1, + } + params.Metrics.Register(prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: params.Namespace, + Name: "vtx_rejected", + })) + avl.Initialize(ctx, params, nil) + } +} + func ParamsTest(t *testing.T, factory Factory) { avl := factory.New() @@ -43,26 +139,6 @@ func ParamsTest(t *testing.T, factory Factory) { BatchSize: 1, } - numProcessing := prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: params.Namespace, - Name: "vtx_processing", - }) - numAccepted := prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: params.Namespace, - Name: "vtx_accepted", - }) - numRejected := prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: params.Namespace, - Name: "vtx_rejected", - }) - - params.Metrics.Register(numProcessing) - params.Metrics.Register(numAccepted) - params.Metrics.Register(numRejected) - avl.Initialize(ctx, params, nil) if p := avl.Parameters(); p.K != params.K { @@ -120,9 +196,9 @@ func AddTest(t *testing.T, factory Factory) { status: choices.Processing, } - avl.Add(vtx0) - - if avl.Finalized() { + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if avl.Finalized() { t.Fatalf("A non-empty avalanche instance is finalized") } else if !ids.UnsortedEquals([]ids.ID{vtx0.id}, avl.Preferences().List()) { t.Fatalf("Initial frontier failed to be set") @@ -139,25 +215,21 @@ func AddTest(t *testing.T, factory Factory) { status: choices.Processing, } - avl.Add(vtx1) - - if avl.Finalized() { + if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if avl.Finalized() { t.Fatalf("A non-empty avalanche instance is finalized") } else if !ids.UnsortedEquals([]ids.ID{vtx0.id}, avl.Preferences().List()) { t.Fatalf("Initial frontier failed to be set") - } - - avl.Add(vtx1) - - if avl.Finalized() { + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if avl.Finalized() { t.Fatalf("A non-empty avalanche instance is finalized") } else if !ids.UnsortedEquals([]ids.ID{vtx0.id}, avl.Preferences().List()) { t.Fatalf("Initial frontier failed to be set") - } - - avl.Add(vts[0]) - - if avl.Finalized() { + } else if err := avl.Add(vts[0]); err != nil { + t.Fatal(err) + } else if avl.Finalized() { t.Fatalf("A non-empty avalanche instance is finalized") } else if !ids.UnsortedEquals([]ids.ID{vtx0.id}, avl.Preferences().List()) { t.Fatalf("Initial frontier failed to be set") @@ -209,11 +281,9 @@ func VertexIssuedTest(t *testing.T, factory Factory) { if avl.VertexIssued(vtx) { t.Fatalf("Vertex reported as issued") - } - - avl.Add(vtx) - - if !avl.VertexIssued(vtx) { + } else if err := avl.Add(vtx); err != nil { + t.Fatal(err) + } else if !avl.VertexIssued(vtx) { t.Fatalf("Vertex reported as not issued") } } @@ -266,9 +336,1178 @@ func TxIssuedTest(t *testing.T, factory Factory) { status: choices.Processing, } - avl.Add(vtx) - - if !avl.TxIssued(tx1) { + if err := avl.Add(vtx); err != nil { + t.Fatal(err) + } else if !avl.TxIssued(tx1) { t.Fatalf("Tx reported as not issued") } } + +func VirtuousTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID(), GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + if virtuous := avl.Virtuous(); virtuous.Len() != 2 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vts[0].ID()) { + t.Fatalf("Wrong virtuous") + } else if !virtuous.Contains(vts[1].ID()) { + t.Fatalf("Wrong virtuous") + } + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + tx2 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx2.Ins.Add(utxos[1]) + + vtx2 := &Vtx{ + dependencies: []Vertex{vtx0}, + id: GenerateID(), + txs: []snowstorm.Tx{tx2}, + height: 2, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if virtuous := avl.Virtuous(); virtuous.Len() != 1 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vtx0.id) { + t.Fatalf("Wrong virtuous") + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if virtuous := avl.Virtuous(); virtuous.Len() != 1 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vtx0.id) { + t.Fatalf("Wrong virtuous") + } else if err := avl.RecordPoll(ids.UniqueBag{}); err != nil { + t.Fatal(err) + } else if virtuous := avl.Virtuous(); virtuous.Len() != 2 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vts[0].ID()) { + t.Fatalf("Wrong virtuous") + } else if !virtuous.Contains(vts[1].ID()) { + t.Fatalf("Wrong virtuous") + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } else if virtuous := avl.Virtuous(); virtuous.Len() != 2 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vts[0].ID()) { + t.Fatalf("Wrong virtuous") + } else if !virtuous.Contains(vts[1].ID()) { + t.Fatalf("Wrong virtuous") + } else if err := avl.RecordPoll(ids.UniqueBag{}); err != nil { + t.Fatal(err) + } else if virtuous := avl.Virtuous(); virtuous.Len() != 2 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vts[0].ID()) { + t.Fatalf("Wrong virtuous") + } else if !virtuous.Contains(vts[1].ID()) { + t.Fatalf("Wrong virtuous") + } +} + +func VotingTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } + + sm := ids.UniqueBag{} + sm.Add(0, vtx1.id) + sm.Add(1, vtx1.id) + if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if avl.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if !avl.Finalized() { + t.Fatalf("An avalanche instance finalized too late") + } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if tx0.Status() != choices.Rejected { + t.Fatalf("Tx should have been rejected") + } else if tx1.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } +} + +func IgnoreInvalidVotingTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 3, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 1, + }, + Parents: 2, + BatchSize: 1, + } + + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } + + sm := ids.UniqueBag{} + sm.Add(0, vtx0.id) + sm.Add(1, vtx1.id) + + // Add Illegal Vote cast by Response 2 + sm.Add(2, vtx0.id) + sm.Add(2, vtx1.id) + + if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if avl.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } +} + +func TransitiveVotingTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID(), GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[1]) + + vtx1 := &Vtx{ + dependencies: []Vertex{vtx0}, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 2, + status: choices.Processing, + } + + vtx2 := &Vtx{ + dependencies: []Vertex{vtx1}, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 3, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } + + sm1 := ids.UniqueBag{} + sm1.Add(0, vtx0.id) + sm1.Add(1, vtx2.id) + if err := avl.RecordPoll(sm1); err != nil { + t.Fatal(err) + } else if avl.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } else if !ids.UnsortedEquals([]ids.ID{vtx2.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if tx0.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } + + sm2 := ids.UniqueBag{} + sm2.Add(0, vtx2.id) + sm2.Add(1, vtx2.id) + if err := avl.RecordPoll(sm2); err != nil { + t.Fatal(err) + } else if !avl.Finalized() { + t.Fatalf("An avalanche instance finalized too late") + } else if !ids.UnsortedEquals([]ids.ID{vtx2.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if tx0.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } else if tx1.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } +} + +func SplitVotingTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } + + sm1 := ids.UniqueBag{} + sm1.Add(0, vtx0.id) // peer 0 votes for the tx though vtx0 + sm1.Add(1, vtx1.id) // peer 1 votes for the tx though vtx1 + if err := avl.RecordPoll(sm1); err != nil { + t.Fatal(err) + } else if !avl.Finalized() { + t.Fatalf("An avalanche instance finalized too late") + } else if !ids.UnsortedEquals([]ids.ID{vtx0.id, vtx1.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if tx0.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } +} + +func TransitiveRejectionTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID(), GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + tx2 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx2.Ins.Add(utxos[1]) + + vtx2 := &Vtx{ + dependencies: []Vertex{vtx0}, + id: GenerateID(), + txs: []snowstorm.Tx{tx2}, + height: 2, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } + + sm := ids.UniqueBag{} + sm.Add(0, vtx1.id) + sm.Add(1, vtx1.id) + if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if avl.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if avl.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if tx0.Status() != choices.Rejected { + t.Fatalf("Tx should have been rejected") + } else if tx1.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } else if tx2.Status() != choices.Processing { + t.Fatalf("Tx should not have been decided") + } else if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if avl.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, avl.Preferences().List()) { + t.Fatalf("Initial frontier failed to be set") + } else if tx0.Status() != choices.Rejected { + t.Fatalf("Tx should have been rejected") + } else if tx1.Status() != choices.Accepted { + t.Fatalf("Tx should have been accepted") + } else if tx2.Status() != choices.Processing { + t.Fatalf("Tx should not have been decided") + } +} + +func IsVirtuousTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID(), GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + if virtuous := avl.Virtuous(); virtuous.Len() != 2 { + t.Fatalf("Wrong number of virtuous.") + } else if !virtuous.Contains(vts[0].ID()) { + t.Fatalf("Wrong virtuous") + } else if !virtuous.Contains(vts[1].ID()) { + t.Fatalf("Wrong virtuous") + } + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + if !avl.IsVirtuous(tx0) { + t.Fatalf("Should be virtuous.") + } else if !avl.IsVirtuous(tx1) { + t.Fatalf("Should be virtuous.") + } else if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if !avl.IsVirtuous(tx0) { + t.Fatalf("Should be virtuous.") + } else if avl.IsVirtuous(tx1) { + t.Fatalf("Should not be virtuous.") + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if avl.IsVirtuous(tx0) { + t.Fatalf("Should not be virtuous.") + } else if avl.IsVirtuous(tx1) { + t.Fatalf("Should not be virtuous.") + } +} + +func QuiesceTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID(), GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + tx2 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx2.Ins.Add(utxos[1]) + + vtx2 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx2}, + height: 2, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if avl.Quiesce() { + t.Fatalf("Shouldn't quiesce") + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if !avl.Quiesce() { + t.Fatalf("Should quiesce") + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } else if avl.Quiesce() { + t.Fatalf("Shouldn't quiesce") + } + + sm := ids.UniqueBag{} + sm.Add(0, vtx2.id) + if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if !avl.Quiesce() { + t.Fatalf("Should quiesce") + } +} + +func OrphansTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: math.MaxInt32, + BetaRogue: math.MaxInt32, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID(), GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + tx2 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx2.Ins.Add(utxos[1]) + + vtx2 := &Vtx{ + dependencies: []Vertex{vtx0}, + id: GenerateID(), + txs: []snowstorm.Tx{tx2}, + height: 2, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if orphans := avl.Orphans(); orphans.Len() != 0 { + t.Fatalf("Wrong number of orphans") + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if orphans := avl.Orphans(); orphans.Len() != 0 { + t.Fatalf("Wrong number of orphans") + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } else if orphans := avl.Orphans(); orphans.Len() != 0 { + t.Fatalf("Wrong number of orphans") + } + + sm := ids.UniqueBag{} + sm.Add(0, vtx1.id) + if err := avl.RecordPoll(sm); err != nil { + t.Fatal(err) + } else if orphans := avl.Orphans(); orphans.Len() != 1 { + t.Fatalf("Wrong number of orphans") + } else if !orphans.Contains(tx2.ID()) { + t.Fatalf("Wrong orphan") + } +} + +func ErrorOnVacuousAcceptTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: math.MaxInt32, + BetaRogue: math.MaxInt32, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + Validity: errors.New(""), + } + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err == nil { + t.Fatalf("Should have errored on vertex issuance") + } +} + +func ErrorOnTxAcceptTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + Validity: errors.New(""), + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } + + votes := ids.UniqueBag{} + votes.Add(0, vtx0.id) + if err := avl.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on vertex acceptance") + } +} + +func ErrorOnVtxAcceptTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + Validity: errors.New(""), + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } + + votes := ids.UniqueBag{} + votes.Add(0, vtx0.id) + if err := avl.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on vertex acceptance") + } +} + +func ErrorOnVtxRejectTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + Validity: errors.New(""), + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } + + votes := ids.UniqueBag{} + votes.Add(0, vtx0.id) + if err := avl.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on vertex rejection") + } +} + +func ErrorOnParentVtxRejectTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + Validity: errors.New(""), + } + + vtx2 := &Vtx{ + dependencies: []Vertex{vtx1}, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } + + votes := ids.UniqueBag{} + votes.Add(0, vtx0.id) + if err := avl.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on vertex rejection") + } +} + +func ErrorOnTransitiveVtxRejectTest(t *testing.T, factory Factory) { + avl := factory.New() + + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 1, + Alpha: 1, + BetaVirtuous: 1, + BetaRogue: 1, + ConcurrentRepolls: 1, + }, + Parents: 2, + BatchSize: 1, + } + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + avl.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + vtx2 := &Vtx{ + dependencies: []Vertex{vtx1}, + id: GenerateID(), + height: 1, + status: choices.Processing, + Validity: errors.New(""), + } + + if err := avl.Add(vtx0); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx1); err != nil { + t.Fatal(err) + } else if err := avl.Add(vtx2); err != nil { + t.Fatal(err) + } + + votes := ids.UniqueBag{} + votes.Add(0, vtx0.id) + if err := avl.RecordPoll(votes); err == nil { + t.Fatalf("Should have errored on vertex rejection") + } +} diff --git a/snow/consensus/avalanche/topological_test.go b/snow/consensus/avalanche/topological_test.go index 3ed648a..4b2c617 100644 --- a/snow/consensus/avalanche/topological_test.go +++ b/snow/consensus/avalanche/topological_test.go @@ -4,830 +4,7 @@ package avalanche import ( - "math" "testing" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/snow" - "github.com/ava-labs/gecko/snow/choices" - "github.com/ava-labs/gecko/snow/consensus/snowball" - "github.com/ava-labs/gecko/snow/consensus/snowstorm" ) -func TestTopologicalParams(t *testing.T) { ParamsTest(t, TopologicalFactory{}) } - -func TestTopologicalAdd(t *testing.T) { AddTest(t, TopologicalFactory{}) } - -func TestTopologicalVertexIssued(t *testing.T) { VertexIssuedTest(t, TopologicalFactory{}) } - -func TestTopologicalTxIssued(t *testing.T) { TxIssuedTest(t, TopologicalFactory{}) } - -func TestAvalancheVoting(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 2, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 2, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - ta.Add(vtx0) - ta.Add(vtx1) - - sm := make(ids.UniqueBag) - sm.Add(0, vtx1.id) - sm.Add(1, vtx1.id) - ta.RecordPoll(sm) - - if ta.Finalized() { - t.Fatalf("An avalanche instance finalized too early") - } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } - - ta.RecordPoll(sm) - - if !ta.Finalized() { - t.Fatalf("An avalanche instance finalized too late") - } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } else if tx0.Status() != choices.Rejected { - t.Fatalf("Tx should have been rejected") - } else if tx1.Status() != choices.Accepted { - t.Fatalf("Tx should have been accepted") - } -} - -func TestAvalancheIgnoreInvalidVoting(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 3, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 1, - }, - Parents: 2, - BatchSize: 1, - } - - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - ta.Add(vtx0) - ta.Add(vtx1) - - sm := make(ids.UniqueBag) - - sm.Add(0, vtx0.id) - sm.Add(1, vtx1.id) - - // Add Illegal Vote cast by Response 2 - sm.Add(2, vtx0.id) - sm.Add(2, vtx1.id) - - ta.RecordPoll(sm) - - if ta.Finalized() { - t.Fatalf("An avalanche instance finalized too early") - } -} - -func TestAvalancheTransitiveVoting(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 2, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 2, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID(), GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[1]) - - vtx1 := &Vtx{ - dependencies: []Vertex{vtx0}, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 2, - status: choices.Processing, - } - - vtx2 := &Vtx{ - dependencies: []Vertex{vtx1}, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 3, - status: choices.Processing, - } - - ta.Add(vtx0) - ta.Add(vtx1) - ta.Add(vtx2) - - sm1 := make(ids.UniqueBag) - sm1.Add(0, vtx0.id) - sm1.Add(1, vtx2.id) - ta.RecordPoll(sm1) - - if ta.Finalized() { - t.Fatalf("An avalanche instance finalized too early") - } else if !ids.UnsortedEquals([]ids.ID{vtx2.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } else if tx0.Status() != choices.Accepted { - t.Fatalf("Tx should have been accepted") - } - - sm2 := make(ids.UniqueBag) - sm2.Add(0, vtx2.id) - sm2.Add(1, vtx2.id) - ta.RecordPoll(sm2) - - if !ta.Finalized() { - t.Fatalf("An avalanche instance finalized too late") - } else if !ids.UnsortedEquals([]ids.ID{vtx2.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } else if tx0.Status() != choices.Accepted { - t.Fatalf("Tx should have been accepted") - } else if tx1.Status() != choices.Accepted { - t.Fatalf("Tx should have been accepted") - } -} - -func TestAvalancheSplitVoting(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 2, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 2, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - ta.Add(vtx0) - ta.Add(vtx1) - - sm1 := make(ids.UniqueBag) - sm1.Add(0, vtx0.id) - sm1.Add(1, vtx1.id) - ta.RecordPoll(sm1) - - if !ta.Finalized() { - t.Fatalf("An avalanche instance finalized too late") - } else if !ids.UnsortedEquals([]ids.ID{vtx0.id, vtx1.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } else if tx0.Status() != choices.Accepted { - t.Fatalf("Tx should have been accepted") - } -} - -func TestAvalancheTransitiveRejection(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 2, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 2, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID(), GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - tx2 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx2.Ins.Add(utxos[1]) - - vtx2 := &Vtx{ - dependencies: []Vertex{vtx0}, - id: GenerateID(), - txs: []snowstorm.Tx{tx2}, - height: 2, - status: choices.Processing, - } - - ta.Add(vtx0) - ta.Add(vtx1) - ta.Add(vtx2) - - sm := make(ids.UniqueBag) - sm.Add(0, vtx1.id) - sm.Add(1, vtx1.id) - ta.RecordPoll(sm) - - if ta.Finalized() { - t.Fatalf("An avalanche instance finalized too early") - } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } - - ta.RecordPoll(sm) - - if ta.Finalized() { - t.Fatalf("An avalanche instance finalized too early") - } else if !ids.UnsortedEquals([]ids.ID{vtx1.id}, ta.Preferences().List()) { - t.Fatalf("Initial frontier failed to be set") - } else if tx0.Status() != choices.Rejected { - t.Fatalf("Tx should have been rejected") - } else if tx1.Status() != choices.Accepted { - t.Fatalf("Tx should have been accepted") - } else if tx2.Status() != choices.Processing { - t.Fatalf("Tx should not have been decided") - } - - ta.preferenceCache = make(map[[32]byte]bool) - ta.virtuousCache = make(map[[32]byte]bool) - - ta.update(vtx2) -} - -func TestAvalancheVirtuous(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 2, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 2, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID(), GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - if virtuous := ta.Virtuous(); virtuous.Len() != 2 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vts[0].ID()) { - t.Fatalf("Wrong virtuous") - } else if !virtuous.Contains(vts[1].ID()) { - t.Fatalf("Wrong virtuous") - } - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - tx2 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx2.Ins.Add(utxos[1]) - - vtx2 := &Vtx{ - dependencies: []Vertex{vtx0}, - id: GenerateID(), - txs: []snowstorm.Tx{tx2}, - height: 2, - status: choices.Processing, - } - - ta.Add(vtx0) - - if virtuous := ta.Virtuous(); virtuous.Len() != 1 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vtx0.id) { - t.Fatalf("Wrong virtuous") - } - - ta.Add(vtx1) - - if virtuous := ta.Virtuous(); virtuous.Len() != 1 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vtx0.id) { - t.Fatalf("Wrong virtuous") - } - - ta.updateFrontiers() - - if virtuous := ta.Virtuous(); virtuous.Len() != 2 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vts[0].ID()) { - t.Fatalf("Wrong virtuous") - } else if !virtuous.Contains(vts[1].ID()) { - t.Fatalf("Wrong virtuous") - } - - ta.Add(vtx2) - - if virtuous := ta.Virtuous(); virtuous.Len() != 2 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vts[0].ID()) { - t.Fatalf("Wrong virtuous") - } else if !virtuous.Contains(vts[1].ID()) { - t.Fatalf("Wrong virtuous") - } - - ta.updateFrontiers() - - if virtuous := ta.Virtuous(); virtuous.Len() != 2 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vts[0].ID()) { - t.Fatalf("Wrong virtuous") - } else if !virtuous.Contains(vts[1].ID()) { - t.Fatalf("Wrong virtuous") - } -} - -func TestAvalancheIsVirtuous(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 2, - Alpha: 2, - BetaVirtuous: 1, - BetaRogue: 2, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID(), GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - if virtuous := ta.Virtuous(); virtuous.Len() != 2 { - t.Fatalf("Wrong number of virtuous.") - } else if !virtuous.Contains(vts[0].ID()) { - t.Fatalf("Wrong virtuous") - } else if !virtuous.Contains(vts[1].ID()) { - t.Fatalf("Wrong virtuous") - } - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - if !ta.IsVirtuous(tx0) { - t.Fatalf("Should be virtuous.") - } else if !ta.IsVirtuous(tx1) { - t.Fatalf("Should be virtuous.") - } - - ta.Add(vtx0) - - if !ta.IsVirtuous(tx0) { - t.Fatalf("Should be virtuous.") - } else if ta.IsVirtuous(tx1) { - t.Fatalf("Should not be virtuous.") - } - - ta.Add(vtx1) - - if ta.IsVirtuous(tx0) { - t.Fatalf("Should not be virtuous.") - } else if ta.IsVirtuous(tx1) { - t.Fatalf("Should not be virtuous.") - } -} - -func TestAvalancheQuiesce(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 1, - Alpha: 1, - BetaVirtuous: 1, - BetaRogue: 1, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID(), GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - tx2 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx2.Ins.Add(utxos[1]) - - vtx2 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx2}, - height: 2, - status: choices.Processing, - } - - ta.Add(vtx0) - - if ta.Quiesce() { - t.Fatalf("Shouldn't quiesce") - } - - ta.Add(vtx1) - - if !ta.Quiesce() { - t.Fatalf("Should quiesce") - } - - ta.Add(vtx2) - - if ta.Quiesce() { - t.Fatalf("Shouldn't quiesce") - } - - sm := make(ids.UniqueBag) - sm.Add(0, vtx2.id) - ta.RecordPoll(sm) - - if !ta.Quiesce() { - t.Fatalf("Should quiesce") - } -} - -func TestAvalancheOrphans(t *testing.T) { - params := Parameters{ - Parameters: snowball.Parameters{ - Metrics: prometheus.NewRegistry(), - K: 1, - Alpha: 1, - BetaVirtuous: math.MaxInt32, - BetaRogue: math.MaxInt32, - ConcurrentRepolls: 1, - }, - Parents: 2, - BatchSize: 1, - } - vts := []Vertex{&Vtx{ - id: GenerateID(), - status: choices.Accepted, - }, &Vtx{ - id: GenerateID(), - status: choices.Accepted, - }} - utxos := []ids.ID{GenerateID(), GenerateID()} - - ta := Topological{} - ta.Initialize(snow.DefaultContextTest(), params, vts) - - tx0 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx0.Ins.Add(utxos[0]) - - vtx0 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx0}, - height: 1, - status: choices.Processing, - } - - tx1 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx1.Ins.Add(utxos[0]) - - vtx1 := &Vtx{ - dependencies: vts, - id: GenerateID(), - txs: []snowstorm.Tx{tx1}, - height: 1, - status: choices.Processing, - } - - tx2 := &snowstorm.TestTx{ - Identifier: GenerateID(), - Stat: choices.Processing, - } - tx2.Ins.Add(utxos[1]) - - vtx2 := &Vtx{ - dependencies: []Vertex{vtx0}, - id: GenerateID(), - txs: []snowstorm.Tx{tx2}, - height: 2, - status: choices.Processing, - } - - ta.Add(vtx0) - - if orphans := ta.Orphans(); orphans.Len() != 0 { - t.Fatalf("Wrong number of orphans") - } - - ta.Add(vtx1) - - if orphans := ta.Orphans(); orphans.Len() != 0 { - t.Fatalf("Wrong number of orphans") - } - - ta.Add(vtx2) - - if orphans := ta.Orphans(); orphans.Len() != 0 { - t.Fatalf("Wrong number of orphans") - } - - sm := make(ids.UniqueBag) - sm.Add(0, vtx1.id) - ta.RecordPoll(sm) - - if orphans := ta.Orphans(); orphans.Len() != 1 { - t.Fatalf("Wrong number of orphans") - } else if !orphans.Contains(tx2.ID()) { - t.Fatalf("Wrong orphan") - } -} +func TestTopological(t *testing.T) { ConsensusTest(t, TopologicalFactory{}) } diff --git a/snow/consensus/avalanche/vertex_test.go b/snow/consensus/avalanche/vertex_test.go index f7aee5a..0270af5 100644 --- a/snow/consensus/avalanche/vertex_test.go +++ b/snow/consensus/avalanche/vertex_test.go @@ -19,7 +19,8 @@ type Vtx struct { height uint64 status choices.Status - bytes []byte + Validity error + bytes []byte } func (v *Vtx) ID() ids.ID { return v.id } @@ -28,9 +29,8 @@ func (v *Vtx) Parents() []Vertex { return v.dependencies } func (v *Vtx) Height() uint64 { return v.height } func (v *Vtx) Txs() []snowstorm.Tx { return v.txs } func (v *Vtx) Status() choices.Status { return v.status } -func (v *Vtx) Live() {} -func (v *Vtx) Accept() error { v.status = choices.Accepted; return nil } -func (v *Vtx) Reject() error { v.status = choices.Rejected; return nil } +func (v *Vtx) Accept() error { v.status = choices.Accepted; return v.Validity } +func (v *Vtx) Reject() error { v.status = choices.Rejected; return v.Validity } func (v *Vtx) Bytes() []byte { return v.bytes } type sortVts []*Vtx From 12297cb0d24733ad33132597590eb9f7ea85533e Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 22 Jun 2020 00:08:11 -0400 Subject: [PATCH 54/80] Clarify that the snowball byzantine struct is only for testing --- snow/consensus/snowball/byzantine.go | 48 -------------------- snow/consensus/snowball/byzantine_test.go | 54 ----------------------- snow/consensus/snowball/consensus_test.go | 40 +++++++++++++++++ 3 files changed, 40 insertions(+), 102 deletions(-) delete mode 100644 snow/consensus/snowball/byzantine.go delete mode 100644 snow/consensus/snowball/byzantine_test.go diff --git a/snow/consensus/snowball/byzantine.go b/snow/consensus/snowball/byzantine.go deleted file mode 100644 index 88fda59..0000000 --- a/snow/consensus/snowball/byzantine.go +++ /dev/null @@ -1,48 +0,0 @@ -// (c) 2019-2020, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package snowball - -import ( - "github.com/ava-labs/gecko/ids" -) - -// ByzantineFactory implements Factory by returning a byzantine struct -type ByzantineFactory struct{} - -// New implements Factory -func (ByzantineFactory) New() Consensus { return &Byzantine{} } - -// Byzantine is a naive implementation of a multi-choice snowball instance -type Byzantine struct { - // params contains all the configurations of a snowball instance - params Parameters - - // Hardcode the preference - preference ids.ID -} - -// Initialize implements the Consensus interface -func (b *Byzantine) Initialize(params Parameters, choice ids.ID) { - b.params = params - b.preference = choice -} - -// Parameters implements the Consensus interface -func (b *Byzantine) Parameters() Parameters { return b.params } - -// Add implements the Consensus interface -func (b *Byzantine) Add(choice ids.ID) {} - -// Preference implements the Consensus interface -func (b *Byzantine) Preference() ids.ID { return b.preference } - -// RecordPoll implements the Consensus interface -func (b *Byzantine) RecordPoll(votes ids.Bag) {} - -// RecordUnsuccessfulPoll implements the Consensus interface -func (b *Byzantine) RecordUnsuccessfulPoll() {} - -// Finalized implements the Consensus interface -func (b *Byzantine) Finalized() bool { return true } -func (b *Byzantine) String() string { return b.preference.String() } diff --git a/snow/consensus/snowball/byzantine_test.go b/snow/consensus/snowball/byzantine_test.go deleted file mode 100644 index cee357b..0000000 --- a/snow/consensus/snowball/byzantine_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// (c) 2019-2020, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package snowball - -import ( - "testing" - - "github.com/ava-labs/gecko/ids" - "github.com/prometheus/client_golang/prometheus" -) - -func TestByzantine(t *testing.T) { - params := Parameters{ - Metrics: prometheus.NewRegistry(), - K: 1, Alpha: 1, BetaVirtuous: 3, BetaRogue: 5, - } - - byzFactory := ByzantineFactory{} - byz := byzFactory.New() - byz.Initialize(params, Blue) - - if ret := byz.Parameters(); ret != params { - t.Fatalf("Should have returned the correct params") - } - - byz.Add(Green) - - if pref := byz.Preference(); !pref.Equals(Blue) { - t.Fatalf("Wrong preference, expected %s returned %s", Blue, pref) - } - - oneGreen := ids.Bag{} - oneGreen.Add(Green) - byz.RecordPoll(oneGreen) - - if pref := byz.Preference(); !pref.Equals(Blue) { - t.Fatalf("Wrong preference, expected %s returned %s", Blue, pref) - } - - byz.RecordUnsuccessfulPoll() - - if pref := byz.Preference(); !pref.Equals(Blue) { - t.Fatalf("Wrong preference, expected %s returned %s", Blue, pref) - } - - if final := byz.Finalized(); !final { - t.Fatalf("Should be marked as accepted") - } - - if str := byz.String(); str != Blue.String() { - t.Fatalf("Wrong string, expected %s returned %s", Blue, str) - } -} diff --git a/snow/consensus/snowball/consensus_test.go b/snow/consensus/snowball/consensus_test.go index 922f606..304bc19 100644 --- a/snow/consensus/snowball/consensus_test.go +++ b/snow/consensus/snowball/consensus_test.go @@ -11,6 +11,46 @@ import ( "github.com/ava-labs/gecko/ids" ) +// ByzantineFactory implements Factory by returning a byzantine struct +type ByzantineFactory struct{} + +// New implements Factory +func (ByzantineFactory) New() Consensus { return &Byzantine{} } + +// Byzantine is a naive implementation of a multi-choice snowball instance +type Byzantine struct { + // params contains all the configurations of a snowball instance + params Parameters + + // Hardcode the preference + preference ids.ID +} + +// Initialize implements the Consensus interface +func (b *Byzantine) Initialize(params Parameters, choice ids.ID) { + b.params = params + b.preference = choice +} + +// Parameters implements the Consensus interface +func (b *Byzantine) Parameters() Parameters { return b.params } + +// Add implements the Consensus interface +func (b *Byzantine) Add(choice ids.ID) {} + +// Preference implements the Consensus interface +func (b *Byzantine) Preference() ids.ID { return b.preference } + +// RecordPoll implements the Consensus interface +func (b *Byzantine) RecordPoll(votes ids.Bag) {} + +// RecordUnsuccessfulPoll implements the Consensus interface +func (b *Byzantine) RecordUnsuccessfulPoll() {} + +// Finalized implements the Consensus interface +func (b *Byzantine) Finalized() bool { return true } +func (b *Byzantine) String() string { return b.preference.String() } + var ( Red = ids.Empty.Prefix(0) Blue = ids.Empty.Prefix(1) From c88c85ea9b0df236ebbbce78b1c7a6e30ffc7507 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 22 Jun 2020 00:14:19 -0400 Subject: [PATCH 55/80] Minor cleanup in snowball consensus --- snow/consensus/snowball/flat.go | 2 +- snow/consensus/snowball/nnary_snowflake.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/snow/consensus/snowball/flat.go b/snow/consensus/snowball/flat.go index 21663c4..464e525 100644 --- a/snow/consensus/snowball/flat.go +++ b/snow/consensus/snowball/flat.go @@ -34,7 +34,7 @@ func (f *Flat) Parameters() Parameters { return f.params } // RecordPoll implements the Consensus interface func (f *Flat) RecordPoll(votes ids.Bag) { if pollMode, numVotes := votes.Mode(); numVotes >= f.params.Alpha { - f.nnarySnowball.RecordSuccessfulPoll(pollMode) + f.RecordSuccessfulPoll(pollMode) } else { f.RecordUnsuccessfulPoll() } diff --git a/snow/consensus/snowball/nnary_snowflake.go b/snow/consensus/snowball/nnary_snowflake.go index 8b461f0..ab580f9 100644 --- a/snow/consensus/snowball/nnary_snowflake.go +++ b/snow/consensus/snowball/nnary_snowflake.go @@ -51,7 +51,7 @@ func (sf *nnarySnowflake) RecordSuccessfulPoll(choice ids.ID) { return // This instace is already decided. } - if preference := sf.nnarySlush.Preference(); preference.Equals(choice) { + if preference := sf.Preference(); preference.Equals(choice) { sf.confidence++ } else { // confidence is set to 1 because there has already been 1 successful From 5cb106d349bb0ce9b9337a3bf9360c966438755b Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 22 Jun 2020 10:53:10 -0400 Subject: [PATCH 56/80] make staking cert/key read-only --- staking/gen_staker_key.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/staking/gen_staker_key.go b/staking/gen_staker_key.go index 8969ea3..37142c4 100644 --- a/staking/gen_staker_key.go +++ b/staking/gen_staker_key.go @@ -40,20 +40,27 @@ func GenerateStakingKeyCert(keyPath, certPath string) error { return fmt.Errorf("couldn't create certificate: %w", err) } - // Write cert to disk - if err := os.MkdirAll(filepath.Dir(certPath), 0755); err != nil { - return fmt.Errorf("couldn't create path for key/cert: %w", err) + // Ensure directory where key/cert will live exist + if err := os.MkdirAll(filepath.Dir(certPath), 0700); err != nil { + return fmt.Errorf("couldn't create path for cert: %w", err) + } else if err := os.MkdirAll(filepath.Dir(keyPath), 0700); err != nil { + return fmt.Errorf("couldn't create path for key: %w", err) } - certOut, err := os.Create(certPath) + + // Write cert to disk + certFile, err := os.Create(certPath) if err != nil { return fmt.Errorf("couldn't create cert file: %w", err) } - if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes}); err != nil { + if err := pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes}); err != nil { return fmt.Errorf("couldn't write cert file: %w", err) } - if err := certOut.Close(); err != nil { + if err := certFile.Close(); err != nil { return fmt.Errorf("couldn't close cert file: %w", err) } + if err := os.Chmod(certPath, 0400); err != nil { // Make cert read-only + return fmt.Errorf("couldn't change permissions on cert: %w", err) + } // Write key to disk keyOut, err := os.Create(keyPath) @@ -70,5 +77,9 @@ func GenerateStakingKeyCert(keyPath, certPath string) error { if err := keyOut.Close(); err != nil { return fmt.Errorf("couldn't close key file: %w", err) } + if err := os.Chmod(keyPath, 0400); err != nil { // Make key read-only + return fmt.Errorf("couldn't change permissions on key") + } + return nil } From 38f7e236473beea8d112875a98d1a69e6c44ce40 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 22 Jun 2020 13:05:47 -0400 Subject: [PATCH 57/80] disable keystore and admin APIs by default --- main/params.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/params.go b/main/params.go index 1e526b3..877d406 100644 --- a/main/params.go +++ b/main/params.go @@ -190,7 +190,7 @@ func init() { consensusIP := fs.String("public-ip", "", "Public IP of this node") // HTTP Server: - httpHost := fs.String("http-host", "", "Address of the HTTP server") + httpHost := fs.String("http-host", "127.0.0.1", "Address of the HTTP server") httpPort := fs.Uint("http-port", 9650, "Port of the HTTP server") fs.BoolVar(&Config.EnableHTTPS, "http-tls-enabled", false, "Upgrade the HTTP server to HTTPs") fs.StringVar(&Config.HTTPSKeyFile, "http-tls-key-file", "", "TLS private key file for the HTTPs server") @@ -225,9 +225,9 @@ func init() { fs.IntVar(&Config.ConsensusParams.ConcurrentRepolls, "snow-concurrent-repolls", 1, "Minimum number of concurrent polls for finalizing consensus") // Enable/Disable APIs: - fs.BoolVar(&Config.AdminAPIEnabled, "api-admin-enabled", true, "If true, this node exposes the Admin API") + fs.BoolVar(&Config.AdminAPIEnabled, "api-admin-enabled", false, "If true, this node exposes the Admin API") fs.BoolVar(&Config.InfoAPIEnabled, "api-info-enabled", true, "If true, this node exposes the Info API") - fs.BoolVar(&Config.KeystoreAPIEnabled, "api-keystore-enabled", true, "If true, this node exposes the Keystore API") + fs.BoolVar(&Config.KeystoreAPIEnabled, "api-keystore-enabled", false, "If true, this node exposes the Keystore API") fs.BoolVar(&Config.MetricsAPIEnabled, "api-metrics-enabled", true, "If true, this node exposes the Metrics API") fs.BoolVar(&Config.HealthAPIEnabled, "api-health-enabled", true, "If true, this node exposes the Health API") fs.BoolVar(&Config.IPCEnabled, "api-ipcs-enabled", false, "If true, IPCs can be opened") From c7356a581df7e063a570d43fb0f1782a17b1fc20 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 22 Jun 2020 13:06:18 -0400 Subject: [PATCH 58/80] open HTTP port iff HTTP server not listening on localhost --- main/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main/main.go b/main/main.go index 98cb581..43fe1b9 100644 --- a/main/main.go +++ b/main/main.go @@ -71,8 +71,10 @@ func main() { mapper := nat.NewDefaultMapper(log, Config.Nat, nat.TCP, "gecko") defer mapper.UnmapAllPorts() - mapper.MapPort(Config.StakingIP.Port, Config.StakingIP.Port) - mapper.MapPort(Config.HTTPPort, Config.HTTPPort) + mapper.MapPort(Config.StakingIP.Port, Config.StakingIP.Port) // Open staking port + if Config.HTTPHost != "127.0.0.1" && Config.HTTPHost != "localhost" { // Open HTTP port iff HTTP server not listening on localhost + mapper.MapPort(Config.HTTPPort, Config.HTTPPort) + } node := node.Node{} From fc40ad802f42e2c6db83443531ab40651360f6de Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 22 Jun 2020 13:18:14 -0400 Subject: [PATCH 59/80] lock, mem and CPU profiles write to a fixed filename --- api/admin/performance.go | 21 +++++++++++++++------ api/admin/service.go | 33 +++++++++------------------------ 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/api/admin/performance.go b/api/admin/performance.go index bf2a460..6035b08 100644 --- a/api/admin/performance.go +++ b/api/admin/performance.go @@ -10,6 +10,15 @@ import ( "runtime/pprof" ) +const ( + // Name of file that CPU profile is written to when StartCPUProfiler called + cpuProfileFile = "cpu.profile" + // Name of file that memory profile is written to when MemoryProfile called + memProfileFile = "mem.profile" + // Name of file that lock profile is written to + lockProfileFile = "lock.profile" +) + var ( errCPUProfilerRunning = errors.New("cpu profiler already running") errCPUProfilerNotRunning = errors.New("cpu profiler doesn't exist") @@ -20,12 +29,12 @@ var ( type Performance struct{ cpuProfileFile *os.File } // StartCPUProfiler starts measuring the cpu utilization of this node -func (p *Performance) StartCPUProfiler(filename string) error { +func (p *Performance) StartCPUProfiler() error { if p.cpuProfileFile != nil { return errCPUProfilerRunning } - file, err := os.Create(filename) + file, err := os.Create(cpuProfileFile) if err != nil { return err } @@ -52,8 +61,8 @@ func (p *Performance) StopCPUProfiler() error { } // MemoryProfile dumps the current memory utilization of this node -func (p *Performance) MemoryProfile(filename string) error { - file, err := os.Create(filename) +func (p *Performance) MemoryProfile() error { + file, err := os.Create(memProfileFile) if err != nil { return err } @@ -66,8 +75,8 @@ func (p *Performance) MemoryProfile(filename string) error { } // LockProfile dumps the current lock statistics of this node -func (p *Performance) LockProfile(filename string) error { - file, err := os.Create(filename) +func (p *Performance) LockProfile() error { + file, err := os.Create(lockProfileFile) if err != nil { return err } diff --git a/api/admin/service.go b/api/admin/service.go index 3d61730..c5059f0 100644 --- a/api/admin/service.go +++ b/api/admin/service.go @@ -39,21 +39,16 @@ func NewService(log logging.Logger, chainManager chains.Manager, peers network.N return &common.HTTPHandler{Handler: newServer} } -// StartCPUProfilerArgs are the arguments for calling StartCPUProfiler -type StartCPUProfilerArgs struct { - Filename string `json:"filename"` -} - // StartCPUProfilerReply are the results from calling StartCPUProfiler type StartCPUProfilerReply struct { Success bool `json:"success"` } // StartCPUProfiler starts a cpu profile writing to the specified file -func (service *Admin) StartCPUProfiler(_ *http.Request, args *StartCPUProfilerArgs, reply *StartCPUProfilerReply) error { - service.log.Info("Admin: StartCPUProfiler called with %s", args.Filename) +func (service *Admin) StartCPUProfiler(_ *http.Request, args *struct{}, reply *StartCPUProfilerReply) error { + service.log.Info("Admin: StartCPUProfiler called") reply.Success = true - return service.performance.StartCPUProfiler(args.Filename) + return service.performance.StartCPUProfiler() } // StopCPUProfilerReply are the results from calling StopCPUProfiler @@ -68,26 +63,16 @@ func (service *Admin) StopCPUProfiler(_ *http.Request, _ *struct{}, reply *StopC return service.performance.StopCPUProfiler() } -// MemoryProfileArgs are the arguments for calling MemoryProfile -type MemoryProfileArgs struct { - Filename string `json:"filename"` -} - // MemoryProfileReply are the results from calling MemoryProfile type MemoryProfileReply struct { Success bool `json:"success"` } // MemoryProfile runs a memory profile writing to the specified file -func (service *Admin) MemoryProfile(_ *http.Request, args *MemoryProfileArgs, reply *MemoryProfileReply) error { - service.log.Info("Admin: MemoryProfile called with %s", args.Filename) +func (service *Admin) MemoryProfile(_ *http.Request, args *struct{}, reply *MemoryProfileReply) error { + service.log.Info("Admin: MemoryProfile called") reply.Success = true - return service.performance.MemoryProfile(args.Filename) -} - -// LockProfileArgs are the arguments for calling LockProfile -type LockProfileArgs struct { - Filename string `json:"filename"` + return service.performance.MemoryProfile() } // LockProfileReply are the results from calling LockProfile @@ -96,10 +81,10 @@ type LockProfileReply struct { } // LockProfile runs a mutex profile writing to the specified file -func (service *Admin) LockProfile(_ *http.Request, args *LockProfileArgs, reply *LockProfileReply) error { - service.log.Info("Admin: LockProfile called with %s", args.Filename) +func (service *Admin) LockProfile(_ *http.Request, args *struct{}, reply *LockProfileReply) error { + service.log.Info("Admin: LockProfile called") reply.Success = true - return service.performance.LockProfile(args.Filename) + return service.performance.LockProfile() } // AliasArgs are the arguments for calling Alias From 3a854ebdecd21a1220a0fe94f3ea7ba21f980d71 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 22 Jun 2020 15:20:35 -0400 Subject: [PATCH 60/80] handler engine gets/sets --- snow/networking/router/handler.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/snow/networking/router/handler.go b/snow/networking/router/handler.go index 9d45baf..03a55f1 100644 --- a/snow/networking/router/handler.go +++ b/snow/networking/router/handler.go @@ -54,6 +54,12 @@ func (h *Handler) Initialize( // Context of this Handler func (h *Handler) Context() *snow.Context { return h.engine.Context() } +// Engine returns the engine this handler dispatches to +func (h *Handler) Engine() common.Engine { return h.engine } + +// SetEngine sets the engine this handler dispatches to +func (h *Handler) SetEngine(engine common.Engine) { h.engine = engine } + // Dispatch waits for incoming messages from the network // and, when they arrive, sends them to the consensus engine func (h *Handler) Dispatch() { From 473bef24b1d44279b9a3b1975c55064d18746fe9 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 22 Jun 2020 15:50:52 -0400 Subject: [PATCH 61/80] removed duplicated batch writes, fixed tests --- database/versiondb/db.go | 3 --- vms/avm/export_tx_test.go | 11 +++++++---- vms/avm/import_tx_test.go | 11 +++++++---- vms/platformvm/vm_test.go | 6 +++--- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/database/versiondb/db.go b/database/versiondb/db.go index 7223c55..b2b5be1 100644 --- a/database/versiondb/db.go +++ b/database/versiondb/db.go @@ -234,9 +234,6 @@ func (db *Database) commitBatch() (database.Batch, error) { return nil, err } } - if err := db.batch.Write(); err != nil { - return nil, err - } return db.batch, nil } diff --git a/vms/avm/export_tx_test.go b/vms/avm/export_tx_test.go index 75b359f..96c7733 100644 --- a/vms/avm/export_tx_test.go +++ b/vms/avm/export_tx_test.go @@ -9,6 +9,7 @@ import ( "github.com/ava-labs/gecko/chains/atomic" "github.com/ava-labs/gecko/database/memdb" + "github.com/ava-labs/gecko/database/prefixdb" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/snow/engine/common" @@ -117,9 +118,10 @@ func TestIssueExportTx(t *testing.T) { genesisBytes := BuildGenesisTest(t) issuer := make(chan common.Message, 1) + baseDB := memdb.New() sm := &atomic.SharedMemory{} - sm.Initialize(logging.NoLog{}, memdb.New()) + sm.Initialize(logging.NoLog{}, prefixdb.New([]byte{0}, baseDB)) ctx := snow.DefaultContextTest() ctx.NetworkID = networkID @@ -138,7 +140,7 @@ func TestIssueExportTx(t *testing.T) { } err := vm.Initialize( ctx, - memdb.New(), + prefixdb.New([]byte{1}, baseDB), genesisBytes, issuer, []*common.Fx{{ @@ -273,9 +275,10 @@ func TestClearForceAcceptedExportTx(t *testing.T) { genesisBytes := BuildGenesisTest(t) issuer := make(chan common.Message, 1) + baseDB := memdb.New() sm := &atomic.SharedMemory{} - sm.Initialize(logging.NoLog{}, memdb.New()) + sm.Initialize(logging.NoLog{}, prefixdb.New([]byte{0}, baseDB)) ctx := snow.DefaultContextTest() ctx.NetworkID = networkID @@ -294,7 +297,7 @@ func TestClearForceAcceptedExportTx(t *testing.T) { } err := vm.Initialize( ctx, - memdb.New(), + prefixdb.New([]byte{1}, baseDB), genesisBytes, issuer, []*common.Fx{{ diff --git a/vms/avm/import_tx_test.go b/vms/avm/import_tx_test.go index e510aff..750d402 100644 --- a/vms/avm/import_tx_test.go +++ b/vms/avm/import_tx_test.go @@ -9,6 +9,7 @@ import ( "github.com/ava-labs/gecko/chains/atomic" "github.com/ava-labs/gecko/database/memdb" + "github.com/ava-labs/gecko/database/prefixdb" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/snow/engine/common" @@ -106,9 +107,10 @@ func TestIssueImportTx(t *testing.T) { genesisBytes := BuildGenesisTest(t) issuer := make(chan common.Message, 1) + baseDB := memdb.New() sm := &atomic.SharedMemory{} - sm.Initialize(logging.NoLog{}, memdb.New()) + sm.Initialize(logging.NoLog{}, prefixdb.New([]byte{0}, baseDB)) ctx := snow.DefaultContextTest() ctx.NetworkID = networkID @@ -127,7 +129,7 @@ func TestIssueImportTx(t *testing.T) { } err := vm.Initialize( ctx, - memdb.New(), + prefixdb.New([]byte{1}, baseDB), genesisBytes, issuer, []*common.Fx{{ @@ -265,9 +267,10 @@ func TestForceAcceptImportTx(t *testing.T) { genesisBytes := BuildGenesisTest(t) issuer := make(chan common.Message, 1) + baseDB := memdb.New() sm := &atomic.SharedMemory{} - sm.Initialize(logging.NoLog{}, memdb.New()) + sm.Initialize(logging.NoLog{}, prefixdb.New([]byte{0}, baseDB)) ctx := snow.DefaultContextTest() ctx.NetworkID = networkID @@ -285,7 +288,7 @@ func TestForceAcceptImportTx(t *testing.T) { err := vm.Initialize( ctx, - memdb.New(), + prefixdb.New([]byte{1}, baseDB), genesisBytes, issuer, []*common.Fx{{ diff --git a/vms/platformvm/vm_test.go b/vms/platformvm/vm_test.go index dcee89a..82989eb 100644 --- a/vms/platformvm/vm_test.go +++ b/vms/platformvm/vm_test.go @@ -137,7 +137,7 @@ func defaultVM() *VM { vm.validators.PutValidatorSet(DefaultSubnetID, defaultSubnet) vm.clock.Set(defaultGenesisTime) - db := memdb.New() + db := prefixdb.New([]byte{0}, memdb.New()) msgChan := make(chan common.Message, 1) ctx := defaultContext() ctx.Lock.Lock() @@ -1189,7 +1189,7 @@ func TestAtomicImport(t *testing.T) { key := keys[0] sm := &atomic.SharedMemory{} - sm.Initialize(logging.NoLog{}, memdb.New()) + sm.Initialize(logging.NoLog{}, prefixdb.New([]byte{0}, vm.DB.GetDatabase())) vm.Ctx.SharedMemory = sm.NewBlockchainSharedMemory(vm.Ctx.ChainID) @@ -1282,7 +1282,7 @@ func TestOptimisticAtomicImport(t *testing.T) { key := keys[0] sm := &atomic.SharedMemory{} - sm.Initialize(logging.NoLog{}, memdb.New()) + sm.Initialize(logging.NoLog{}, prefixdb.New([]byte{0}, vm.DB.GetDatabase())) vm.Ctx.SharedMemory = sm.NewBlockchainSharedMemory(vm.Ctx.ChainID) From fc15e3cfe69eb46d21dfb40753bbcffa0ef81c43 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 22 Jun 2020 16:35:42 -0400 Subject: [PATCH 62/80] prevent potential memory leaks --- database/encdb/db.go | 6 +++++- database/memdb/db.go | 9 ++++++--- database/prefixdb/db.go | 6 +++++- database/rpcdb/db_client.go | 6 +++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/database/encdb/db.go b/database/encdb/db.go index fe33fa7..ddf47e0 100644 --- a/database/encdb/db.go +++ b/database/encdb/db.go @@ -17,6 +17,10 @@ import ( "github.com/ava-labs/gecko/utils/hashing" ) +const ( + minBatchSize = 32 +) + // Database encrypts all values that are provided type Database struct { lock sync.RWMutex @@ -201,7 +205,7 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { - b.writes = b.writes[:0] + b.writes = make([]keyValue, 0, minBatchSize) b.Batch.Reset() } diff --git a/database/memdb/db.go b/database/memdb/db.go index de0cae3..5bbd3a2 100644 --- a/database/memdb/db.go +++ b/database/memdb/db.go @@ -13,8 +13,11 @@ import ( "github.com/ava-labs/gecko/utils" ) -// DefaultSize is the default initial size of the memory database -const DefaultSize = 1 << 10 +const ( + // DefaultSize is the default initial size of the memory database + DefaultSize = 1 << 10 + minBatchSize = 32 +) // Database is an ephemeral key-value store that implements the Database // interface. @@ -191,7 +194,7 @@ func (b *batch) Write() error { // Reset implements the Batch interface func (b *batch) Reset() { - b.writes = b.writes[:0] + b.writes = make([]keyValue, 0, minBatchSize) b.size = 0 } diff --git a/database/prefixdb/db.go b/database/prefixdb/db.go index 34bc50d..a413846 100644 --- a/database/prefixdb/db.go +++ b/database/prefixdb/db.go @@ -12,6 +12,10 @@ import ( "github.com/ava-labs/gecko/utils/hashing" ) +const ( + minBatchSize = 32 +) + // Database partitions a database into a sub-database by prefixing all keys with // a unique value. type Database struct { @@ -199,7 +203,7 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { - b.writes = b.writes[:0] + b.writes = make([]keyValue, 0, minBatchSize) b.Batch.Reset() } diff --git a/database/rpcdb/db_client.go b/database/rpcdb/db_client.go index dc3f60b..f1a3abc 100644 --- a/database/rpcdb/db_client.go +++ b/database/rpcdb/db_client.go @@ -14,6 +14,10 @@ import ( "github.com/ava-labs/gecko/utils" ) +const ( + minBatchSize = 32 +) + var ( errClosed = fmt.Sprintf("rpc error: code = Unknown desc = %s", database.ErrClosed) errNotFound = fmt.Sprintf("rpc error: code = Unknown desc = %s", database.ErrNotFound) @@ -180,7 +184,7 @@ func (b *batch) Write() error { } func (b *batch) Reset() { - b.writes = b.writes[:0] + b.writes = make([]keyValue, 0, minBatchSize) b.size = 0 } From c9aa8eedc2a17b6067c7d1df32940ea132710ef1 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 22 Jun 2020 16:50:31 -0400 Subject: [PATCH 63/80] pre-allocate arrays --- ids/short_set.go | 18 +++++++++++++----- snow/validators/set.go | 7 ++++--- vms/platformvm/vm.go | 8 +++++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/ids/short_set.go b/ids/short_set.go index 6977863..90766cd 100644 --- a/ids/short_set.go +++ b/ids/short_set.go @@ -57,15 +57,23 @@ func (ids *ShortSet) Remove(idList ...ShortID) { // Clear empties this set func (ids *ShortSet) Clear() { *ids = nil } -// CappedList returns a list of length at most [size]. Size should be >= 0 +// CappedList returns a list of length at most [size]. +// Size should be >= 0. If size < 0, returns empty list. func (ids ShortSet) CappedList(size int) []ShortID { - idList := make([]ShortID, size)[:0] + if size < 0 { + return make([]ShortID, 0, 0) + } + if l := ids.Len(); l < size { + size = l + } + i := 0 + idList := make([]ShortID, size) for id := range ids { - if size <= 0 { + if i >= size { break } - size-- - idList = append(idList, NewShortID(id)) + idList[i] = NewShortID(id) + i++ } return idList } diff --git a/snow/validators/set.go b/snow/validators/set.go index 50210bf..c33395f 100644 --- a/snow/validators/set.go +++ b/snow/validators/set.go @@ -71,9 +71,10 @@ func (s *set) Set(vdrs []Validator) { } func (s *set) set(vdrs []Validator) { - s.vdrMap = make(map[[20]byte]int, len(vdrs)) - s.vdrSlice = s.vdrSlice[:0] - s.sampler.Weights = s.sampler.Weights[:0] + lenVdrs := len(vdrs) + s.vdrMap = make(map[[20]byte]int, lenVdrs) + s.vdrSlice = make([]Validator, 0, lenVdrs) + s.sampler.Weights = make([]uint64, 0, lenVdrs) for _, vdr := range vdrs { s.add(vdr) diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go index baff040..01bb6a4 100644 --- a/vms/platformvm/vm.go +++ b/vms/platformvm/vm.go @@ -18,6 +18,7 @@ import ( "github.com/ava-labs/gecko/snow/consensus/snowman" "github.com/ava-labs/gecko/snow/engine/common" "github.com/ava-labs/gecko/snow/validators" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/utils/crypto" "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/utils/logging" @@ -26,7 +27,6 @@ import ( "github.com/ava-labs/gecko/utils/units" "github.com/ava-labs/gecko/utils/wrappers" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/core" "github.com/ava-labs/gecko/vms/secp256k1fx" ) @@ -808,9 +808,11 @@ func (vm *VM) getValidators(validatorEvents *EventHeap) []validators.Validator { validator.Wght = weight } - vdrList := make([]validators.Validator, len(vdrMap))[:0] + vdrList := make([]validators.Validator, len(vdrMap), len(vdrMap)) + i := 0 for _, validator := range vdrMap { - vdrList = append(vdrList, validator) + vdrList[i] = validator + i++ } return vdrList } From 5b6debbabad459dce10611e71fea7a58c3a33660 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 22 Jun 2020 18:08:20 -0400 Subject: [PATCH 64/80] added regression test --- database/versiondb/db_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/database/versiondb/db_test.go b/database/versiondb/db_test.go index 70cf8ff..0c284e2 100644 --- a/database/versiondb/db_test.go +++ b/database/versiondb/db_test.go @@ -299,6 +299,10 @@ func TestCommitBatch(t *testing.T) { if err := db.Put(key1, value1); err != nil { t.Fatalf("Unexpected error on db.Put: %s", err) + } else if has, err := baseDB.Has(key1); err != nil { + t.Fatalf("Unexpected error on db.Has: %s", err) + } else if has { + t.Fatalf("Unexpected result of db.Has: %v", has) } batch, err := db.CommitBatch() @@ -307,7 +311,11 @@ func TestCommitBatch(t *testing.T) { } db.Abort() - if err := batch.Write(); err != nil { + if has, err := baseDB.Has(key1); err != nil { + t.Fatalf("Unexpected error on db.Has: %s", err) + } else if has { + t.Fatalf("Unexpected result of db.Has: %v", has) + } else if err := batch.Write(); err != nil { t.Fatalf("Unexpected error on batch.Write: %s", err) } From 7ef37af0d666abaa791a407be1a2a8142c9c5737 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 22 Jun 2020 18:14:35 -0400 Subject: [PATCH 65/80] changed test to enforce abortions --- database/versiondb/db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/versiondb/db_test.go b/database/versiondb/db_test.go index 0c284e2..345655a 100644 --- a/database/versiondb/db_test.go +++ b/database/versiondb/db_test.go @@ -311,7 +311,7 @@ func TestCommitBatch(t *testing.T) { } db.Abort() - if has, err := baseDB.Has(key1); err != nil { + if has, err := db.Has(key1); err != nil { t.Fatalf("Unexpected error on db.Has: %s", err) } else if has { t.Fatalf("Unexpected result of db.Has: %v", has) From 998f4bff40b89cc474c4ac276ce3f0d34e1a2e9e Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 13:03:23 -0400 Subject: [PATCH 66/80] add comments; remove unnceccessary batch write; avoid possible memory leak; reset batch after write --- database/versiondb/db.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/database/versiondb/db.go b/database/versiondb/db.go index 7223c55..050de0a 100644 --- a/database/versiondb/db.go +++ b/database/versiondb/db.go @@ -195,6 +195,7 @@ func (db *Database) Commit() error { if err := batch.Write(); err != nil { return err } + batch.Reset() db.abort() return nil } @@ -209,9 +210,10 @@ func (db *Database) Abort() { func (db *Database) abort() { db.mem = make(map[string]valueDelete, memdb.DefaultSize) } -// CommitBatch returns a batch that will commit all pending writes to the -// underlying database. The returned batch should be written before future calls -// to this DB unless the batch will never be written. +// CommitBatch returns a batch that contains all uncommitted puts/deletes. +// Calling Write() on the returned batch causes the puts/deletes to be +// written to the underlying database. The returned batch should be written before +// future calls to this DB unless the batch will never be written. func (db *Database) CommitBatch() (database.Batch, error) { db.lock.Lock() defer db.lock.Unlock() @@ -219,6 +221,8 @@ func (db *Database) CommitBatch() (database.Batch, error) { return db.commitBatch() } +// Put all of the puts/deletes in memory into db.batch +// and return the batch func (db *Database) commitBatch() (database.Batch, error) { if db.mem == nil { return nil, database.ErrClosed @@ -234,9 +238,6 @@ func (db *Database) commitBatch() (database.Batch, error) { return nil, err } } - if err := db.batch.Write(); err != nil { - return nil, err - } return db.batch, nil } @@ -249,6 +250,7 @@ func (db *Database) Close() error { if db.mem == nil { return database.ErrClosed } + db.batch = nil db.mem = nil db.db = nil return nil @@ -303,7 +305,7 @@ func (b *batch) Write() error { // Reset implements the Database interface func (b *batch) Reset() { - b.writes = b.writes[:0] + b.writes = make([]keyValue, 0) b.size = 0 } From f92fa88d242ac7d7db38ef9066948926c03d659d Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 13:04:10 -0400 Subject: [PATCH 67/80] commit db after parsing tx to avoid memory leak --- vms/avm/vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/avm/vm.go b/vms/avm/vm.go index 715ce95..026760c 100644 --- a/vms/avm/vm.go +++ b/vms/avm/vm.go @@ -492,10 +492,10 @@ func (vm *VM) parseTx(b []byte) (*UniqueTx, error) { if err := vm.state.SetTx(tx.ID(), tx.Tx); err != nil { return nil, err } - if err := tx.setStatus(choices.Processing); err != nil { return nil, err } + return tx, vm.db.Commit() } return tx, nil From 55079aa893e91f4d15dab6342a4d02fb1aef0dd3 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 15:01:55 -0400 Subject: [PATCH 68/80] add CappedList for ids.Set and use it in fetchANeededVtx --- ids/set.go | 23 +++++++++++++- ids/set_test.go | 43 +++++++++++++++++++++++++++ snow/engine/avalanche/bootstrapper.go | 2 +- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/ids/set.go b/ids/set.go index c3aa024..d632949 100644 --- a/ids/set.go +++ b/ids/set.go @@ -78,7 +78,7 @@ func (ids *Set) Clear() { *ids = nil } // List converts this set into a list func (ids Set) List() []ID { - idList := make([]ID, ids.Len(), ids.Len()) + idList := make([]ID, ids.Len()) i := 0 for id := range ids { idList[i] = NewID(id) @@ -87,6 +87,27 @@ func (ids Set) List() []ID { return idList } +// CappedList returns a list of length at most [size]. +// Size should be >= 0. If size < 0, returns nil. +func (ids Set) CappedList(size int) []ID { + if size < 0 { + return nil + } + if l := ids.Len(); l < size { + size = l + } + i := 0 + idList := make([]ID, size) + for id := range ids { + if i >= size { + break + } + idList[i] = NewID(id) + i++ + } + return idList +} + // Equals returns true if the sets contain the same elements func (ids Set) Equals(oIDs Set) bool { if ids.Len() != oIDs.Len() { diff --git a/ids/set_test.go b/ids/set_test.go index 3c7ab15..b4e05db 100644 --- a/ids/set_test.go +++ b/ids/set_test.go @@ -55,3 +55,46 @@ func TestSet(t *testing.T) { t.Fatalf("Sets overlap") } } + +func TestSetCappedList(t *testing.T) { + set := Set{} + + id := Empty + + if list := set.CappedList(0); len(list) != 0 { + t.Fatalf("List should have been empty but was %v", list) + } + + set.Add(id) + + if list := set.CappedList(0); len(list) != 0 { + t.Fatalf("List should have been empty but was %v", list) + } else if list := set.CappedList(1); len(list) != 1 { + t.Fatalf("List should have had length %d but had %d", 1, len(list)) + } else if returnedID := list[0]; !id.Equals(returnedID) { + t.Fatalf("List should have been %s but was %s", id, returnedID) + } else if list := set.CappedList(2); len(list) != 1 { + t.Fatalf("List should have had length %d but had %d", 1, len(list)) + } else if returnedID := list[0]; !id.Equals(returnedID) { + t.Fatalf("List should have been %s but was %s", id, returnedID) + } + + id2 := NewID([32]byte{1}) + set.Add(id2) + + if list := set.CappedList(0); len(list) != 0 { + t.Fatalf("List should have been empty but was %v", list) + } else if list := set.CappedList(1); len(list) != 1 { + t.Fatalf("List should have had length %d but had %d", 1, len(list)) + } else if returnedID := list[0]; !id.Equals(returnedID) && !id2.Equals(returnedID) { + t.Fatalf("List should have been %s but was %s", id, returnedID) + } else if list := set.CappedList(2); len(list) != 2 { + t.Fatalf("List should have had length %d but had %d", 2, len(list)) + } else if list := set.CappedList(3); len(list) != 2 { + t.Fatalf("List should have had length %d but had %d", 2, len(list)) + } else if returnedID := list[0]; !id.Equals(returnedID) && !id2.Equals(returnedID) { + t.Fatalf("list contains unexpected element %s", returnedID) + } else if returnedID := list[1]; !id.Equals(returnedID) && !id2.Equals(returnedID) { + t.Fatalf("list contains unexpected element %s", returnedID) + } +} diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index 749694e..352f40b 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -111,7 +111,7 @@ func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set { // Calls fetch for a pending vertex if there are any func (b *bootstrapper) fetchANeededVtx() error { if b.needToFetch.Len() > 0 { - return b.fetch(b.needToFetch.List()[0]) + return b.fetch(b.needToFetch.CappedList(1)[0]) } return nil } From 7f5693dfd33fc31839c28ca5d94f44429b7218bc Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 15:08:15 -0400 Subject: [PATCH 69/80] reduce MaxTimeFetchingAncestors from 100ms to 50ms --- snow/engine/common/bootstrapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snow/engine/common/bootstrapper.go b/snow/engine/common/bootstrapper.go index 8c9c745..f1f58db 100644 --- a/snow/engine/common/bootstrapper.go +++ b/snow/engine/common/bootstrapper.go @@ -24,7 +24,7 @@ const ( var ( // MaxTimeFetchingAncestors is the maximum amount of time to spend fetching vertices during a call to GetAncestors - MaxTimeFetchingAncestors = 100 * time.Millisecond + MaxTimeFetchingAncestors = 50 * time.Millisecond ) // Bootstrapper implements the Engine interface. From 6c6136d5512f89ee99c796fb74757052dd6da79e Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 16:44:02 -0400 Subject: [PATCH 70/80] only downsize underlying arrays if they're much too large --- database/common.go | 14 ++++++++++++++ database/encdb/db.go | 10 +++++----- database/memdb/db.go | 9 ++++++--- database/prefixdb/db.go | 10 +++++----- database/rpcdb/db_client.go | 10 +++++----- database/versiondb/db.go | 6 +++++- ids/short_set.go | 4 ++-- snow/validators/set.go | 35 +++++++++++++++++++++++++++++++++-- vms/platformvm/vm.go | 2 +- 9 files changed, 76 insertions(+), 24 deletions(-) create mode 100644 database/common.go diff --git a/database/common.go b/database/common.go new file mode 100644 index 0000000..26b0531 --- /dev/null +++ b/database/common.go @@ -0,0 +1,14 @@ +package database + +const ( + // MaxExcessCapacityFactor ... + // If, when a batch is reset, the cap(batch)/len(batch) > MaxExcessCapacityFactor, + // the underlying array's capacity will be reduced by a factor of capacityReductionFactor. + // Higher value for MaxExcessCapacityFactor --> less aggressive array downsizing --> less memory allocations + // but more unnecessary data in the underlying array that can't be garbage collected. + // Higher value for CapacityReductionFactor --> more aggressive array downsizing --> more memory allocations + // but less unnecessary data in the underlying array that can't be garbage collected. + MaxExcessCapacityFactor = 4 + // CapacityReductionFactor ... + CapacityReductionFactor = 2 +) diff --git a/database/encdb/db.go b/database/encdb/db.go index ddf47e0..8f0d8e3 100644 --- a/database/encdb/db.go +++ b/database/encdb/db.go @@ -17,10 +17,6 @@ import ( "github.com/ava-labs/gecko/utils/hashing" ) -const ( - minBatchSize = 32 -) - // Database encrypts all values that are provided type Database struct { lock sync.RWMutex @@ -205,7 +201,11 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { - b.writes = make([]keyValue, 0, minBatchSize) + if cap(b.writes) > len(b.writes)*database.MaxExcessCapacityFactor { + b.writes = make([]keyValue, 0, cap(b.writes)/database.CapacityReductionFactor) + } else { + b.writes = b.writes[:0] + } b.Batch.Reset() } diff --git a/database/memdb/db.go b/database/memdb/db.go index 5bbd3a2..94ba395 100644 --- a/database/memdb/db.go +++ b/database/memdb/db.go @@ -15,8 +15,7 @@ import ( const ( // DefaultSize is the default initial size of the memory database - DefaultSize = 1 << 10 - minBatchSize = 32 + DefaultSize = 1 << 10 ) // Database is an ephemeral key-value store that implements the Database @@ -194,7 +193,11 @@ func (b *batch) Write() error { // Reset implements the Batch interface func (b *batch) Reset() { - b.writes = make([]keyValue, 0, minBatchSize) + if cap(b.writes) > len(b.writes)*database.MaxExcessCapacityFactor { + b.writes = make([]keyValue, 0, cap(b.writes)/database.CapacityReductionFactor) + } else { + b.writes = b.writes[:0] + } b.size = 0 } diff --git a/database/prefixdb/db.go b/database/prefixdb/db.go index a413846..7f606b2 100644 --- a/database/prefixdb/db.go +++ b/database/prefixdb/db.go @@ -12,10 +12,6 @@ import ( "github.com/ava-labs/gecko/utils/hashing" ) -const ( - minBatchSize = 32 -) - // Database partitions a database into a sub-database by prefixing all keys with // a unique value. type Database struct { @@ -203,7 +199,11 @@ func (b *batch) Write() error { // Reset resets the batch for reuse. func (b *batch) Reset() { - b.writes = make([]keyValue, 0, minBatchSize) + if cap(b.writes) > len(b.writes)*database.MaxExcessCapacityFactor { + b.writes = make([]keyValue, 0, cap(b.writes)/database.CapacityReductionFactor) + } else { + b.writes = b.writes[:0] + } b.Batch.Reset() } diff --git a/database/rpcdb/db_client.go b/database/rpcdb/db_client.go index f1a3abc..401e404 100644 --- a/database/rpcdb/db_client.go +++ b/database/rpcdb/db_client.go @@ -14,10 +14,6 @@ import ( "github.com/ava-labs/gecko/utils" ) -const ( - minBatchSize = 32 -) - var ( errClosed = fmt.Sprintf("rpc error: code = Unknown desc = %s", database.ErrClosed) errNotFound = fmt.Sprintf("rpc error: code = Unknown desc = %s", database.ErrNotFound) @@ -184,7 +180,11 @@ func (b *batch) Write() error { } func (b *batch) Reset() { - b.writes = make([]keyValue, 0, minBatchSize) + if cap(b.writes) > len(b.writes)*database.MaxExcessCapacityFactor { + b.writes = make([]keyValue, 0, cap(b.writes)/database.CapacityReductionFactor) + } else { + b.writes = b.writes[:0] + } b.size = 0 } diff --git a/database/versiondb/db.go b/database/versiondb/db.go index 050de0a..a1f9a18 100644 --- a/database/versiondb/db.go +++ b/database/versiondb/db.go @@ -305,7 +305,11 @@ func (b *batch) Write() error { // Reset implements the Database interface func (b *batch) Reset() { - b.writes = make([]keyValue, 0) + if cap(b.writes) > len(b.writes)*database.MaxExcessCapacityFactor { + b.writes = make([]keyValue, 0, cap(b.writes)/database.CapacityReductionFactor) + } else { + b.writes = b.writes[:0] + } b.size = 0 } diff --git a/ids/short_set.go b/ids/short_set.go index 90766cd..9bcd37d 100644 --- a/ids/short_set.go +++ b/ids/short_set.go @@ -58,10 +58,10 @@ func (ids *ShortSet) Remove(idList ...ShortID) { func (ids *ShortSet) Clear() { *ids = nil } // CappedList returns a list of length at most [size]. -// Size should be >= 0. If size < 0, returns empty list. +// Size should be >= 0. If size < 0, returns nil. func (ids ShortSet) CappedList(size int) []ShortID { if size < 0 { - return make([]ShortID, 0, 0) + return nil } if l := ids.Len(); l < size { size = l diff --git a/snow/validators/set.go b/snow/validators/set.go index c33395f..4fddf98 100644 --- a/snow/validators/set.go +++ b/snow/validators/set.go @@ -13,6 +13,19 @@ import ( "github.com/ava-labs/gecko/utils/random" ) +const ( + // maxExcessCapacityFactor ... + // If, when the validator set is reset, cap(set)/len(set) > MaxExcessCapacityFactor, + // the underlying arrays' capacities will be reduced by a factor of capacityReductionFactor. + // Higher value for maxExcessCapacityFactor --> less aggressive array downsizing --> less memory allocations + // but more unnecessary data in the underlying array that can't be garbage collected. + // Higher value for capacityReductionFactor --> more aggressive array downsizing --> more memory allocations + // but less unnecessary data in the underlying array that can't be garbage collected. + maxExcessCapacityFactor = 4 + // CapacityReductionFactor ... + capacityReductionFactor = 2 +) + // Set of validators that can be sampled type Set interface { fmt.Stringer @@ -72,9 +85,27 @@ func (s *set) Set(vdrs []Validator) { func (s *set) set(vdrs []Validator) { lenVdrs := len(vdrs) + // If the underlying arrays are much larger than necessary, resize them to + // allow garbage collection of unused memory + if cap(s.vdrSlice) > len(s.vdrSlice)*maxExcessCapacityFactor { + newCap := cap(s.vdrSlice) / capacityReductionFactor + if newCap < lenVdrs { + newCap = lenVdrs + } + s.vdrSlice = make([]Validator, 0, newCap) + } else { + s.vdrSlice = s.vdrSlice[:0] + } + if cap(s.sampler.Weights) > len(s.sampler.Weights)*maxExcessCapacityFactor { + newCap := cap(s.sampler.Weights) / capacityReductionFactor + if newCap < lenVdrs { + newCap = lenVdrs + } + s.sampler.Weights = make([]uint64, 0, newCap) + } else { + s.sampler.Weights = s.sampler.Weights[:0] + } s.vdrMap = make(map[[20]byte]int, lenVdrs) - s.vdrSlice = make([]Validator, 0, lenVdrs) - s.sampler.Weights = make([]uint64, 0, lenVdrs) for _, vdr := range vdrs { s.add(vdr) diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go index 01bb6a4..8b9350f 100644 --- a/vms/platformvm/vm.go +++ b/vms/platformvm/vm.go @@ -808,7 +808,7 @@ func (vm *VM) getValidators(validatorEvents *EventHeap) []validators.Validator { validator.Wght = weight } - vdrList := make([]validators.Validator, len(vdrMap), len(vdrMap)) + vdrList := make([]validators.Validator, len(vdrMap)) i := 0 for _, validator := range vdrMap { vdrList[i] = validator From 8ce7bda92afb35b663b2db8a0aba34e422eef276 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 16:54:03 -0400 Subject: [PATCH 71/80] cleanup --- snow/validators/set.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/snow/validators/set.go b/snow/validators/set.go index 4fddf98..edaecd7 100644 --- a/snow/validators/set.go +++ b/snow/validators/set.go @@ -93,17 +93,11 @@ func (s *set) set(vdrs []Validator) { newCap = lenVdrs } s.vdrSlice = make([]Validator, 0, newCap) - } else { - s.vdrSlice = s.vdrSlice[:0] - } - if cap(s.sampler.Weights) > len(s.sampler.Weights)*maxExcessCapacityFactor { - newCap := cap(s.sampler.Weights) / capacityReductionFactor - if newCap < lenVdrs { - newCap = lenVdrs - } s.sampler.Weights = make([]uint64, 0, newCap) } else { + s.vdrSlice = s.vdrSlice[:0] s.sampler.Weights = s.sampler.Weights[:0] + } s.vdrMap = make(map[[20]byte]int, lenVdrs) From 875b2d0cab12be0bdefb7561e60ee8046e1ac3ad Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 16:54:25 -0400 Subject: [PATCH 72/80] remove errant newline --- snow/validators/set.go | 1 - 1 file changed, 1 deletion(-) diff --git a/snow/validators/set.go b/snow/validators/set.go index edaecd7..610a85f 100644 --- a/snow/validators/set.go +++ b/snow/validators/set.go @@ -97,7 +97,6 @@ func (s *set) set(vdrs []Validator) { } else { s.vdrSlice = s.vdrSlice[:0] s.sampler.Weights = s.sampler.Weights[:0] - } s.vdrMap = make(map[[20]byte]int, lenVdrs) From fa11fecbb0a0ddde986d858e112017e3bd507b6d Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 17:15:25 -0400 Subject: [PATCH 73/80] pre-allocate map capacity in consensus --- snow/consensus/avalanche/topological.go | 18 +++++++++++------- snow/consensus/snowman/topological.go | 6 +++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/snow/consensus/avalanche/topological.go b/snow/consensus/avalanche/topological.go index d786a0f..b8d128e 100644 --- a/snow/consensus/avalanche/topological.go +++ b/snow/consensus/avalanche/topological.go @@ -10,6 +10,10 @@ import ( "github.com/ava-labs/gecko/snow/consensus/snowstorm" ) +const ( + minMapSize = 16 +) + // TopologicalFactory implements Factory by returning a topological struct type TopologicalFactory struct{} @@ -65,12 +69,12 @@ func (ta *Topological) Initialize(ctx *snow.Context, params Parameters, frontier ta.ctx.Log.Error("%s", err) } - ta.nodes = make(map[[32]byte]Vertex) + ta.nodes = make(map[[32]byte]Vertex, minMapSize) ta.cg = &snowstorm.Directed{} ta.cg.Initialize(ctx, params.Parameters) - ta.frontier = make(map[[32]byte]Vertex) + ta.frontier = make(map[[32]byte]Vertex, minMapSize) for _, vtx := range frontier { ta.frontier[vtx.ID().Key()] = vtx } @@ -159,7 +163,7 @@ func (ta *Topological) Finalized() bool { return ta.cg.Finalized() } // the non-transitively applied votes. Also returns the list of leaf nodes. func (ta *Topological) calculateInDegree( responses ids.UniqueBag) (map[[32]byte]kahnNode, []ids.ID) { - kahns := make(map[[32]byte]kahnNode) + kahns := make(map[[32]byte]kahnNode, minMapSize) leaves := ids.Set{} for _, vote := range responses.List() { @@ -233,7 +237,7 @@ func (ta *Topological) pushVotes( kahnNodes map[[32]byte]kahnNode, leaves []ids.ID) ids.Bag { votes := make(ids.UniqueBag) - txConflicts := make(map[[32]byte]ids.Set) + txConflicts := make(map[[32]byte]ids.Set, minMapSize) for len(leaves) > 0 { newLeavesSize := len(leaves) - 1 @@ -443,9 +447,9 @@ func (ta *Topological) updateFrontiers() error { ta.preferred.Clear() ta.virtuous.Clear() ta.orphans.Clear() - ta.frontier = make(map[[32]byte]Vertex) - ta.preferenceCache = make(map[[32]byte]bool) - ta.virtuousCache = make(map[[32]byte]bool) + ta.frontier = make(map[[32]byte]Vertex, minMapSize) + ta.preferenceCache = make(map[[32]byte]bool, minMapSize) + ta.virtuousCache = make(map[[32]byte]bool, minMapSize) ta.orphans.Union(ta.cg.Virtuous()) // Initially, nothing is preferred diff --git a/snow/consensus/snowman/topological.go b/snow/consensus/snowman/topological.go index 6f98751..51612db 100644 --- a/snow/consensus/snowman/topological.go +++ b/snow/consensus/snowman/topological.go @@ -9,6 +9,10 @@ import ( "github.com/ava-labs/gecko/snow/consensus/snowball" ) +const ( + minMapSize = 16 +) + // TopologicalFactory implements Factory by returning a topological struct type TopologicalFactory struct{} @@ -183,7 +187,7 @@ func (ts *Topological) Finalized() bool { return len(ts.blocks) == 1 } // the non-transitively applied votes. Also returns the list of leaf blocks. func (ts *Topological) calculateInDegree( votes ids.Bag) (map[[32]byte]kahnNode, []ids.ID) { - kahns := make(map[[32]byte]kahnNode) + kahns := make(map[[32]byte]kahnNode, minMapSize) leaves := ids.Set{} for _, vote := range votes.List() { From 3d374a73db1e55e95a2dc51620d70c4057436a51 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 23 Jun 2020 17:30:45 -0400 Subject: [PATCH 74/80] enable keystore by default --- main/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/params.go b/main/params.go index 877d406..4d55919 100644 --- a/main/params.go +++ b/main/params.go @@ -227,7 +227,7 @@ func init() { // Enable/Disable APIs: fs.BoolVar(&Config.AdminAPIEnabled, "api-admin-enabled", false, "If true, this node exposes the Admin API") fs.BoolVar(&Config.InfoAPIEnabled, "api-info-enabled", true, "If true, this node exposes the Info API") - fs.BoolVar(&Config.KeystoreAPIEnabled, "api-keystore-enabled", false, "If true, this node exposes the Keystore API") + fs.BoolVar(&Config.KeystoreAPIEnabled, "api-keystore-enabled", true, "If true, this node exposes the Keystore API") fs.BoolVar(&Config.MetricsAPIEnabled, "api-metrics-enabled", true, "If true, this node exposes the Metrics API") fs.BoolVar(&Config.HealthAPIEnabled, "api-health-enabled", true, "If true, this node exposes the Health API") fs.BoolVar(&Config.IPCEnabled, "api-ipcs-enabled", false, "If true, IPCs can be opened") From 7591e93e08182bb24f5314d28973341a311f8ffd Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 23 Jun 2020 17:36:20 -0400 Subject: [PATCH 75/80] remove go-plugin logging; use coreth 0.2.5 --- go.mod | 2 +- go.sum | 2 ++ scripts/build.sh | 2 +- vms/rpcchainvm/factory.go | 5 +++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a8a8f39..c8d28c2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/AppsFlyer/go-sundheit v0.2.0 github.com/allegro/bigcache v1.2.1 // indirect github.com/aristanetworks/goarista v0.0.0-20200520141224-0f14e646773f // indirect - github.com/ava-labs/coreth v0.2.4 // Added manually; don't delete + github.com/ava-labs/coreth v0.2.5 // Added manually; don't delete github.com/ava-labs/go-ethereum v1.9.3 // indirect github.com/deckarep/golang-set v1.7.1 // indirect github.com/decred/dcrd/dcrec/secp256k1 v1.0.3 diff --git a/go.sum b/go.sum index 3f809ed..fb197cd 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/aristanetworks/goarista v0.0.0-20200520141224-0f14e646773f/go.mod h1: github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/ava-labs/coreth v0.2.4 h1:MhnbuRyMcij7WU4+frayp40quc44AMPc4IrxXhmucWw= github.com/ava-labs/coreth v0.2.4/go.mod h1:pGolKipwq5vGIY2IBBcBkMYrqniXMsS5SBn+BBi4+Js= +github.com/ava-labs/coreth v0.2.5 h1:2Al753rpPHvvZfcz7w96YbKhGFvrcZzsIZ/sIp0A0Ao= +github.com/ava-labs/coreth v0.2.5/go.mod h1:pGolKipwq5vGIY2IBBcBkMYrqniXMsS5SBn+BBi4+Js= github.com/ava-labs/go-ethereum v1.9.3 h1:GmnMZ/dlvVAPFmWBzEpRJX49pUAymPfoASLNRJqR0AY= github.com/ava-labs/go-ethereum v1.9.3/go.mod h1:a+agc6fXfZFsPZCylA3ry4Y8CLCqLKg3Rc23NXZ9aw8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/scripts/build.sh b/scripts/build.sh index 9bee1cf..a0a59d6 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -15,7 +15,7 @@ GECKO_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Directory BUILD_DIR=$GECKO_PATH/build # Where binaries go PLUGIN_DIR="$BUILD_DIR/plugins" # Where plugin binaries (namely coreth) go -CORETH_VER="0.2.4" # Should match coreth version in go.mod +CORETH_VER="0.2.5" # Should match coreth version in go.mod CORETH_PATH="$GOPATH/pkg/mod/github.com/ava-labs/coreth@v$CORETH_VER" # Build Gecko diff --git a/vms/rpcchainvm/factory.go b/vms/rpcchainvm/factory.go index a48bb01..9b7db9c 100644 --- a/vms/rpcchainvm/factory.go +++ b/vms/rpcchainvm/factory.go @@ -5,6 +5,8 @@ package rpcchainvm import ( "errors" + "io/ioutil" + "log" "os/exec" "github.com/hashicorp/go-plugin" @@ -31,6 +33,9 @@ func (f *Factory) New(ctx *snow.Context) (interface{}, error) { }, } if ctx != nil { + // disable go-plugin logging (since it is not controlled by Gecko's own + // logging facility) + log.SetOutput(ioutil.Discard) config.Stderr = ctx.Log config.SyncStdout = ctx.Log config.SyncStderr = ctx.Log From 1d4c36846237e3b38c12537364e8708968527291 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Tue, 23 Jun 2020 18:23:22 -0400 Subject: [PATCH 76/80] added local path to plugin --- main/params.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/main/params.go b/main/params.go index 4d55919..53e7b01 100644 --- a/main/params.go +++ b/main/params.go @@ -35,17 +35,19 @@ const ( // Results of parsing the CLI var ( - Config = node.Config{} - Err error - defaultNetworkName = genesis.TestnetName - defaultDbDir = os.ExpandEnv(filepath.Join("$HOME", ".gecko", "db")) - defaultStakingKeyPath = os.ExpandEnv(filepath.Join("$HOME", ".gecko", "staking", "staker.key")) - defaultStakingCertPath = os.ExpandEnv(filepath.Join("$HOME", ".gecko", "staking", "staker.crt")) + Config = node.Config{} + Err error + defaultNetworkName = genesis.TestnetName - defaultPluginDirs = []string{ - "./build/plugins", - "./plugins", - os.ExpandEnv(filepath.Join("$HOME", ".gecko", "plugins")), + homeDir = os.ExpandEnv("$HOME") + defaultDbDir = filepath.Join(homeDir, ".gecko", "db") + defaultStakingKeyPath = filepath.Join(homeDir, ".gecko", "staking", "staker.key") + defaultStakingCertPath = filepath.Join(homeDir, ".gecko", "staking", "staker.crt") + defaultPluginDirs = []string{ + filepath.Join(".", "build", "plugins"), + filepath.Join(".", "plugins"), + filepath.Join("/", "usr", "local", "lib", "gecko"), + filepath.Join(homeDir, ".gecko", "plugins"), } ) From d40fbe8f75721bcd58b9de949a330ffe6657e9e3 Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 23 Jun 2020 18:33:57 -0400 Subject: [PATCH 77/80] improve plugin logging --- go.mod | 1 + vms/rpcchainvm/factory.go | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c8d28c2..5afd406 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/gorilla/mux v1.7.4 github.com/gorilla/rpc v1.2.0 github.com/gorilla/websocket v1.4.2 + github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd github.com/hashicorp/go-plugin v1.3.0 github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/huin/goupnp v1.0.0 diff --git a/vms/rpcchainvm/factory.go b/vms/rpcchainvm/factory.go index 9b7db9c..5b2c2ce 100644 --- a/vms/rpcchainvm/factory.go +++ b/vms/rpcchainvm/factory.go @@ -5,13 +5,12 @@ package rpcchainvm import ( "errors" + "github.com/ava-labs/gecko/snow" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" "io/ioutil" "log" "os/exec" - - "github.com/hashicorp/go-plugin" - - "github.com/ava-labs/gecko/snow" ) var ( @@ -33,12 +32,18 @@ func (f *Factory) New(ctx *snow.Context) (interface{}, error) { }, } if ctx != nil { - // disable go-plugin logging (since it is not controlled by Gecko's own - // logging facility) - log.SetOutput(ioutil.Discard) + log.SetOutput(ctx.Log) config.Stderr = ctx.Log - config.SyncStdout = ctx.Log - config.SyncStderr = ctx.Log + config.Logger = hclog.New(&hclog.LoggerOptions{ + Output: ctx.Log, + Level: hclog.Info, + }) + } else { + log.SetOutput(ioutil.Discard) + config.Stderr = ioutil.Discard + config.Logger = hclog.New(&hclog.LoggerOptions{ + Output: ioutil.Discard, + }) } client := plugin.NewClient(config) From 8c7934515c8e1ce1cb080b19ea02baf27ce65140 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Tue, 23 Jun 2020 19:41:22 -0400 Subject: [PATCH 78/80] removed mutually recursive functions for fetching --- snow/engine/avalanche/bootstrapper.go | 105 ++++++++++---------------- 1 file changed, 40 insertions(+), 65 deletions(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index 352f40b..d8f1d6b 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -42,18 +42,16 @@ type bootstrapper struct { metrics common.Bootstrapper - // true if all of the vertices in the original accepted frontier have been processed - processedStartingAcceptedFrontier bool - // number of vertices fetched so far numFetched uint32 // tracks which validators were asked for which containers in which requests outstandingRequests common.Requests - // IDs of vertices that we will send a GetAncestors request for once we are not at the - // max number of outstanding requests - // Invariant: The intersection of needToFetch and outstandingRequests is empty + // IDs of vertices that we will send a GetAncestors request for once we are + // not at the max number of outstanding requests + // Invariant: The intersection of needToFetch and outstandingRequests is + // empty needToFetch ids.Set // Contains IDs of vertices that have recently been processed @@ -108,49 +106,36 @@ func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set { return acceptedVtxIDs } -// Calls fetch for a pending vertex if there are any -func (b *bootstrapper) fetchANeededVtx() error { - if b.needToFetch.Len() > 0 { - return b.fetch(b.needToFetch.CappedList(1)[0]) - } - return nil -} +// Fetch vertices and their ancestors from the set of vertices that are needed +// to be fetched. +func (b *bootstrapper) fetch(vtxIDs ...ids.ID) error { + b.needToFetch.Add(vtxIDs...) + for b.needToFetch.Len() > 0 && b.outstandingRequests.Len() < common.MaxOutstandingRequests { + vtxID := b.needToFetch.CappedList(1)[0] + b.needToFetch.Remove(vtxID) -// Get vertex [vtxID] and its ancestors. -// If [vtxID] has already been requested or is already fetched, and there are -// unrequested vertices, requests one such vertex instead of [vtxID] -func (b *bootstrapper) fetch(vtxID ids.ID) error { - // Make sure we haven't already requested this block - if b.outstandingRequests.Contains(vtxID) { - return b.fetchANeededVtx() - } - - // Make sure we don't already have this vertex - if _, err := b.State.GetVertex(vtxID); err == nil { - if numPending := b.outstandingRequests.Len(); numPending == 0 && b.processedStartingAcceptedFrontier { - return b.finish() + // Make sure we haven't already requested this vertex + if b.outstandingRequests.Contains(vtxID) { + continue } - b.needToFetch.Remove(vtxID) // we have this vertex. no need to request it. - return b.fetchANeededVtx() - } - // If we're already at maximum number of outstanding requests, queue for later - if b.outstandingRequests.Len() >= common.MaxOutstandingRequests { - b.needToFetch.Add(vtxID) - return nil - } + // Make sure we don't already have this vertex + if _, err := b.State.GetVertex(vtxID); err == nil { + continue + } - validators := b.BootstrapConfig.Validators.Sample(1) // validator to send request to - if len(validators) == 0 { - return fmt.Errorf("Dropping request for %s as there are no validators", vtxID) - } - validatorID := validators[0].ID() - b.RequestID++ + validators := b.BootstrapConfig.Validators.Sample(1) // validator to send request to + if len(validators) == 0 { + return fmt.Errorf("Dropping request for %s as there are no validators", vtxID) + } + validatorID := validators[0].ID() + b.RequestID++ - b.outstandingRequests.Add(validatorID, b.RequestID, vtxID) - b.needToFetch.Remove(vtxID) // maintains invariant that intersection with outstandingRequests is empty - b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, vtxID) // request vertex and ancestors - return nil + b.outstandingRequests.Add(validatorID, b.RequestID, vtxID) + b.needToFetch.Remove(vtxID) // maintains invariant that intersection with outstandingRequests is empty + b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, vtxID) // request vertex and ancestors + } + return b.finish() } // Process vertices @@ -164,14 +149,17 @@ func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error { for toProcess.Len() > 0 { vtx := toProcess.Pop() + vtxID := vtx.ID() + switch vtx.Status() { case choices.Unknown: - if err := b.fetch(vtx.ID()); err != nil { - return err - } + b.fetch(vtxID) case choices.Rejected: + b.needToFetch.Remove(vtxID) return fmt.Errorf("tried to accept %s even though it was previously rejected", vtx.ID()) case choices.Processing: + b.needToFetch.Remove(vtxID) + if err := b.VtxBlocked.Push(&vertexJob{ log: b.BootstrapConfig.Context.Log, numAccepted: b.numBSVtx, @@ -216,10 +204,7 @@ func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error { return err } - if numPending := b.outstandingRequests.Len(); numPending == 0 && b.processedStartingAcceptedFrontier { - return b.finish() - } - return nil + return b.fetch() } // MultiPut handles the receipt of multiple containers. Should be received in response to a GetAncestors message to [vdr] @@ -263,11 +248,6 @@ func (b *bootstrapper) MultiPut(vdr ids.ShortID, requestID uint32, vtxs [][]byte } } - // Now there is one less outstanding request; send another if needed - if err := b.fetchANeededVtx(); err != nil { - return err - } - return b.process(processVertices...) } @@ -293,24 +273,19 @@ func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error { for _, vtxID := range acceptedContainerIDs.List() { if vtx, err := b.State.GetVertex(vtxID); err == nil { storedVtxs = append(storedVtxs, vtx) - } else if err := b.fetch(vtxID); err != nil { - return err + } else { + b.needToFetch.Add(vtxID) } } if err := b.process(storedVtxs...); err != nil { return err } - b.processedStartingAcceptedFrontier = true - - if numPending := b.outstandingRequests.Len(); numPending == 0 { - return b.finish() - } - return nil + return b.fetch() } // Finish bootstrapping func (b *bootstrapper) finish() error { - if b.finished { + if b.finished || b.outstandingRequests.Len() > 0 || b.needToFetch.Len() > 0 { return nil } b.BootstrapConfig.Context.Log.Info("finished fetching vertices. executing transaction state transitions...") From 16f006edc98aa0421fc0bb7b38c519a858042730 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Tue, 23 Jun 2020 19:43:03 -0400 Subject: [PATCH 79/80] Removed no longer upheld invariant --- snow/engine/avalanche/bootstrapper.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index d8f1d6b..a9c4e64 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -50,8 +50,6 @@ type bootstrapper struct { // IDs of vertices that we will send a GetAncestors request for once we are // not at the max number of outstanding requests - // Invariant: The intersection of needToFetch and outstandingRequests is - // empty needToFetch ids.Set // Contains IDs of vertices that have recently been processed From 26edbc5e6ecfce15d9c1f49e50f3a5fd3a9aa823 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Tue, 23 Jun 2020 19:57:44 -0400 Subject: [PATCH 80/80] cleaned up avalanche bootstrapping --- snow/engine/avalanche/bootstrapper.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index a9c4e64..1af48bf 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -151,7 +151,7 @@ func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error { switch vtx.Status() { case choices.Unknown: - b.fetch(vtxID) + b.needToFetch.Add(vtxID) case choices.Rejected: b.needToFetch.Remove(vtxID) return fmt.Errorf("tried to accept %s even though it was previously rejected", vtx.ID()) @@ -275,10 +275,7 @@ func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error { b.needToFetch.Add(vtxID) } } - if err := b.process(storedVtxs...); err != nil { - return err - } - return b.fetch() + return b.process(storedVtxs...) } // Finish bootstrapping