From 280734d0e37f16b725906ec2b151c0b20e7dfaf1 Mon Sep 17 00:00:00 2001 From: colin axner Date: Thu, 11 Jul 2019 03:56:43 -0700 Subject: [PATCH] collect all invariants for a block before stopping (#4707) --- simapp/sim_test.go | 12 ++-- simapp/utils.go | 23 +++---- types/invariant.go | 11 +++- x/bank/internal/keeper/invariants.go | 35 +++------- x/crisis/handler.go | 9 +-- x/crisis/handler_test.go | 5 +- x/crisis/internal/keeper/keeper.go | 4 +- x/distribution/keeper/invariants.go | 74 +++++++++------------ x/gov/invariants.go | 14 ++-- x/simulation/simulate.go | 16 ++--- x/simulation/util.go | 27 ++++++-- x/staking/keeper/invariants.go | 89 +++++++++++++++----------- x/supply/internal/keeper/invariants.go | 16 ++--- 13 files changed, 172 insertions(+), 163 deletions(-) diff --git a/simapp/sim_test.go b/simapp/sim_test.go index ef95a105d..b882f6b9b 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -42,15 +42,16 @@ func init() { flag.BoolVar(&commit, "SimulationCommit", false, "have the simulation commit") flag.IntVar(&period, "SimulationPeriod", 1, "run slow invariants only once every period assertions") flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation") + flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found") } // helper function for populating input for SimulateFromSeed func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) ( testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64, - simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool) { + simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool, bool) { return tb, w, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, lean, onOperation + testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit, lean, onOperation, allInvariants } func appStateFn( @@ -602,6 +603,7 @@ func TestAppStateDeterminism(t *testing.T) { true, false, false, + false, ) appHash := app.LastCommitID().Hash appHashList[j] = appHash @@ -627,7 +629,7 @@ func BenchmarkInvariants(b *testing.B) { // 2. Run parameterized simulation (w/o invariants) _, err := simulation.SimulateFromSeed( b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app), - []sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, + []sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, false, ) if err != nil { fmt.Println(err) @@ -642,8 +644,8 @@ func BenchmarkInvariants(b *testing.B) { // their respective metadata which makes it useful for testing/benchmarking. for _, cr := range app.crisisKeeper.Routes() { b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) { - if err := cr.Invar(ctx); err != nil { - fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, numBlocks, err) + if res, stop := cr.Invar(ctx); stop { + fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, numBlocks, res) b.FailNow() } }) diff --git a/simapp/utils.go b/simapp/utils.go index 5b2345229..0624b4e9f 100644 --- a/simapp/utils.go +++ b/simapp/utils.go @@ -37,17 +37,18 @@ import ( ) var ( - genesisFile string - paramsFile string - seed int64 - numBlocks int - blockSize int - enabled bool - verbose bool - lean bool - commit bool - period int - onOperation bool // TODO Remove in favor of binary search for invariant violation + genesisFile string + paramsFile string + seed int64 + numBlocks int + blockSize int + enabled bool + verbose bool + lean bool + commit bool + period int + onOperation bool // TODO Remove in favor of binary search for invariant violation + allInvariants bool ) // NewSimAppUNSAFE is used for debugging purposes only. diff --git a/types/invariant.go b/types/invariant.go index 3a87904ad..1f06a6f6e 100644 --- a/types/invariant.go +++ b/types/invariant.go @@ -1,10 +1,12 @@ package types +import "fmt" + // An Invariant is a function which tests a particular invariant. // If the invariant has been broken, it should return an error // containing a descriptive message about what happened. // The simulator will then halt and print the logs. -type Invariant func(ctx Context) error +type Invariant func(ctx Context) (string, bool) // Invariants defines a group of invariants type Invariants []Invariant @@ -13,3 +15,10 @@ type Invariants []Invariant type InvariantRegistry interface { RegisterRoute(moduleName, route string, invar Invariant) } + +// FormatInvariant returns a standardized invariant message along with +// broken boolean. +func FormatInvariant(module, name, msg string, broken bool) (string, bool) { + return fmt.Sprintf("%s: %s invariant\n%s\nInvariant Broken: %v\n", + module, name, msg, broken), broken +} diff --git a/x/bank/internal/keeper/invariants.go b/x/bank/internal/keeper/invariants.go index 83eba493d..20205da05 100644 --- a/x/bank/internal/keeper/invariants.go +++ b/x/bank/internal/keeper/invariants.go @@ -1,11 +1,9 @@ package keeper import ( - "errors" "fmt" sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/x/auth/exported" "github.com/cosmos/cosmos-sdk/x/bank/internal/types" ) @@ -17,36 +15,23 @@ func RegisterInvariants(ir sdk.InvariantRegistry, ak types.AccountKeeper) { // NonnegativeBalanceInvariant checks that all accounts in the application have non-negative balances func NonnegativeBalanceInvariant(ak types.AccountKeeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { + var msg string + var amt int + accts := ak.GetAllAccounts(ctx) for _, acc := range accts { coins := acc.GetCoins() if coins.IsAnyNegative() { - return fmt.Errorf("%s has a negative denomination of %s", + amt++ + msg += fmt.Sprintf("\t%s has a negative denomination of %s\n", acc.GetAddress().String(), coins.String()) } } - return nil - } -} - -// TotalCoinsInvariant checks that the sum of the coins across all accounts -// is what is expected -func TotalCoinsInvariant(ak types.AccountKeeper, totalSupplyFn func() sdk.Coins) sdk.Invariant { - return func(ctx sdk.Context) error { - totalCoins := sdk.NewCoins() - - chkAccount := func(acc exported.Account) bool { - coins := acc.GetCoins() - totalCoins = totalCoins.Add(coins) - return false - } - - ak.IterateAccounts(ctx, chkAccount) - if !totalSupplyFn().IsEqual(totalCoins) { - return errors.New("total calculated coins doesn't equal expected coins") - } - return nil + broken := amt != 0 + + return sdk.FormatInvariant(types.ModuleName, "nonnegative-outstanding", + fmt.Sprintf("amount of negative accounts found %d\n%s", amt, msg), broken) } } diff --git a/x/crisis/handler.go b/x/crisis/handler.go index b2b14d54e..9ff52c2f3 100644 --- a/x/crisis/handler.go +++ b/x/crisis/handler.go @@ -40,10 +40,11 @@ func handleMsgVerifyInvariant(ctx sdk.Context, msg types.MsgVerifyInvariant, k k found := false msgFullRoute := msg.FullInvariantRoute() - var invarianceErr error + var res string + var stop bool for _, invarRoute := range k.Routes() { if invarRoute.FullRoute() == msgFullRoute { - invarianceErr = invarRoute.Invar(cacheCtx) + res, stop = invarRoute.Invar(cacheCtx) found = true break } @@ -53,7 +54,7 @@ func handleMsgVerifyInvariant(ctx sdk.Context, msg types.MsgVerifyInvariant, k k return types.ErrUnknownInvariant(types.DefaultCodespace).Result() } - if invarianceErr != nil { + if stop { // NOTE currently, because the chain halts here, this transaction will never be included // in the blockchain thus the constant fee will have never been deducted. Thus no // refund is required. @@ -70,7 +71,7 @@ func handleMsgVerifyInvariant(ctx sdk.Context, msg types.MsgVerifyInvariant, k k //} // TODO replace with circuit breaker - panic(invarianceErr) + panic(res) } ctx.EventManager().EmitEvents(sdk.Events{ diff --git a/x/crisis/handler_test.go b/x/crisis/handler_test.go index 7616f74e0..fa4a0f567 100644 --- a/x/crisis/handler_test.go +++ b/x/crisis/handler_test.go @@ -1,7 +1,6 @@ package crisis_test import ( - "errors" "fmt" "strings" "testing" @@ -17,8 +16,8 @@ import ( var ( testModuleName = "dummy" - dummyRouteWhichPasses = crisis.NewInvarRoute(testModuleName, "which-passes", func(_ sdk.Context) error { return nil }) - dummyRouteWhichFails = crisis.NewInvarRoute(testModuleName, "which-fails", func(_ sdk.Context) error { return errors.New("whoops") }) + dummyRouteWhichPasses = crisis.NewInvarRoute(testModuleName, "which-passes", func(_ sdk.Context) (string, bool) { return "", false }) + dummyRouteWhichFails = crisis.NewInvarRoute(testModuleName, "which-fails", func(_ sdk.Context) (string, bool) { return "whoops", true }) addrs = distr.TestAddrs ) diff --git a/x/crisis/internal/keeper/keeper.go b/x/crisis/internal/keeper/keeper.go index 07d50633f..7b34b7794 100644 --- a/x/crisis/internal/keeper/keeper.go +++ b/x/crisis/internal/keeper/keeper.go @@ -67,13 +67,13 @@ func (k Keeper) AssertInvariants(ctx sdk.Context) { start := time.Now() invarRoutes := k.Routes() for _, ir := range invarRoutes { - if err := ir.Invar(ctx); err != nil { + if res, stop := ir.Invar(ctx); stop { // TODO: Include app name as part of context to allow for this to be // variable. panic(fmt.Errorf("invariant broken: %s\n"+ "\tCRITICAL please submit the following transaction:\n"+ - "\t\t tx crisis invariant-broken %v %v", err, ir.ModuleName, ir.Route)) + "\t\t tx crisis invariant-broken %s %s", res, ir.ModuleName, ir.Route)) } } diff --git a/x/distribution/keeper/invariants.go b/x/distribution/keeper/invariants.go index 013a25535..27c8a2691 100644 --- a/x/distribution/keeper/invariants.go +++ b/x/distribution/keeper/invariants.go @@ -22,18 +22,18 @@ func RegisterInvariants(ir sdk.InvariantRegistry, k Keeper) { // AllInvariants runs all invariants of the distribution module func AllInvariants(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { - err := CanWithdrawInvariant(k)(ctx) - if err != nil { - return err + return func(ctx sdk.Context) (string, bool) { + res, stop := CanWithdrawInvariant(k)(ctx) + if stop { + return res, stop } - err = NonNegativeOutstandingInvariant(k)(ctx) - if err != nil { - return err + res, stop = NonNegativeOutstandingInvariant(k)(ctx) + if stop { + return res, stop } - err = ReferenceCountInvariant(k)(ctx) - if err != nil { - return err + res, stop = ReferenceCountInvariant(k)(ctx) + if stop { + return res, stop } return ModuleAccountInvariant(k)(ctx) } @@ -41,30 +41,29 @@ func AllInvariants(k Keeper) sdk.Invariant { // NonNegativeOutstandingInvariant checks that outstanding unwithdrawn fees are never negative func NonNegativeOutstandingInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { - + return func(ctx sdk.Context) (string, bool) { + var msg string + var amt int var outstanding sdk.DecCoins - k.IterateValidatorOutstandingRewards(ctx, func(_ sdk.ValAddress, rewards types.ValidatorOutstandingRewards) (stop bool) { + k.IterateValidatorOutstandingRewards(ctx, func(addr sdk.ValAddress, rewards types.ValidatorOutstandingRewards) (stop bool) { outstanding = rewards if outstanding.IsAnyNegative() { - return true + amt++ + msg += fmt.Sprintf("\t%v has negative outstanding coins: %v\n", addr, outstanding) } return false }) + broken := amt != 0 - if outstanding.IsAnyNegative() { - return fmt.Errorf("negative outstanding coins: %v", outstanding) - } - - return nil - + return sdk.FormatInvariant(types.ModuleName, "nonnegative outstanding", + fmt.Sprintf("found %d validators with negative outstanding rewards\n%s", amt, msg), broken) } } // CanWithdrawInvariant checks that current rewards can be completely withdrawn func CanWithdrawInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { // cache, we don't want to write changes ctx, _ = ctx.CacheContext() @@ -98,17 +97,15 @@ func CanWithdrawInvariant(k Keeper) sdk.Invariant { return false }) - if len(remaining) > 0 && remaining[0].Amount.LT(sdk.ZeroDec()) { - return fmt.Errorf("negative remaining coins: %v", remaining) - } - - return nil + broken := len(remaining) > 0 && remaining[0].Amount.LT(sdk.ZeroDec()) + return sdk.FormatInvariant(types.ModuleName, "can withdraw", + fmt.Sprintf("remaining coins: %v", remaining), broken) } } // ReferenceCountInvariant checks that the number of historical rewards records is correct func ReferenceCountInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { valCount := uint64(0) k.stakingKeeper.IterateValidators(ctx, func(_ int64, val exported.ValidatorI) (stop bool) { @@ -127,21 +124,18 @@ func ReferenceCountInvariant(k Keeper) sdk.Invariant { // delegation (previous period), one record per slash (previous period) expected := valCount + uint64(len(dels)) + slashCount count := k.GetValidatorHistoricalReferenceCount(ctx) + broken := count != expected - if count != expected { - return fmt.Errorf("unexpected number of historical rewards records: "+ - "expected %v (%v vals + %v dels + %v slashes), got %v", - expected, valCount, len(dels), slashCount, count) - } - - return nil + return sdk.FormatInvariant(types.ModuleName, "reference count", fmt.Sprintf("unexpected number of historical rewards records: "+ + "expected %v (%v vals + %v dels + %v slashes), got %v", + expected, valCount, len(dels), slashCount, count), broken) } } // ModuleAccountInvariant checks that the coins held by the distr ModuleAccount // is consistent with the sum of validator outstanding rewards func ModuleAccountInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { var expectedCoins sdk.DecCoins k.IterateValidatorOutstandingRewards(ctx, func(_ sdk.ValAddress, rewards types.ValidatorOutstandingRewards) (stop bool) { @@ -154,12 +148,8 @@ func ModuleAccountInvariant(k Keeper) sdk.Invariant { macc := k.GetDistributionAccount(ctx) - if !macc.GetCoins().IsEqual(expectedInt) { - return fmt.Errorf("distribution ModuleAccount coins invariance:\n"+ - "\texpected ModuleAccount coins: %s\n"+ - "\tdistribution ModuleAccount coins : %s", expectedInt, macc.GetCoins()) - } - - return nil + broken := !macc.GetCoins().IsEqual(expectedInt) + return sdk.FormatInvariant(types.ModuleName, "ModuleAccount coins", fmt.Sprintf("expected ModuleAccount coins: %s\n"+ + "\tdistribution ModuleAccount coins : %s", expectedInt, macc.GetCoins()), broken) } } diff --git a/x/gov/invariants.go b/x/gov/invariants.go index 1e9454978..5d9b9572b 100644 --- a/x/gov/invariants.go +++ b/x/gov/invariants.go @@ -14,7 +14,7 @@ func RegisterInvariants(ir sdk.InvariantRegistry, keeper Keeper) { // AllInvariants runs all invariants of the governance module func AllInvariants(keeper Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { return ModuleAccountInvariant(keeper)(ctx) } } @@ -22,7 +22,7 @@ func AllInvariants(keeper Keeper) sdk.Invariant { // ModuleAccountInvariant checks that the module account coins reflects the sum of // deposit amounts held on store func ModuleAccountInvariant(keeper Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { var expectedDeposits sdk.Coins keeper.IterateAllDeposits(ctx, func(deposit types.Deposit) bool { @@ -31,12 +31,10 @@ func ModuleAccountInvariant(keeper Keeper) sdk.Invariant { }) macc := keeper.GetGovernanceAccount(ctx) - if !macc.GetCoins().IsEqual(expectedDeposits) { - return fmt.Errorf("deposits invariance:\n"+ - "\tgov ModuleAccount coins: %s\n"+ - "\tsum of deposit amounts: %s", macc.GetCoins(), expectedDeposits) - } + broken := !macc.GetCoins().IsEqual(expectedDeposits) - return nil + return sdk.FormatInvariant(types.ModuleName, "deposits", + fmt.Sprintf("\tgov ModuleAccount coins: %s\n\tsum of deposit amounts: %s", + macc.GetCoins(), expectedDeposits), broken) } } diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index b6f753933..30450aeb7 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -46,7 +46,7 @@ func SimulateFromSeed( tb testing.TB, w io.Writer, app *baseapp.BaseApp, appStateFn AppStateFn, seed int64, ops WeightedOperations, invariants sdk.Invariants, - numBlocks, blockSize int, commit, lean, onOperation bool, + numBlocks, blockSize int, commit, lean, onOperation, allInvariants bool, ) (stopEarly bool, simError error) { // in case we have to end early, don't os.Exit so that we can run cleanup code. @@ -108,7 +108,7 @@ func SimulateFromSeed( blockSimulator := createBlockSimulator( testingMode, tb, t, w, params, eventStats.tally, invariants, ops, operationQueue, timeOperationQueue, - numBlocks, blockSize, logWriter, lean, onOperation) + numBlocks, blockSize, logWriter, lean, onOperation, allInvariants) if !testingMode { b.ResetTimer() @@ -137,7 +137,7 @@ func SimulateFromSeed( app.BeginBlock(request) if testingMode { - assertAllInvariants(t, app, invariants, "BeginBlock", logWriter) + assertAllInvariants(t, app, invariants, "BeginBlock", logWriter, allInvariants) } ctx := app.NewContext(false, header) @@ -152,14 +152,14 @@ func SimulateFromSeed( tb, r, app, ctx, accs, logWriter, eventStats.tally, lean) if testingMode && onOperation { - assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter) + assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, allInvariants) } // run standard operations operations := blockSimulator(r, app, ctx, accs, header) opCount += operations + numQueuedOpsRan + numQueuedTimeOpsRan if testingMode { - assertAllInvariants(t, app, invariants, "StandardOperations", logWriter) + assertAllInvariants(t, app, invariants, "StandardOperations", logWriter, allInvariants) } res := app.EndBlock(abci.RequestEndBlock{}) @@ -172,7 +172,7 @@ func SimulateFromSeed( logWriter.AddEntry(EndBlockEntry(int64(height))) if testingMode { - assertAllInvariants(t, app, invariants, "EndBlock", logWriter) + assertAllInvariants(t, app, invariants, "EndBlock", logWriter, allInvariants) } if commit { app.Commit() @@ -221,7 +221,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params, event func(string), invariants sdk.Invariants, ops WeightedOperations, operationQueue OperationQueue, timeOperationQueue []FutureOperation, - totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation bool) blockSimFn { + totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn { lastBlocksizeState := 0 // state for [4 * uniform distribution] blocksize := 0 @@ -269,7 +269,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize) eventStr := fmt.Sprintf("operation: %v", opMsg.String()) - assertAllInvariants(t, app, invariants, eventStr, logWriter) + assertAllInvariants(t, app, invariants, eventStr, logWriter, allInvariants) } else if opCount%50 == 0 { fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize) diff --git a/x/simulation/util.go b/x/simulation/util.go index 4d23f9f3d..0eeabe01e 100644 --- a/x/simulation/util.go +++ b/x/simulation/util.go @@ -14,17 +14,30 @@ import ( // assertAll asserts the all invariants against application state func assertAllInvariants(t *testing.T, app *baseapp.BaseApp, invs sdk.Invariants, - event string, logWriter LogWriter) { + event string, logWriter LogWriter, allInvariants bool) { ctx := app.NewContext(false, abci.Header{Height: app.LastBlockHeight() + 1}) + var broken bool + var invariantResults []string for i := 0; i < len(invs); i++ { - if err := invs[i](ctx); err != nil { - fmt.Printf("Invariants broken after %s\n%s\n", event, err.Error()) - logWriter.PrintLogs() - t.Fatal() + res, stop := invs[i](ctx) + if stop { + broken = true + invariantResults = append(invariantResults, res) + } else if allInvariants { + invariantResults = append(invariantResults, res) } } + + if broken { + fmt.Printf("Invariants broken after %s\n\n", event) + for _, res := range invariantResults { + fmt.Printf("%s\n", res) + } + logWriter.PrintLogs() + t.Fatal() + } } func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B) { @@ -66,11 +79,11 @@ func getBlockSize(r *rand.Rand, params Params, func PeriodicInvariants(invariants []sdk.Invariant, period, offset int) []sdk.Invariant { var outInvariants []sdk.Invariant for _, invariant := range invariants { - outInvariant := func(ctx sdk.Context) error { + outInvariant := func(ctx sdk.Context) (string, bool) { if int(ctx.BlockHeight())%period == offset { return invariant(ctx) } - return nil + return "", false } outInvariants = append(outInvariants, outInvariant) } diff --git a/x/staking/keeper/invariants.go b/x/staking/keeper/invariants.go index 7339eddc0..9a3c75d41 100644 --- a/x/staking/keeper/invariants.go +++ b/x/staking/keeper/invariants.go @@ -25,20 +25,20 @@ func RegisterInvariants(ir sdk.InvariantRegistry, k Keeper) { // AllInvariants runs all invariants of the staking module. func AllInvariants(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { - err := ModuleAccountInvariants(k)(ctx) - if err != nil { - return err + return func(ctx sdk.Context) (string, bool) { + res, stop := ModuleAccountInvariants(k)(ctx) + if stop { + return res, stop } - err = NonNegativePowerInvariant(k)(ctx) - if err != nil { - return err + res, stop = NonNegativePowerInvariant(k)(ctx) + if stop { + return res, stop } - err = PositiveDelegationInvariant(k)(ctx) - if err != nil { - return err + res, stop = PositiveDelegationInvariant(k)(ctx) + if stop { + return res, stop } return DelegatorSharesInvariant(k)(ctx) @@ -48,7 +48,7 @@ func AllInvariants(k Keeper) sdk.Invariant { // ModuleAccountInvariants checks that the bonded and notBonded ModuleAccounts pools // reflects the tokens actively bonded and not bonded func ModuleAccountInvariants(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { bonded := sdk.ZeroInt() notBonded := sdk.ZeroInt() bondedPool := k.GetBondedPool(ctx) @@ -76,31 +76,29 @@ func ModuleAccountInvariants(k Keeper) sdk.Invariant { poolBonded := bondedPool.GetCoins().AmountOf(bondDenom) poolNotBonded := notBondedPool.GetCoins().AmountOf(bondDenom) + broken := !poolBonded.Equal(bonded) || !poolNotBonded.Equal(notBonded) // Bonded tokens should equal sum of tokens with bonded validators // Not-bonded tokens should equal unbonding delegations plus tokens on unbonded validators - if !poolBonded.Equal(bonded) || !poolNotBonded.Equal(notBonded) { - return fmt.Errorf( - "bonded token invariance:\n"+ - "\tPool's bonded tokens: %v\n"+ - "\tsum of bonded tokens: %v\n"+ - "not bonded token invariance:\n"+ - "\tPool's not bonded tokens: %v\n"+ - "\tsum of not bonded tokens: %v\n"+ - "module accounts total (bonded + not bonded):\n"+ - "\tModule Accounts' tokens: %v\n"+ - "\tsum tokens: %v\n", - poolBonded, bonded, poolNotBonded, notBonded, poolBonded.Add(poolNotBonded), bonded.Add(notBonded), - ) - } - - return nil + return sdk.FormatInvariant(types.ModuleName, "bonded and not bonded module account coins", fmt.Sprintf( + "\tPool's bonded tokens: %v\n"+ + "\tsum of bonded tokens: %v\n"+ + "not bonded token invariance:\n"+ + "\tPool's not bonded tokens: %v\n"+ + "\tsum of not bonded tokens: %v\n"+ + "module accounts total (bonded + not bonded):\n"+ + "\tModule Accounts' tokens: %v\n"+ + "\tsum tokens: %v\n", + poolBonded, bonded, poolNotBonded, notBonded, poolBonded.Add(poolNotBonded), bonded.Add(notBonded)), broken) } } // NonNegativePowerInvariant checks that all stored validators have >= 0 power. func NonNegativePowerInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { + var msg string + var broken bool + iterator := k.ValidatorsPowerStoreIterator(ctx) for ; iterator.Valid(); iterator.Next() { @@ -112,42 +110,54 @@ func NonNegativePowerInvariant(k Keeper) sdk.Invariant { powerKey := types.GetValidatorsByPowerIndexKey(validator) if !bytes.Equal(iterator.Key(), powerKey) { - return fmt.Errorf("power store invariance:\n\tvalidator.Power: %v"+ + broken = true + msg += fmt.Sprintf("power store invariance:\n\tvalidator.Power: %v"+ "\n\tkey should be: %v\n\tkey in store: %v", validator.GetConsensusPower(), powerKey, iterator.Key()) } if validator.Tokens.IsNegative() { - return fmt.Errorf("negative tokens for validator: %v", validator) + broken = true + msg += fmt.Sprintf("negative tokens for validator: %v", validator) } } iterator.Close() - return nil + return sdk.FormatInvariant(types.ModuleName, "nonnegative power", msg, broken) } } // PositiveDelegationInvariant checks that all stored delegations have > 0 shares. func PositiveDelegationInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { + var msg string + var amt int + delegations := k.GetAllDelegations(ctx) for _, delegation := range delegations { if delegation.Shares.IsNegative() { - return fmt.Errorf("delegation with negative shares: %+v", delegation) + amt++ + msg += fmt.Sprintf("\tdelegation with negative shares: %+v\n", delegation) } if delegation.Shares.IsZero() { - return fmt.Errorf("delegation with zero shares: %+v", delegation) + amt++ + msg += fmt.Sprintf("\tdelegation with zero shares: %+v\n", delegation) } } + broken := amt != 0 - return nil + return sdk.FormatInvariant(types.ModuleName, "positive delegations", fmt.Sprintf( + "%d invalid delegations found\n%s", amt, msg), broken) } } // DelegatorSharesInvariant checks whether all the delegator shares which persist // in the delegator object add up to the correct total delegator shares -// amount stored in each validator +// amount stored in each validator. func DelegatorSharesInvariant(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { + var msg string + var broken bool + validators := k.GetAllValidators(ctx) for _, validator := range validators { @@ -160,11 +170,12 @@ func DelegatorSharesInvariant(k Keeper) sdk.Invariant { } if !valTotalDelShares.Equal(totalDelShares) { - return fmt.Errorf("broken delegator shares invariance:\n"+ + broken = true + msg += fmt.Sprintf("broken delegator shares invariance:\n"+ "\tvalidator.DelegatorShares: %v\n"+ "\tsum of Delegator.Shares: %v", valTotalDelShares, totalDelShares) } } - return nil + return sdk.FormatInvariant(types.ModuleName, "delegator shares", msg, broken) } } diff --git a/x/supply/internal/keeper/invariants.go b/x/supply/internal/keeper/invariants.go index 51c957fa9..9a5bbc6a5 100644 --- a/x/supply/internal/keeper/invariants.go +++ b/x/supply/internal/keeper/invariants.go @@ -15,14 +15,14 @@ func RegisterInvariants(ir sdk.InvariantRegistry, k Keeper) { // AllInvariants runs all invariants of the supply module. func AllInvariants(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { return TotalSupply(k)(ctx) } } // TotalSupply checks that the total supply reflects all the coins held in accounts func TotalSupply(k Keeper) sdk.Invariant { - return func(ctx sdk.Context) error { + return func(ctx sdk.Context) (string, bool) { var expectedTotal sdk.Coins supply := k.GetSupply(ctx) @@ -31,12 +31,12 @@ func TotalSupply(k Keeper) sdk.Invariant { return false }) - if !expectedTotal.IsEqual(supply.Total) { - return fmt.Errorf("total supply invariance:\n"+ - "\tsum of accounts coins: %v\n"+ - "\tsupply.Total: %v", expectedTotal, supply.Total) - } + broken := !expectedTotal.IsEqual(supply.Total) - return nil + return sdk.FormatInvariant(types.ModuleName, "total supply", + fmt.Sprintf( + "\tsum of accounts coins: %v\n"+ + "\tsupply.Total: %v", + expectedTotal, supply.Total), broken) } }