Merge PR #5103: Migration Testing

.golangci.yml

@@ -15,6 +15,7 @@ linters:
     - errcheck
     - scopelint
     - varcheck
+    - godox
 
 issues:
   exclude-rules:

server/export.go

@@ -92,5 +92,5 @@ func ExportCmd(ctx *Context, cdc *codec.Codec, appExporter AppExporter) *cobra.C
 }
 
 func isEmptyState(db dbm.DB) bool {
-	return db.Stats()["leveldb.sstables"] != ""
+	return db.Stats()["leveldb.sstables"] == ""
 }

simapp/sim_bench_test.go

@@ -0,0 +1,123 @@
+package simapp
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/libs/log"
+	dbm "github.com/tendermint/tm-db"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/x/simulation"
+)
+
+// Profile with:
+// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
+func BenchmarkFullAppSimulation(b *testing.B) {
+	logger := log.NewNopLogger()
+	config := NewConfigFromFlags()
+
+	var db dbm.DB
+	dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
+	db, _ = sdk.NewLevelDB("Simulation", dir)
+	defer func() {
+		db.Close()
+		os.RemoveAll(dir)
+	}()
+
+	app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
+
+	// Run randomized simulation
+	// TODO: parameterize numbers, save for a later PR
+	_, simParams, simErr := simulation.SimulateFromSeed(
+		b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
+		testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
+	)
+
+	// export state and params before the simulation error is checked
+	if config.ExportStatePath != "" {
+		if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
+			fmt.Println(err)
+			b.Fail()
+		}
+	}
+
+	if config.ExportParamsPath != "" {
+		if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
+			fmt.Println(err)
+			b.Fail()
+		}
+	}
+
+	if simErr != nil {
+		fmt.Println(simErr)
+		b.FailNow()
+	}
+
+	if config.Commit {
+		fmt.Println("\nGoLevelDB Stats")
+		fmt.Println(db.Stats()["leveldb.stats"])
+		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
+	}
+}
+
+func BenchmarkInvariants(b *testing.B) {
+	logger := log.NewNopLogger()
+
+	config := NewConfigFromFlags()
+	config.AllInvariants = false
+
+	dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
+	db, _ := sdk.NewLevelDB("simulation", dir)
+
+	defer func() {
+		db.Close()
+		os.RemoveAll(dir)
+	}()
+
+	app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
+
+	// 2. Run parameterized simulation (w/o invariants)
+	_, simParams, simErr := simulation.SimulateFromSeed(
+		b, ioutil.Discard, app.BaseApp, AppStateFn(app.Codec(), app.sm),
+		testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
+	)
+
+	// export state and params before the simulation error is checked
+	if config.ExportStatePath != "" {
+		if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
+			fmt.Println(err)
+			b.Fail()
+		}
+	}
+
+	if config.ExportParamsPath != "" {
+		if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
+			fmt.Println(err)
+			b.Fail()
+		}
+	}
+
+	if simErr != nil {
+		fmt.Println(simErr)
+		b.FailNow()
+	}
+
+	ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
+
+	// 3. Benchmark each invariant separately
+	//
+	// NOTE: We use the crisis keeper as it has all the invariants registered with
+	// their respective metadata which makes it useful for testing/benchmarking.
+	for _, cr := range app.CrisisKeeper.Routes() {
+		b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
+			if res, stop := cr.Invar(ctx); stop {
+				fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
+				b.FailNow()
+			}
+		})
+	}
+}

simapp/sim_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/log"
 	dbm "github.com/tendermint/tm-db"
@@ -292,56 +291,6 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) {
 	return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager())
 }
 
-// Profile with:
-// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
-func BenchmarkFullAppSimulation(b *testing.B) {
-	logger := log.NewNopLogger()
-	config := NewConfigFromFlags()
-
-	var db dbm.DB
-	dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
-	db, _ = sdk.NewLevelDB("Simulation", dir)
-	defer func() {
-		db.Close()
-		os.RemoveAll(dir)
-	}()
-
-	app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
-
-	// Run randomized simulation
-	// TODO: parameterize numbers, save for a later PR
-	_, simParams, simErr := simulation.SimulateFromSeed(
-		b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
-		testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
-	)
-
-	// export state and params before the simulation error is checked
-	if config.ExportStatePath != "" {
-		if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
-	}
-
-	if config.ExportParamsPath != "" {
-		if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
-	}
-
-	if simErr != nil {
-		fmt.Println(simErr)
-		b.FailNow()
-	}
-
-	if config.Commit {
-		fmt.Println("\nGoLevelDB Stats")
-		fmt.Println(db.Stats()["leveldb.stats"])
-		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
-	}
-}
-
 func TestFullAppSimulation(t *testing.T) {
 	if !FlagEnabledValue {
 		t.Skip("skipping application simulation")
@@ -652,61 +601,3 @@ func TestAppStateDeterminism(t *testing.T) {
 		}
 	}
 }
-
-func BenchmarkInvariants(b *testing.B) {
-	logger := log.NewNopLogger()
-
-	config := NewConfigFromFlags()
-	config.AllInvariants = false
-
-	dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
-	db, _ := sdk.NewLevelDB("simulation", dir)
-
-	defer func() {
-		db.Close()
-		os.RemoveAll(dir)
-	}()
-
-	app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
-
-	// 2. Run parameterized simulation (w/o invariants)
-	_, simParams, simErr := simulation.SimulateFromSeed(
-		b, ioutil.Discard, app.BaseApp, AppStateFn(app.Codec(), app.sm),
-		testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
-	)
-
-	// export state and params before the simulation error is checked
-	if config.ExportStatePath != "" {
-		if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
-	}
-
-	if config.ExportParamsPath != "" {
-		if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
-	}
-
-	if simErr != nil {
-		fmt.Println(simErr)
-		b.FailNow()
-	}
-
-	ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
-
-	// 3. Benchmark each invariant separately
-	//
-	// NOTE: We use the crisis keeper as it has all the invariants registered with
-	// their respective metadata which makes it useful for testing/benchmarking.
-	for _, cr := range app.CrisisKeeper.Routes() {
-		b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
-			if res, stop := cr.Invar(ctx); stop {
-				fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
-				b.FailNow()
-			}
-		})
-	}
-}

x/genutil/client/cli/migrate.go

@@ -2,10 +2,10 @@ package cli
 
 import (
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/pkg/errors"
-
 	"github.com/spf13/cobra"
 	"github.com/tendermint/tendermint/types"
 
@@ -18,19 +18,40 @@ import (
 	v038 "github.com/cosmos/cosmos-sdk/x/genutil/legacy/v0_38"
 )
 
-// Allow applications to extend and modify the migration process.
-//
-// Ref: https://github.com/cosmos/cosmos-sdk/issues/5041
-var migrationMap = extypes.MigrationMap{
-	"v0.36": v036.Migrate,
-	"v0.38": v038.Migrate,
-}
-
 const (
 	flagGenesisTime = "genesis-time"
 	flagChainID     = "chain-id"
 )
 
+// Allow applications to extend and modify the migration process.
+//
+// Ref: https://github.com/cosmos/cosmos-sdk/issues/5041
+var migrationMap = extypes.MigrationMap{
+	"v0.36": v036.Migrate,
+	"v0.38": v038.Migrate, // NOTE: v0.37 and v0.38 are genesis compatible
+}
+
+// GetMigrationCallback returns a MigrationCallback for a given version.
+func GetMigrationCallback(version string) extypes.MigrationCallback {
+	return migrationMap[version]
+}
+
+// GetMigrationVersions get all migration version in a sorted slice.
+func GetMigrationVersions() []string {
+	versions := make([]string, len(migrationMap))
+
+	var i int
+	for version := range migrationMap {
+		versions[i] = version
+		i++
+	}
+
+	sort.Strings(versions)
+	return versions
+}
+
+// MigrateGenesisCmd returns a command to execute genesis state migration.
+// nolint: funlen
 func MigrateGenesisCmd(_ *server.Context, cdc *codec.Codec) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "migrate [target-version] [genesis-file]",
@@ -42,6 +63,8 @@ $ %s migrate v0.36 /path/to/genesis.json --chain-id=cosmoshub-3 --genesis-time=2
 `, version.ServerName),
 		Args: cobra.ExactArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+
 			target := args[0]
 			importGenesis := args[1]
 
@@ -51,14 +74,22 @@ $ %s migrate v0.36 /path/to/genesis.json --chain-id=cosmoshub-3 --genesis-time=2
 			}
 
 			var initialState extypes.AppMap
-			cdc.MustUnmarshalJSON(genDoc.AppState, &initialState)
+			if err := cdc.UnmarshalJSON(genDoc.AppState, &initialState); err != nil {
-
+				return errors.Wrap(err, "failed to JSON unmarshal initial genesis state")
-			if migrationMap[target] == nil {
-				return fmt.Errorf("unknown migration function version: %s", target)
 			}
 
-			newGenState := migrationMap[target](initialState)
+			migrationFunc := GetMigrationCallback(target)
-			genDoc.AppState = cdc.MustMarshalJSON(newGenState)
+			if migrationFunc == nil {
+				return fmt.Errorf("unknown migration function for version: %s", target)
+			}
+
+			// TODO: handler error from migrationFunc call
+			newGenState := migrationFunc(initialState)
+
+			genDoc.AppState, err = cdc.MarshalJSON(newGenState)
+			if err != nil {
+				return errors.Wrap(err, "failed to JSON marshal migrated genesis state")
+			}
 
 			genesisTime := cmd.Flag(flagGenesisTime).Value.String()
 			if genesisTime != "" {
@@ -77,18 +108,23 @@ $ %s migrate v0.36 /path/to/genesis.json --chain-id=cosmoshub-3 --genesis-time=2
 				genDoc.ChainID = chainID
 			}
 
-			out, err := cdc.MarshalJSONIndent(genDoc, "", "  ")
+			bz, err := cdc.MarshalJSONIndent(genDoc, "", "  ")
 			if err != nil {
 				return errors.Wrap(err, "failed to marshal genesis doc")
 			}
 
-			fmt.Println(string(sdk.MustSortJSON(out)))
+			sortedBz, err := sdk.SortJSON(bz)
+			if err != nil {
+				return errors.Wrap(err, "failed to sort JSON genesis doc")
+			}
+
+			fmt.Println(string(sortedBz))
 			return nil
 		},
 	}
 
-	cmd.Flags().String(flagGenesisTime, "", "Override genesis_time with this flag")
+	cmd.Flags().String(flagGenesisTime, "", "override genesis_time with this flag")
-	cmd.Flags().String(flagChainID, "", "Override chain_id with this flag")
+	cmd.Flags().String(flagChainID, "", "override chain_id with this flag")
 
 	return cmd
 }

x/genutil/client/cli/migrate_test.go

@@ -30,6 +30,12 @@ func setupCmd(genesisTime string, chainID string) *cobra.Command {
 	return c
 }
 
+func TestGetMigrationCallback(t *testing.T) {
+	for _, version := range GetMigrationVersions() {
+		require.NotNil(t, GetMigrationCallback(version))
+	}
+}
+
 func TestMigrateGenesis(t *testing.T) {
 	home, cleanup := tests.NewTestCaseDir(t)
 	viper.Set(cli.HomeFlag, home)

x/genutil/legacy/v0_38/migrate.go

@@ -6,6 +6,8 @@ import (
 	v038auth "github.com/cosmos/cosmos-sdk/x/auth/legacy/v0_38"
 	v036genaccounts "github.com/cosmos/cosmos-sdk/x/genaccounts/legacy/v0_36"
 	"github.com/cosmos/cosmos-sdk/x/genutil"
+	v036staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_36"
+	v038staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_38"
 )
 
 // Migrate migrates exported state from v0.34 to a v0.36 genesis state.
@@ -35,5 +37,14 @@ func Migrate(appState genutil.AppMap) genutil.AppMap {
 		delete(appState, v036genaccounts.ModuleName)
 	}
 
+	// migrate staking state
+	if appState[v036staking.ModuleName] != nil {
+		var stakingGenState v036staking.GenesisState
+		v036Codec.MustUnmarshalJSON(appState[v036staking.ModuleName], &stakingGenState)
+
+		delete(appState, v036staking.ModuleName) // delete old key in case the name changed
+		appState[v038staking.ModuleName] = v038Codec.MustMarshalJSON(v038staking.Migrate(stakingGenState))
+	}
+
 	return appState
 }

x/genutil/legacy/v0_38/migrate_test.go

@@ -0,0 +1,142 @@
+package v038_test
+
+import (
+	"testing"
+
+	v036auth "github.com/cosmos/cosmos-sdk/x/auth/legacy/v0_36"
+	v036genaccounts "github.com/cosmos/cosmos-sdk/x/genaccounts/legacy/v0_36"
+	"github.com/cosmos/cosmos-sdk/x/genutil"
+	v038 "github.com/cosmos/cosmos-sdk/x/genutil/legacy/v0_38"
+	v036staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_36"
+
+	"github.com/stretchr/testify/require"
+)
+
+var genAccountsState = []byte(`[
+		{
+			"account_number": "0",
+			"address": "cosmos1q7380u26f7ntke3facjmynajs4umlr329vr4ja",
+			"coins": [
+				{
+					"amount": "1000000000",
+					"denom": "node0token"
+				},
+				{
+					"amount": "400000198",
+					"denom": "stake"
+				}
+			],
+			"delegated_free": [],
+			"delegated_vesting": [],
+			"end_time": "0",
+			"module_name": "",
+			"module_permissions": [],
+			"original_vesting": [],
+			"sequence_number": "1",
+			"start_time": "0"
+		},
+		{
+			"account_number": "0",
+			"address": "cosmos1tygms3xhhs3yv487phx3dw4a95jn7t7lpm470r",
+			"coins": [],
+			"delegated_free": [],
+			"delegated_vesting": [],
+			"end_time": "0",
+			"module_name": "not_bonded_tokens_pool",
+			"module_permissions": [
+				"burner",
+				"staking"
+			],
+			"original_vesting": [],
+			"sequence_number": "0",
+			"start_time": "0"
+		},
+		{
+			"account_number": "0",
+			"address": "cosmos1m3h30wlvsf8llruxtpukdvsy0km2kum8g38c8q",
+			"coins": [],
+			"delegated_free": [],
+			"delegated_vesting": [],
+			"end_time": "0",
+			"module_name": "mint",
+			"module_permissions": [
+				"minter"
+			],
+			"original_vesting": [],
+			"sequence_number": "0",
+			"start_time": "0"
+		}
+	]`)
+
+var genAuthState = []byte(`{
+  "params": {
+    "max_memo_characters": "256",
+    "sig_verify_cost_ed25519": "590",
+    "sig_verify_cost_secp256k1": "1000",
+    "tx_sig_limit": "7",
+    "tx_size_cost_per_byte": "10"
+  }
+}`)
+
+var genStakingState = []byte(`{
+  "delegations": [
+    {
+      "delegator_address": "cosmos1q7380u26f7ntke3facjmynajs4umlr329vr4ja",
+      "shares": "100000000.000000000000000000",
+      "validator_address": "cosmosvaloper1q7380u26f7ntke3facjmynajs4umlr32qchq7w"
+    }
+  ],
+  "exported": true,
+  "last_total_power": "400",
+  "last_validator_powers": [
+    {
+      "Address": "cosmosvaloper1q7380u26f7ntke3facjmynajs4umlr32qchq7w",
+      "Power": "100"
+    }
+  ],
+  "params": {
+    "bond_denom": "stake",
+    "max_entries": 7,
+    "max_validators": 100,
+    "unbonding_time": "259200000000000"
+  },
+  "redelegations": null,
+  "unbonding_delegations": null,
+  "validators": [
+    {
+      "commission": {
+        "commission_rates": {
+          "max_change_rate": "0.000000000000000000",
+          "max_rate": "0.000000000000000000",
+          "rate": "0.000000000000000000"
+        },
+        "update_time": "2019-09-24T23:11:22.9692177Z"
+      },
+      "consensus_pubkey": "cosmosvalconspub1zcjduepqygqrt0saxf76lhsmp56rx52j0acdxyjvcdkq3tqvwrsmmm0ke28q36kh9h",
+      "delegator_shares": "100000000.000000000000000000",
+      "description": {
+        "details": "",
+        "identity": "",
+        "moniker": "node0",
+        "website": ""
+      },
+      "jailed": false,
+      "min_self_delegation": "1",
+      "operator_address": "cosmosvaloper1q7380u26f7ntke3facjmynajs4umlr32qchq7w",
+      "status": 2,
+      "tokens": "100000000",
+      "unbonding_height": "0",
+      "unbonding_time": "1970-01-01T00:00:00Z"
+    }
+  ]
+}`)
+
+func TestMigrate(t *testing.T) {
+	genesis := genutil.AppMap{
+		v036auth.ModuleName:        genAuthState,
+		v036genaccounts.ModuleName: genAccountsState,
+		v036staking.ModuleName:     genStakingState,
+	}
+
+	require.NotPanics(t, func() { v038.Migrate(genesis) })
+}

x/genutil/types/types.go

@@ -9,11 +9,16 @@ import (
 // DONTCOVER
 
 type (
-	// AppMap map modules names with their json raw representation
+	// AppMap map modules names with their json raw representation.
 	AppMap map[string]json.RawMessage
-	// MigrationCallback converts a genesis map from the previous version to the targeted one
+
+	// MigrationCallback converts a genesis map from the previous version to the
+	// targeted one.
+	//
+	// TODO: MigrationCallback should also return an error upon failure.
 	MigrationCallback func(AppMap) AppMap
-	// MigrationMap defines a mapping from a version to a MigrationCallback
+
+	// MigrationMap defines a mapping from a version to a MigrationCallback.
 	MigrationMap map[string]MigrationCallback
 )
 

x/staking/legacy/v0_38/migrate.go

@@ -1,14 +1,14 @@
 // DONTCOVER
 // nolint
-package v0_37
+package v0_38
 
 import (
 	v036staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_36"
 )
 
-// Migrate accepts exported genesis state from v0.34 and migrates it to v0.36
+// Migrate accepts exported genesis state from v0.36 or v0.37 and migrates it to
-// genesis state. All entries are identical except for validator slashing events
+// v0.38 genesis state. All entries are identical except for validator descriptions
-// which now include the period.
+// which now include a security contact.
 func Migrate(oldGenState v036staking.GenesisState) GenesisState {
 	return NewGenesisState(
 		oldGenState.Params,

x/staking/legacy/v0_38/types.go

@@ -1,6 +1,6 @@
 // DONTCOVER
 // nolint
-package v0_37
+package v0_38
 
 import (
 	"time"
@@ -69,9 +69,7 @@ type (
 )
 
 // NewDescription creates a new Description object
-func NewDescription(moniker, identity, website,
+func NewDescription(moniker, identity, website, securityContact, details string) Description {
-	securityContact, details string) Description {
-
 	return Description{
 		Moniker:         moniker,
 		Identity:        identity,