certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #5103: Migration Testing

This commit is contained in:
Alexander Bezobchuk 2019-09-26 09:07:15 -07:00 committed by GitHub
parent 9d0bed8f4f
commit fbdf0e778e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 353 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.golangci.yml
View File

@ -15,6 +15,7 @@ linters:
- errcheck
- scopelint
- varcheck
- godox
issues:
exclude-rules:

2
server/export.go
View File

@ -92,5 +92,5 @@ func ExportCmd(ctx *Context, cdc *codec.Codec, appExporter AppExporter) *cobra.C
}
func isEmptyState(db dbm.DB) bool {
return db.Stats()["leveldb.sstables"] != ""
return db.Stats()["leveldb.sstables"] == ""
}

123
simapp/sim_bench_test.go Normal file
View File

@ -0,0 +1,123 @@
package simapp
import (
"fmt"
"io/ioutil"
"os"
"testing"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/x/simulation"
)
// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
var db dbm.DB
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
db, _ = sdk.NewLevelDB("Simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
// Run randomized simulation
// TODO: parameterize numbers, save for a later PR
_, simParams, simErr := simulation.SimulateFromSeed(
b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if config.ExportParamsPath != "" {
if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}
if config.Commit {
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
}
func BenchmarkInvariants(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
config.AllInvariants = false
dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
db, _ := sdk.NewLevelDB("simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
// 2. Run parameterized simulation (w/o invariants)
_, simParams, simErr := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, AppStateFn(app.Codec(), app.sm),
testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if config.ExportParamsPath != "" {
if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}
ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
// 3. Benchmark each invariant separately
//
// NOTE: We use the crisis keeper as it has all the invariants registered with
// their respective metadata which makes it useful for testing/benchmarking.
for _, cr := range app.CrisisKeeper.Routes() {
b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
if res, stop := cr.Invar(ctx); stop {
fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
b.FailNow()
}
})
}
}

109
simapp/sim_test.go
View File

@ -9,7 +9,6 @@ import (
"testing"
"github.com/stretchr/testify/require"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
@ -292,56 +291,6 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) {
return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager())
}
// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
var db dbm.DB
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
db, _ = sdk.NewLevelDB("Simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
// Run randomized simulation
// TODO: parameterize numbers, save for a later PR
_, simParams, simErr := simulation.SimulateFromSeed(
b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if config.ExportParamsPath != "" {
if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}
if config.Commit {
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
}
func TestFullAppSimulation(t *testing.T) {
if !FlagEnabledValue {
t.Skip("skipping application simulation")
@ -652,61 +601,3 @@ func TestAppStateDeterminism(t *testing.T) {
}
}
}
func BenchmarkInvariants(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
config.AllInvariants = false
dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
db, _ := sdk.NewLevelDB("simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
// 2. Run parameterized simulation (w/o invariants)
_, simParams, simErr := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, AppStateFn(app.Codec(), app.sm),
testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if config.ExportParamsPath != "" {
if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}
ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
// 3. Benchmark each invariant separately
//
// NOTE: We use the crisis keeper as it has all the invariants registered with
// their respective metadata which makes it useful for testing/benchmarking.
for _, cr := range app.CrisisKeeper.Routes() {
b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
if res, stop := cr.Invar(ctx); stop {
fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
b.FailNow()
}
})
}
}

74
x/genutil/client/cli/migrate.go
View File

@ -2,10 +2,10 @@ package cli
import (
"fmt"
"sort"
"time"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/tendermint/tendermint/types"
@ -18,19 +18,40 @@ import (
v038 "github.com/cosmos/cosmos-sdk/x/genutil/legacy/v0_38"
)
// Allow applications to extend and modify the migration process.
//
// Ref: https://github.com/cosmos/cosmos-sdk/issues/5041
var migrationMap = extypes.MigrationMap{
"v0.36": v036.Migrate,
"v0.38": v038.Migrate,
}
const (
flagGenesisTime = "genesis-time"
flagChainID = "chain-id"
)
// Allow applications to extend and modify the migration process.
//
// Ref: https://github.com/cosmos/cosmos-sdk/issues/5041
var migrationMap = extypes.MigrationMap{
"v0.36": v036.Migrate,
"v0.38": v038.Migrate, // NOTE: v0.37 and v0.38 are genesis compatible
}
// GetMigrationCallback returns a MigrationCallback for a given version.
func GetMigrationCallback(version string) extypes.MigrationCallback {
return migrationMap[version]
}
// GetMigrationVersions get all migration version in a sorted slice.
func GetMigrationVersions() []string {
versions := make([]string, len(migrationMap))
var i int
for version := range migrationMap {
versions[i] = version
i++
}
sort.Strings(versions)
return versions
}
// MigrateGenesisCmd returns a command to execute genesis state migration.
// nolint: funlen
func MigrateGenesisCmd(_ *server.Context, cdc *codec.Codec) *cobra.Command {
cmd := &cobra.Command{
Use: "migrate [target-version] [genesis-file]",
@ -42,6 +63,8 @@ $ %s migrate v0.36 /path/to/genesis.json --chain-id=cosmoshub-3 --genesis-time=2
`, version.ServerName),
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
var err error
target := args[0]
importGenesis := args[1]
@ -51,14 +74,22 @@ $ %s migrate v0.36 /path/to/genesis.json --chain-id=cosmoshub-3 --genesis-time=2
}
var initialState extypes.AppMap
cdc.MustUnmarshalJSON(genDoc.AppState, &initialState)
if migrationMap[target] == nil {
return fmt.Errorf("unknown migration function version: %s", target)
if err := cdc.UnmarshalJSON(genDoc.AppState, &initialState); err != nil {
return errors.Wrap(err, "failed to JSON unmarshal initial genesis state")
}
newGenState := migrationMap[target](initialState)
genDoc.AppState = cdc.MustMarshalJSON(newGenState)
migrationFunc := GetMigrationCallback(target)
if migrationFunc == nil {
return fmt.Errorf("unknown migration function for version: %s", target)
}
// TODO: handler error from migrationFunc call
newGenState := migrationFunc(initialState)
genDoc.AppState, err = cdc.MarshalJSON(newGenState)
if err != nil {
return errors.Wrap(err, "failed to JSON marshal migrated genesis state")
}
genesisTime := cmd.Flag(flagGenesisTime).Value.String()
if genesisTime != "" {
@ -77,18 +108,23 @@ $ %s migrate v0.36 /path/to/genesis.json --chain-id=cosmoshub-3 --genesis-time=2
genDoc.ChainID = chainID
}
out, err := cdc.MarshalJSONIndent(genDoc, "", " ")
bz, err := cdc.MarshalJSONIndent(genDoc, "", " ")
if err != nil {
return errors.Wrap(err, "failed to marshal genesis doc")
}
fmt.Println(string(sdk.MustSortJSON(out)))
sortedBz, err := sdk.SortJSON(bz)
if err != nil {
return errors.Wrap(err, "failed to sort JSON genesis doc")
}
fmt.Println(string(sortedBz))
return nil
},
}
cmd.Flags().String(flagGenesisTime, "", "Override genesis_time with this flag")
cmd.Flags().String(flagChainID, "", "Override chain_id with this flag")
cmd.Flags().String(flagGenesisTime, "", "override genesis_time with this flag")
cmd.Flags().String(flagChainID, "", "override chain_id with this flag")
return cmd
}

6
x/genutil/client/cli/migrate_test.go
View File

@ -30,6 +30,12 @@ func setupCmd(genesisTime string, chainID string) *cobra.Command {
return c
}
func TestGetMigrationCallback(t *testing.T) {
for _, version := range GetMigrationVersions() {
require.NotNil(t, GetMigrationCallback(version))
}
}
func TestMigrateGenesis(t *testing.T) {
home, cleanup := tests.NewTestCaseDir(t)
viper.Set(cli.HomeFlag, home)

11
x/genutil/legacy/v0_38/migrate.go
View File

@ -6,6 +6,8 @@ import (
v038auth "github.com/cosmos/cosmos-sdk/x/auth/legacy/v0_38"
v036genaccounts "github.com/cosmos/cosmos-sdk/x/genaccounts/legacy/v0_36"
"github.com/cosmos/cosmos-sdk/x/genutil"
v036staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_36"
v038staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_38"
)
// Migrate migrates exported state from v0.34 to a v0.36 genesis state.
@ -35,5 +37,14 @@ func Migrate(appState genutil.AppMap) genutil.AppMap {
delete(appState, v036genaccounts.ModuleName)
}
// migrate staking state
if appState[v036staking.ModuleName] != nil {
var stakingGenState v036staking.GenesisState
v036Codec.MustUnmarshalJSON(appState[v036staking.ModuleName], &stakingGenState)
delete(appState, v036staking.ModuleName) // delete old key in case the name changed
appState[v038staking.ModuleName] = v038Codec.MustMarshalJSON(v038staking.Migrate(stakingGenState))
}
return appState
}

142
x/genutil/legacy/v0_38/migrate_test.go Normal file
View File

@ -0,0 +1,142 @@
package v038_test
import (
"testing"
v036auth "github.com/cosmos/cosmos-sdk/x/auth/legacy/v0_36"
v036genaccounts "github.com/cosmos/cosmos-sdk/x/genaccounts/legacy/v0_36"
"github.com/cosmos/cosmos-sdk/x/genutil"
v038 "github.com/cosmos/cosmos-sdk/x/genutil/legacy/v0_38"
v036staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_36"
"github.com/stretchr/testify/require"
)
var genAccountsState = []byte(`[
{
"account_number": "0",
"address": "cosmos1q7380u26f7ntke3facjmynajs4umlr329vr4ja",
"coins": [
{
"amount": "1000000000",
"denom": "node0token"
},
{
"amount": "400000198",
"denom": "stake"
}
],
"delegated_free": [],
"delegated_vesting": [],
"end_time": "0",
"module_name": "",
"module_permissions": [],
"original_vesting": [],
"sequence_number": "1",
"start_time": "0"
},
{
"account_number": "0",
"address": "cosmos1tygms3xhhs3yv487phx3dw4a95jn7t7lpm470r",
"coins": [],
"delegated_free": [],
"delegated_vesting": [],
"end_time": "0",
"module_name": "not_bonded_tokens_pool",
"module_permissions": [
"burner",
"staking"
],
"original_vesting": [],
"sequence_number": "0",
"start_time": "0"
},
{
"account_number": "0",
"address": "cosmos1m3h30wlvsf8llruxtpukdvsy0km2kum8g38c8q",
"coins": [],
"delegated_free": [],
"delegated_vesting": [],
"end_time": "0",
"module_name": "mint",
"module_permissions": [
"minter"
],
"original_vesting": [],
"sequence_number": "0",
"start_time": "0"
}
]`)
var genAuthState = []byte(`{
"params": {
"max_memo_characters": "256",
"sig_verify_cost_ed25519": "590",
"sig_verify_cost_secp256k1": "1000",
"tx_sig_limit": "7",
"tx_size_cost_per_byte": "10"
}
}`)
var genStakingState = []byte(`{
"delegations": [
{
"delegator_address": "cosmos1q7380u26f7ntke3facjmynajs4umlr329vr4ja",
"shares": "100000000.000000000000000000",
"validator_address": "cosmosvaloper1q7380u26f7ntke3facjmynajs4umlr32qchq7w"
}
],
"exported": true,
"last_total_power": "400",
"last_validator_powers": [
{
"Address": "cosmosvaloper1q7380u26f7ntke3facjmynajs4umlr32qchq7w",
"Power": "100"
}
],
"params": {
"bond_denom": "stake",
"max_entries": 7,
"max_validators": 100,
"unbonding_time": "259200000000000"
},
"redelegations": null,
"unbonding_delegations": null,
"validators": [
{
"commission": {
"commission_rates": {
"max_change_rate": "0.000000000000000000",
"max_rate": "0.000000000000000000",
"rate": "0.000000000000000000"
},
"update_time": "2019-09-24T23:11:22.9692177Z"
},
"consensus_pubkey": "cosmosvalconspub1zcjduepqygqrt0saxf76lhsmp56rx52j0acdxyjvcdkq3tqvwrsmmm0ke28q36kh9h",
"delegator_shares": "100000000.000000000000000000",
"description": {
"details": "",
"identity": "",
"moniker": "node0",
"website": ""
},
"jailed": false,
"min_self_delegation": "1",
"operator_address": "cosmosvaloper1q7380u26f7ntke3facjmynajs4umlr32qchq7w",
"status": 2,
"tokens": "100000000",
"unbonding_height": "0",
"unbonding_time": "1970-01-01T00:00:00Z"
}
]
}`)
func TestMigrate(t *testing.T) {
genesis := genutil.AppMap{
v036auth.ModuleName: genAuthState,
v036genaccounts.ModuleName: genAccountsState,
v036staking.ModuleName: genStakingState,
}
require.NotPanics(t, func() { v038.Migrate(genesis) })
}

11
x/genutil/types/types.go
View File

@ -9,11 +9,16 @@ import (
// DONTCOVER
type (
// AppMap map modules names with their json raw representation
// AppMap map modules names with their json raw representation.
AppMap map[string]json.RawMessage
// MigrationCallback converts a genesis map from the previous version to the targeted one
// MigrationCallback converts a genesis map from the previous version to the
// targeted one.
//
// TODO: MigrationCallback should also return an error upon failure.
MigrationCallback func(AppMap) AppMap
// MigrationMap defines a mapping from a version to a MigrationCallback
// MigrationMap defines a mapping from a version to a MigrationCallback.
MigrationMap map[string]MigrationCallback
)

8
x/staking/legacy/v0_37/migrate.go → x/staking/legacy/v0_38/migrate.go
View File

@ -1,14 +1,14 @@
// DONTCOVER
// nolint
package v0_37
package v0_38
import (
v036staking "github.com/cosmos/cosmos-sdk/x/staking/legacy/v0_36"
)
// Migrate accepts exported genesis state from v0.34 and migrates it to v0.36
// genesis state. All entries are identical except for validator slashing events
// which now include the period.
// Migrate accepts exported genesis state from v0.36 or v0.37 and migrates it to
// v0.38 genesis state. All entries are identical except for validator descriptions
// which now include a security contact.
func Migrate(oldGenState v036staking.GenesisState) GenesisState {
return NewGenesisState(
oldGenState.Params,

6
x/staking/legacy/v0_37/types.go → x/staking/legacy/v0_38/types.go
View File

@ -1,6 +1,6 @@
// DONTCOVER
// nolint
package v0_37
package v0_38
import (
"time"
@ -69,9 +69,7 @@ type (
)
// NewDescription creates a new Description object
func NewDescription(moniker, identity, website,
securityContact, details string) Description {
func NewDescription(moniker, identity, website, securityContact, details string) Description {
return Description{
Moniker: moniker,
Identity: identity,