certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #5538: Refactor IAVL Pruning

This commit is contained in:
Alexander Bezobchuk 2020-01-22 14:52:56 -05:00 committed by GitHub
parent 415eab7a56
commit f18005d2f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 106 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -201,6 +201,7 @@ that allows for arbitrary vesting periods.
### Improvements ### Improvements
* (iavl) [\#5538](https://github.com/cosmos/cosmos-sdk/pull/5538) Remove manual IAVL pruning in favor of IAVL's internal pruning strategy.
* (server) [\#4215](https://github.com/cosmos/cosmos-sdk/issues/4215) The `--pruning` flag * (server) [\#4215](https://github.com/cosmos/cosmos-sdk/issues/4215) The `--pruning` flag
has been moved to the configuration file, to allow easier node configuration. has been moved to the configuration file, to allow easier node configuration.
* (cli) [\#5116](https://github.com/cosmos/cosmos-sdk/issues/5116) The `CLIContext` now supports multiple verifiers * (cli) [\#5116](https://github.com/cosmos/cosmos-sdk/issues/5116) The `CLIContext` now supports multiple verifiers

4
baseapp/baseapp_test.go
View File

@ -151,7 +151,7 @@ func useFileUpgradeLoader(upgradeInfoPath string) func(*BaseApp) {
func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) { func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) {
rs := rootmulti.NewStore(db) rs := rootmulti.NewStore(db)
rs.SetPruning(store.PruneSyncable) rs.SetPruning(store.PruneNothing)
key := sdk.NewKVStoreKey(storeKey) key := sdk.NewKVStoreKey(storeKey)
rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil) rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil)
err := rs.LoadLatestVersion() err := rs.LoadLatestVersion()
@ -244,7 +244,7 @@ func TestSetLoader(t *testing.T) {
initStore(t, db, tc.origStoreKey, k, v) initStore(t, db, tc.origStoreKey, k, v)
// load the app with the existing db // load the app with the existing db
opts := []func(*BaseApp){SetPruning(store.PruneSyncable)} opts := []func(*BaseApp){SetPruning(store.PruneNothing)}
if tc.setLoader != nil { if tc.setLoader != nil {
opts = append(opts, tc.setLoader) opts = append(opts, tc.setLoader)
} }

6
store/cache/cache_test.go vendored
View File

@ -20,7 +20,7 @@ func TestGetOrSetStoreCache(t *testing.T) {
sKey := types.NewKVStoreKey("test") sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100) tree, err := iavl.NewMutableTree(db, 100)
require.NoError(t, err) require.NoError(t, err)
store := iavlstore.UnsafeNewStore(tree, 10, 10) store := iavlstore.UnsafeNewStore(tree)
store2 := mngr.GetStoreCache(sKey, store) store2 := mngr.GetStoreCache(sKey, store)
require.NotNil(t, store2) require.NotNil(t, store2)
@ -34,7 +34,7 @@ func TestUnwrap(t *testing.T) {
sKey := types.NewKVStoreKey("test") sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100) tree, err := iavl.NewMutableTree(db, 100)
require.NoError(t, err) require.NoError(t, err)
store := iavlstore.UnsafeNewStore(tree, 10, 10) store := iavlstore.UnsafeNewStore(tree)
_ = mngr.GetStoreCache(sKey, store) _ = mngr.GetStoreCache(sKey, store)
require.Equal(t, store, mngr.Unwrap(sKey)) require.Equal(t, store, mngr.Unwrap(sKey))
@ -48,7 +48,7 @@ func TestStoreCache(t *testing.T) {
sKey := types.NewKVStoreKey("test") sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100) tree, err := iavl.NewMutableTree(db, 100)
require.NoError(t, err) require.NoError(t, err)
store := iavlstore.UnsafeNewStore(tree, 10, 10) store := iavlstore.UnsafeNewStore(tree)
kvStore := mngr.GetStoreCache(sKey, store) kvStore := mngr.GetStoreCache(sKey, store)
for i := uint(0); i < cache.DefaultCommitKVStoreCacheSize*2; i++ { for i := uint(0); i < cache.DefaultCommitKVStoreCacheSize*2; i++ {

62
store/iavl/store.go
View File

@ -4,7 +4,6 @@ import (
"io" "io"
"sync" "sync"
"github.com/pkg/errors"
"github.com/tendermint/iavl" "github.com/tendermint/iavl"
abci "github.com/tendermint/tendermint/abci/types" abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/crypto/merkle"
@ -31,25 +30,18 @@ var (
// Store Implements types.KVStore and CommitKVStore. // Store Implements types.KVStore and CommitKVStore.
type Store struct { type Store struct {
tree Tree tree Tree
// How many old versions we hold onto.
// A value of 0 means keep no recent states.
numRecent int64
// This is the distance between state-sync waypoint states to be stored.
// See https://github.com/tendermint/tendermint/issues/828
// A value of 1 means store every state.
// A value of 0 means store no waypoints. (node cannot assist in state-sync)
// By default this value should be set the same across all nodes,
// so that nodes can know the waypoints their peers store.
storeEvery int64
} }
// LoadStore returns an IAVL Store as a CommitKVStore. Internally it will load the // LoadStore returns an IAVL Store as a CommitKVStore. Internally it will load the
// store's version (id) from the provided DB. An error is returned if the version // store's version (id) from the provided DB. An error is returned if the version
// fails to load. // fails to load.
func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyLoading bool) (types.CommitKVStore, error) { func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyLoading bool) (types.CommitKVStore, error) {
tree, err := iavl.NewMutableTree(db, defaultIAVLCacheSize) tree, err := iavl.NewMutableTreeWithOpts(
db,
dbm.NewMemDB(),
defaultIAVLCacheSize,
iavl.PruningOptions(pruning.KeepEvery(), pruning.KeepRecent()),
)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -64,21 +56,15 @@ func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyL
return nil, err return nil, err
} }
iavl := UnsafeNewStore(tree, int64(0), int64(0)) return &Store{tree: tree}, nil
iavl.SetPruning(pruning)
return iavl, nil
} }
// UnsafeNewStore returns a reference to a new IAVL Store. // UnsafeNewStore returns a reference to a new IAVL Store with a given mutable
// IAVL tree reference.
// //
// CONTRACT: The IAVL tree should be fully loaded. // CONTRACT: The IAVL tree should be fully loaded.
func UnsafeNewStore(tree *iavl.MutableTree, numRecent int64, storeEvery int64) *Store { func UnsafeNewStore(tree *iavl.MutableTree) *Store {
return &Store{ return &Store{tree: tree}
tree: tree,
numRecent: numRecent,
storeEvery: storeEvery,
}
} }
// GetImmutable returns a reference to a new store backed by an immutable IAVL // GetImmutable returns a reference to a new store backed by an immutable IAVL
@ -96,11 +82,7 @@ func (st *Store) GetImmutable(version int64) (*Store, error) {
return nil, err return nil, err
} }
return &Store{ return &Store{tree: &immutableTree{iTree}}, nil
tree: &immutableTree{iTree},
numRecent: 0,
storeEvery: 0,
}, nil
} }
// Implements Committer. // Implements Committer.
@ -112,18 +94,6 @@ func (st *Store) Commit() types.CommitID {
panic(err) panic(err)
} }
// Release an old version of history, if not a sync waypoint.
previous := version - 1
if st.numRecent < previous {
toRelease := previous - st.numRecent
if st.storeEvery == 0 || toRelease%st.storeEvery != 0 {
err := st.tree.DeleteVersion(toRelease)
if errCause := errors.Cause(err); errCause != nil && errCause != iavl.ErrVersionDoesNotExist {
panic(err)
}
}
}
return types.CommitID{ return types.CommitID{
Version: version, Version: version,
Hash: hash, Hash: hash,
@ -138,10 +108,10 @@ func (st *Store) LastCommitID() types.CommitID {
} }
} }
// Implements Committer. // SetPruning panics as pruning options should be provided at initialization
func (st *Store) SetPruning(opt types.PruningOptions) { // since IAVl accepts pruning options directly.
st.numRecent = opt.KeepRecent() func (st *Store) SetPruning(_ types.PruningOptions) {
st.storeEvery = opt.KeepEvery() panic("cannot set pruning options on an initialized IAVL store")
} }
// VersionExists returns whether or not a given version is stored. // VersionExists returns whether or not a given version is stored.

86
store/iavl/store_test.go
View File

@ -14,13 +14,8 @@ import (
) )
var ( var (
cacheSize = 100 cacheSize = 100
numRecent int64 = 5 treeData = map[string]string{
storeEvery int64 = 3
)
var (
treeData = map[string]string{
"hello": "goodbye", "hello": "goodbye",
"aloha": "shalom", "aloha": "shalom",
} }
@ -30,7 +25,6 @@ var (
func randBytes(numBytes int) []byte { func randBytes(numBytes int) []byte {
b := make([]byte, numBytes) b := make([]byte, numBytes)
_, _ = crand.Read(b) _, _ = crand.Read(b)
return b return b
} }
@ -58,7 +52,7 @@ func newAlohaTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, types.CommitID) {
func TestGetImmutable(t *testing.T) { func TestGetImmutable(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, cID := newAlohaTree(t, db) tree, cID := newAlohaTree(t, db)
store := UnsafeNewStore(tree, 10, 10) store := UnsafeNewStore(tree)
require.True(t, tree.Set([]byte("hello"), []byte("adios"))) require.True(t, tree.Set([]byte("hello"), []byte("adios")))
hash, ver, err := tree.SaveVersion() hash, ver, err := tree.SaveVersion()
@ -88,7 +82,7 @@ func TestGetImmutable(t *testing.T) {
func TestTestGetImmutableIterator(t *testing.T) { func TestTestGetImmutableIterator(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, cID := newAlohaTree(t, db) tree, cID := newAlohaTree(t, db)
store := UnsafeNewStore(tree, 10, 10) store := UnsafeNewStore(tree)
newStore, err := store.GetImmutable(cID.Version) newStore, err := store.GetImmutable(cID.Version)
require.NoError(t, err) require.NoError(t, err)
@ -111,7 +105,7 @@ func TestTestGetImmutableIterator(t *testing.T) {
func TestIAVLStoreGetSetHasDelete(t *testing.T) { func TestIAVLStoreGetSetHasDelete(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, _ := newAlohaTree(t, db) tree, _ := newAlohaTree(t, db)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
key := "hello" key := "hello"
@ -136,14 +130,14 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) {
func TestIAVLStoreNoNilSet(t *testing.T) { func TestIAVLStoreNoNilSet(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, _ := newAlohaTree(t, db) tree, _ := newAlohaTree(t, db)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
require.Panics(t, func() { iavlStore.Set([]byte("key"), nil) }, "setting a nil value should panic") require.Panics(t, func() { iavlStore.Set([]byte("key"), nil) }, "setting a nil value should panic")
} }
func TestIAVLIterator(t *testing.T) { func TestIAVLIterator(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, _ := newAlohaTree(t, db) tree, _ := newAlohaTree(t, db)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz")) iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz"))
expected := []string{"aloha", "hello"} expected := []string{"aloha", "hello"}
var i int var i int
@ -219,7 +213,7 @@ func TestIAVLReverseIterator(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize) tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
iavlStore.Set([]byte{0x00}, []byte("0")) iavlStore.Set([]byte{0x00}, []byte("0"))
iavlStore.Set([]byte{0x00, 0x00}, []byte("0 0")) iavlStore.Set([]byte{0x00, 0x00}, []byte("0 0"))
@ -252,7 +246,7 @@ func TestIAVLPrefixIterator(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize) tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test1"), []byte("test1"))
iavlStore.Set([]byte("test2"), []byte("test2")) iavlStore.Set([]byte("test2"), []byte("test2"))
@ -316,7 +310,7 @@ func TestIAVLReversePrefixIterator(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize) tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test1"), []byte("test1"))
iavlStore.Set([]byte("test2"), []byte("test2")) iavlStore.Set([]byte("test2"), []byte("test2"))
@ -389,16 +383,16 @@ func TestIAVLDefaultPruning(t *testing.T) {
{[]int64{1, 2, 3}, []int64{}}, {[]int64{1, 2, 3}, []int64{}},
{[]int64{1, 2, 3, 4}, []int64{}}, {[]int64{1, 2, 3, 4}, []int64{}},
{[]int64{1, 2, 3, 4, 5}, []int64{}}, {[]int64{1, 2, 3, 4, 5}, []int64{}},
{[]int64{1, 2, 3, 4, 5, 6}, []int64{}}, {[]int64{2, 3, 4, 5, 6}, []int64{1}},
{[]int64{2, 3, 4, 5, 6, 7}, []int64{1}}, {[]int64{3, 4, 5, 6, 7}, []int64{1, 2}},
{[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}}, {[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}},
{[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}}, {[]int64{3, 5, 6, 7, 8, 9}, []int64{1, 2, 4}},
{[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}}, {[]int64{3, 6, 7, 8, 9, 10}, []int64{1, 2, 4, 5}},
{[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}}, {[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}},
{[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}}, {[]int64{3, 6, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5, 7}},
{[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}}, {[]int64{3, 6, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7, 8}},
{[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}}, {[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}},
{[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}}, {[]int64{3, 6, 9, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8, 10}},
} }
testPruning(t, int64(5), int64(3), states) testPruning(t, int64(5), int64(3), states)
} }
@ -411,18 +405,18 @@ func TestIAVLAlternativePruning(t *testing.T) {
{[]int64{1}, []int64{}}, {[]int64{1}, []int64{}},
{[]int64{1, 2}, []int64{}}, {[]int64{1, 2}, []int64{}},
{[]int64{1, 2, 3}, []int64{}}, {[]int64{1, 2, 3}, []int64{}},
{[]int64{1, 2, 3, 4}, []int64{}}, {[]int64{2, 3, 4}, []int64{1}},
{[]int64{2, 3, 4, 5}, []int64{1}}, {[]int64{3, 4, 5}, []int64{1, 2}},
{[]int64{3, 4, 5, 6}, []int64{1, 2}}, {[]int64{4, 5, 6}, []int64{1, 2, 3}},
{[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, {[]int64{5, 6, 7}, []int64{1, 2, 3, 4}},
{[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}},
{[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, {[]int64{5, 7, 8, 9}, []int64{1, 2, 3, 4, 6}},
{[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}}, {[]int64{5, 8, 9, 10}, []int64{1, 2, 3, 4, 6, 7}},
{[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}}, {[]int64{5, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7, 8}},
{[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}}, {[]int64{5, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
{[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, {[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
{[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, {[]int64{5, 10, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}},
{[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}}, {[]int64{5, 10, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11, 12}},
} }
testPruning(t, int64(3), int64(5), states) testPruning(t, int64(3), int64(5), states)
} }
@ -434,21 +428,23 @@ type pruneState struct {
func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) { func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, err := iavl.NewMutableTree(db, cacheSize) iavlOpts := iavl.PruningOptions(storeEvery, numRecent)
tree, err := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
for step, state := range states { for step, state := range states {
for _, ver := range state.stored { for _, ver := range state.stored {
require.True(t, iavlStore.VersionExists(ver), require.True(t, iavlStore.VersionExists(ver),
"Missing version %d with latest version %d. Should save last %d and every %d", "missing version %d with latest version %d; should save last %d and every %d",
ver, step, numRecent, storeEvery) ver, step, numRecent, storeEvery)
} }
for _, ver := range state.deleted { for _, ver := range state.deleted {
require.False(t, iavlStore.VersionExists(ver), require.False(t, iavlStore.VersionExists(ver),
"Unpruned version %d with latest version %d. Should prune all but last %d and every %d", "not pruned version %d with latest version %d; should prune all but last %d and every %d",
ver, step, numRecent, storeEvery) ver, step, numRecent, storeEvery)
} }
@ -461,7 +457,7 @@ func TestIAVLNoPrune(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize) tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, numRecent, int64(1)) iavlStore := UnsafeNewStore(tree)
nextVersion(iavlStore) nextVersion(iavlStore)
for i := 1; i < 100; i++ { for i := 1; i < 100; i++ {
@ -477,21 +473,23 @@ func TestIAVLNoPrune(t *testing.T) {
func TestIAVLPruneEverything(t *testing.T) { func TestIAVLPruneEverything(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, err := iavl.NewMutableTree(db, cacheSize) iavlOpts := iavl.PruningOptions(0, 1) // only store latest version in memory
tree, err := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, int64(0), int64(0)) iavlStore := UnsafeNewStore(tree)
nextVersion(iavlStore) nextVersion(iavlStore)
for i := 1; i < 100; i++ { for i := 1; i < 100; i++ {
for j := 1; j < i; j++ { for j := 1; j < i; j++ {
require.False(t, iavlStore.VersionExists(int64(j)), require.False(t, iavlStore.VersionExists(int64(j)),
"Unpruned version %d with latest version %d. Should prune all old versions", "not pruned version %d with latest version %d; should prune all old versions",
j, i) j, i)
} }
require.True(t, iavlStore.VersionExists(int64(i)), require.True(t, iavlStore.VersionExists(int64(i)),
"Missing current version on step %d, should not prune current state tree", "missing current version on step %d; should not prune current state tree",
i) i)
nextVersion(iavlStore) nextVersion(iavlStore)
@ -503,7 +501,7 @@ func TestIAVLStoreQuery(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize) tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err) require.NoError(t, err)
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
k1, v1 := []byte("key1"), []byte("val1") k1, v1 := []byte("key1"), []byte("val1")
k2, v2 := []byte("key2"), []byte("val2") k2, v2 := []byte("key2"), []byte("val2")
@ -602,7 +600,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) {
tree.Set(key, value) tree.Set(key, value)
} }
iavlStore := UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := UnsafeNewStore(tree)
iterators := make([]types.Iterator, b.N/treeSize) iterators := make([]types.Iterator, b.N/treeSize)
for i := 0; i < len(iterators); i++ { for i := 0; i < len(iterators); i++ {

2
store/prefix/store_test.go
View File

@ -90,7 +90,7 @@ func TestIAVLStorePrefix(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
tree, err := tiavl.NewMutableTree(db, cacheSize) tree, err := tiavl.NewMutableTree(db, cacheSize)
require.NoError(t, err) require.NoError(t, err)
iavlStore := iavl.UnsafeNewStore(tree, numRecent, storeEvery) iavlStore := iavl.UnsafeNewStore(tree)
testPrefixStore(t, iavlStore, []byte("test")) testPrefixStore(t, iavlStore, []byte("test"))
} }

2
store/rootmulti/proof_test.go
View File

@ -61,7 +61,7 @@ func TestVerifyMultiStoreQueryProof(t *testing.T) {
iavlStoreKey := types.NewKVStoreKey("iavlStoreKey") iavlStoreKey := types.NewKVStoreKey("iavlStoreKey")
store.MountStoreWithDB(iavlStoreKey, types.StoreTypeIAVL, nil) store.MountStoreWithDB(iavlStoreKey, types.StoreTypeIAVL, nil)
store.LoadVersion(0) require.NoError(t, store.LoadVersion(0))
iavlStore := store.GetCommitStore(iavlStoreKey).(*iavl.Store) iavlStore := store.GetCommitStore(iavlStoreKey).(*iavl.Store)
iavlStore.Set([]byte("MYKEY"), []byte("MYVALUE")) iavlStore.Set([]byte("MYKEY"), []byte("MYVALUE"))

14
store/rootmulti/store.go
View File

@ -45,17 +45,26 @@ type Store struct {
var _ types.CommitMultiStore = (*Store)(nil) var _ types.CommitMultiStore = (*Store)(nil)
var _ types.Queryable = (*Store)(nil) var _ types.Queryable = (*Store)(nil)
// nolint // NewStore returns a reference to a new Store object with the provided DB. The
// store will be created with a PruneNothing pruning strategy by default. After
// a store is created, KVStores must be mounted and finally LoadLatestVersion or
// LoadVersion must be called.
func NewStore(db dbm.DB) *Store { func NewStore(db dbm.DB) *Store {
return &Store{ return &Store{
db: db, db: db,
pruningOpts: types.PruneNothing,
storesParams: make(map[types.StoreKey]storeParams), storesParams: make(map[types.StoreKey]storeParams),
stores: make(map[types.StoreKey]types.CommitKVStore), stores: make(map[types.StoreKey]types.CommitKVStore),
keysByName: make(map[string]types.StoreKey), keysByName: make(map[string]types.StoreKey),
} }
} }
// Implements CommitMultiStore // SetPruning sets the pruning strategy on the root store and all the sub-stores.
// Note, calling SetPruning on the root store prior to LoadVersion or
// LoadLatestVersion performs a no-op as the stores aren't mounted yet.
//
// TODO: Consider removing this API altogether on sub-stores as a pruning
// strategy should only be provided on initialization.
func (rs *Store) SetPruning(pruningOpts types.PruningOptions) { func (rs *Store) SetPruning(pruningOpts types.PruningOptions) {
rs.pruningOpts = pruningOpts rs.pruningOpts = pruningOpts
for _, substore := range rs.stores { for _, substore := range rs.stores {
@ -156,7 +165,6 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error {
// load each Store (note this doesn't panic on unmounted keys now) // load each Store (note this doesn't panic on unmounted keys now)
var newStores = make(map[types.StoreKey]types.CommitKVStore) var newStores = make(map[types.StoreKey]types.CommitKVStore)
for key, storeParams := range rs.storesParams { for key, storeParams := range rs.storesParams {
// Load it // Load it
store, err := rs.loadCommitStoreFromParams(key, rs.getCommitID(infos, key.Name()), storeParams) store, err := rs.loadCommitStoreFromParams(key, rs.getCommitID(infos, key.Name()), storeParams)
if err != nil { if err != nil {

59
store/rootmulti/store_test.go
View File

@ -21,7 +21,7 @@ func TestStoreType(t *testing.T) {
func TestGetCommitKVStore(t *testing.T) { func TestGetCommitKVStore(t *testing.T) {
var db dbm.DB = dbm.NewMemDB() var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db) ms := newMultiStoreWithMounts(db, types.PruneSyncable)
err := ms.LoadLatestVersion() err := ms.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -53,7 +53,7 @@ func TestStoreMount(t *testing.T) {
func TestCacheMultiStoreWithVersion(t *testing.T) { func TestCacheMultiStoreWithVersion(t *testing.T) {
var db dbm.DB = dbm.NewMemDB() var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db) ms := newMultiStoreWithMounts(db, types.PruneSyncable)
err := ms.LoadLatestVersion() err := ms.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -90,7 +90,7 @@ func TestCacheMultiStoreWithVersion(t *testing.T) {
func TestHashStableWithEmptyCommit(t *testing.T) { func TestHashStableWithEmptyCommit(t *testing.T) {
var db dbm.DB = dbm.NewMemDB() var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db) ms := newMultiStoreWithMounts(db, types.PruneSyncable)
err := ms.LoadLatestVersion() err := ms.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -114,7 +114,7 @@ func TestHashStableWithEmptyCommit(t *testing.T) {
func TestMultistoreCommitLoad(t *testing.T) { func TestMultistoreCommitLoad(t *testing.T) {
var db dbm.DB = dbm.NewMemDB() var db dbm.DB = dbm.NewMemDB()
store := newMultiStoreWithMounts(db) store := newMultiStoreWithMounts(db, types.PruneSyncable)
err := store.LoadLatestVersion() err := store.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -139,7 +139,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
} }
// Load the latest multistore again and check version. // Load the latest multistore again and check version.
store = newMultiStoreWithMounts(db) store = newMultiStoreWithMounts(db, types.PruneSyncable)
err = store.LoadLatestVersion() err = store.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
commitID = getExpectedCommitID(store, nCommits) commitID = getExpectedCommitID(store, nCommits)
@ -152,7 +152,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
// Load an older multistore and check version. // Load an older multistore and check version.
ver := nCommits - 1 ver := nCommits - 1
store = newMultiStoreWithMounts(db) store = newMultiStoreWithMounts(db, types.PruneSyncable)
err = store.LoadVersion(ver) err = store.LoadVersion(ver)
require.Nil(t, err) require.Nil(t, err)
commitID = getExpectedCommitID(store, ver) commitID = getExpectedCommitID(store, ver)
@ -165,7 +165,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
// XXX: confirm old commit is overwritten and we have rolled back // XXX: confirm old commit is overwritten and we have rolled back
// LatestVersion // LatestVersion
store = newMultiStoreWithMounts(db) store = newMultiStoreWithMounts(db, types.PruneSyncable)
err = store.LoadLatestVersion() err = store.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
commitID = getExpectedCommitID(store, ver+1) commitID = getExpectedCommitID(store, ver+1)
@ -174,7 +174,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
func TestMultistoreLoadWithUpgrade(t *testing.T) { func TestMultistoreLoadWithUpgrade(t *testing.T) {
var db dbm.DB = dbm.NewMemDB() var db dbm.DB = dbm.NewMemDB()
store := newMultiStoreWithMounts(db) store := newMultiStoreWithMounts(db, types.PruneNothing)
err := store.LoadLatestVersion() err := store.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -206,7 +206,8 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
checkContains(t, ci.StoreInfos, []string{"store1", "store2", "store3"}) checkContains(t, ci.StoreInfos, []string{"store1", "store2", "store3"})
// Load without changes and make sure it is sensible // Load without changes and make sure it is sensible
store = newMultiStoreWithMounts(db) store = newMultiStoreWithMounts(db, types.PruneNothing)
err = store.LoadLatestVersion() err = store.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
commitID = getExpectedCommitID(store, 1) commitID = getExpectedCommitID(store, 1)
@ -218,7 +219,7 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
require.Equal(t, v2, s2.Get(k2)) require.Equal(t, v2, s2.Get(k2))
// now, let's load with upgrades... // now, let's load with upgrades...
restore, upgrades := newMultiStoreWithModifiedMounts(db) restore, upgrades := newMultiStoreWithModifiedMounts(db, types.PruneNothing)
err = restore.LoadLatestVersionAndUpgrade(upgrades) err = restore.LoadLatestVersionAndUpgrade(upgrades)
require.Nil(t, err) require.Nil(t, err)
@ -245,7 +246,7 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
migratedID := restore.Commit() migratedID := restore.Commit()
require.Equal(t, migratedID.Version, int64(2)) require.Equal(t, migratedID.Version, int64(2))
reload, _ := newMultiStoreWithModifiedMounts(db) reload, _ := newMultiStoreWithModifiedMounts(db, types.PruneNothing)
err = reload.LoadLatestVersion() err = reload.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
require.Equal(t, migratedID, reload.LastCommitID()) require.Equal(t, migratedID, reload.LastCommitID())
@ -290,7 +291,7 @@ func TestParsePath(t *testing.T) {
func TestMultiStoreQuery(t *testing.T) { func TestMultiStoreQuery(t *testing.T) {
db := dbm.NewMemDB() db := dbm.NewMemDB()
multi := newMultiStoreWithMounts(db) multi := newMultiStoreWithMounts(db, types.PruneNothing)
err := multi.LoadLatestVersion() err := multi.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -317,7 +318,7 @@ func TestMultiStoreQuery(t *testing.T) {
ver := cid.Version ver := cid.Version
// Reload multistore from database // Reload multistore from database
multi = newMultiStoreWithMounts(db) multi = newMultiStoreWithMounts(db, types.PruneNothing)
err = multi.LoadLatestVersion() err = multi.LoadLatestVersion()
require.Nil(t, err) require.Nil(t, err)
@ -361,29 +362,24 @@ func TestMultiStoreQuery(t *testing.T) {
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// utils // utils
func newMultiStoreWithMounts(db dbm.DB) *Store { func newMultiStoreWithMounts(db dbm.DB, pruningOpts types.PruningOptions) *Store {
store := NewStore(db) store := NewStore(db)
store.pruningOpts = types.PruneSyncable store.pruningOpts = pruningOpts
store.MountStoreWithDB(
types.NewKVStoreKey("store1"), types.StoreTypeIAVL, nil) store.MountStoreWithDB(types.NewKVStoreKey("store1"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB( store.MountStoreWithDB(types.NewKVStoreKey("store2"), types.StoreTypeIAVL, nil)
types.NewKVStoreKey("store2"), types.StoreTypeIAVL, nil) store.MountStoreWithDB(types.NewKVStoreKey("store3"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB(
types.NewKVStoreKey("store3"), types.StoreTypeIAVL, nil)
return store return store
} }
// store2 -> restore2 func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts types.PruningOptions) (*Store, *types.StoreUpgrades) {
// store3 dropped data (but mount still there to test)
func newMultiStoreWithModifiedMounts(db dbm.DB) (*Store, *types.StoreUpgrades) {
store := NewStore(db) store := NewStore(db)
store.pruningOpts = types.PruneSyncable store.pruningOpts = pruningOpts
store.MountStoreWithDB(
types.NewKVStoreKey("store1"), types.StoreTypeIAVL, nil) store.MountStoreWithDB(types.NewKVStoreKey("store1"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB( store.MountStoreWithDB(types.NewKVStoreKey("restore2"), types.StoreTypeIAVL, nil)
types.NewKVStoreKey("restore2"), types.StoreTypeIAVL, nil) store.MountStoreWithDB(types.NewKVStoreKey("store3"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB(
types.NewKVStoreKey("store3"), types.StoreTypeIAVL, nil)
upgrades := &types.StoreUpgrades{ upgrades := &types.StoreUpgrades{
Renamed: []types.StoreRename{{ Renamed: []types.StoreRename{{
@ -392,6 +388,7 @@ func newMultiStoreWithModifiedMounts(db dbm.DB) (*Store, *types.StoreUpgrades) {
}}, }},
Deleted: []string{"store3"}, Deleted: []string{"store3"},
} }
return store, upgrades return store, upgrades
} }

2
store/types/pruning.go
View File

@ -27,7 +27,7 @@ func (po PruningOptions) KeepEvery() int64 {
// default pruning strategies // default pruning strategies
var ( var (
// PruneEverything means all saved states will be deleted, storing only the current state // PruneEverything means all saved states will be deleted, storing only the current state
PruneEverything = NewPruningOptions(0, 0) PruneEverything = NewPruningOptions(1, 0)
// PruneNothing means all historic states will be saved, nothing will be deleted // PruneNothing means all historic states will be saved, nothing will be deleted
PruneNothing = NewPruningOptions(0, 1) PruneNothing = NewPruningOptions(0, 1)
// PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) // PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th)