certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1389 from cosmos/jlandrews/pruning 

State tree pruning
This commit is contained in:
Ethan Buchman 2018-07-02 20:42:54 -04:00 committed by GitHub
parent 0492882586 56f293850d
commit c9e561eaf1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 119 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
store/iavlstore.go
View File

@ -15,7 +15,8 @@ import (
const (
defaultIAVLCacheSize = 10000
defaultIAVLNumHistory = 1<<53 - 1 // DEPRECATED
defaultIAVLNumRecent = 100
defaultIAVLStoreEvery = 10000
)
// load the iavl store
@ -25,7 +26,7 @@ func LoadIAVLStore(db dbm.DB, id CommitID) (CommitStore, error) {
if err != nil {
return nil, err
}
store := newIAVLStore(tree, defaultIAVLNumHistory)
store := newIAVLStore(tree, defaultIAVLNumRecent, defaultIAVLStoreEvery)
return store, nil
}
@ -42,17 +43,25 @@ type iavlStore struct {
tree *iavl.VersionedTree
// How many old versions we hold onto.
// A value of 0 means keep all history.
numHistory int64
// A value of 0 means keep no recent states
numRecent int64
// Distance between state-sync waypoint states to be stored
// See https://github.com/tendermint/tendermint/issues/828
// A value of 1 means store every state
// A value of 0 means store no waypoints (node cannot assist in state-sync)
// By default this value should be set the same across all nodes,
// so that nodes can know the waypoints their peers store
// TODO if set to non-default, signal to peers that the node is not suitable as a state sync source
storeEvery int64
}
// CONTRACT: tree should be fully loaded.
// TODO: use more numHistory's, so the below nolint can be removed
// nolint: unparam
func newIAVLStore(tree *iavl.VersionedTree, numHistory int64) *iavlStore {
func newIAVLStore(tree *iavl.VersionedTree, numRecent int64, storeEvery int64) *iavlStore {
st := &iavlStore{
tree: tree,
numHistory: numHistory,
numRecent: numRecent,
storeEvery: storeEvery,
}
return st
}
@ -67,13 +76,15 @@ func (st *iavlStore) Commit() CommitID {
panic(err)
}
// Release an old version of history
if st.numHistory > 0 && (st.numHistory < st.tree.Version64()) {
toRelease := version - st.numHistory
err := st.tree.DeleteVersion(toRelease)
if err != nil {
// TODO: Handle with #870
panic(err)
// Release an old version of history, if not a sync waypoint
previous := version - 1
if st.numRecent < previous {
toRelease := previous - st.numRecent
if st.storeEvery == 0 || toRelease%st.storeEvery != 0 {
err := st.tree.DeleteVersion(toRelease)
if err != nil {
panic(err)
}
}
}
@ -91,6 +102,11 @@ func (st *iavlStore) LastCommitID() CommitID {
}
}
// VersionExists returns whether or not a given version is stored
func (st *iavlStore) VersionExists(version int64) bool {
return st.tree.VersionExists(version)
}
// Implements Store.
func (st *iavlStore) GetStoreType() StoreType {
return sdk.StoreTypeIAVL

93
store/iavlstore_test.go
View File

@ -1,6 +1,7 @@
package store
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
@ -15,7 +16,8 @@ import (
var (
cacheSize = 100
numHistory int64 = 5
numRecent int64 = 5
storeEvery int64 = 3
)
var (
@ -45,7 +47,7 @@ func newTree(t *testing.T, db dbm.DB) (*iavl.VersionedTree, CommitID) {
func TestIAVLStoreGetSetHasDelete(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newTree(t, db)
iavlStore := newIAVLStore(tree, numHistory)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
key := "hello"
@ -70,7 +72,7 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) {
func TestIAVLIterator(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newTree(t, db)
iavlStore := newIAVLStore(tree, numHistory)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz"))
expected := []string{"aloha", "hello"}
var i int
@ -143,7 +145,7 @@ func TestIAVLIterator(t *testing.T) {
func TestIAVLSubspaceIterator(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newTree(t, db)
iavlStore := newIAVLStore(tree, numHistory)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
iavlStore.Set([]byte("test1"), []byte("test1"))
iavlStore.Set([]byte("test2"), []byte("test2"))
@ -202,7 +204,7 @@ func TestIAVLSubspaceIterator(t *testing.T) {
func TestIAVLReverseSubspaceIterator(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newTree(t, db)
iavlStore := newIAVLStore(tree, numHistory)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
iavlStore.Set([]byte("test1"), []byte("test1"))
iavlStore.Set([]byte("test2"), []byte("test2"))
@ -258,10 +260,89 @@ func TestIAVLReverseSubspaceIterator(t *testing.T) {
require.Equal(t, len(expected), i)
}
func nextVersion(iavl *iavlStore) {
key := []byte(fmt.Sprintf("Key for tree: %d", iavl.LastCommitID().Version))
value := []byte(fmt.Sprintf("Value for tree: %d", iavl.LastCommitID().Version))
iavl.Set(key, value)
iavl.Commit()
}
func TestIAVLDefaultPruning(t *testing.T) {
//Expected stored / deleted version numbers for:
//numRecent = 5, storeEvery = 3
var states = []struct {
stored []int64
deleted []int64
}{
{[]int64{}, []int64{}},
{[]int64{1}, []int64{}},
{[]int64{1, 2}, []int64{}},
{[]int64{1, 2, 3}, []int64{}},
{[]int64{1, 2, 3, 4}, []int64{}},
{[]int64{1, 2, 3, 4, 5}, []int64{}},
{[]int64{1, 2, 3, 4, 5, 6}, []int64{}},
{[]int64{2, 3, 4, 5, 6, 7}, []int64{1}},
{[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}},
{[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}},
{[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}},
{[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}},
{[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}},
{[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}},
{[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}},
{[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}},
}
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
for step, state := range states {
for _, ver := range state.stored {
require.True(t, iavlStore.VersionExists(ver),
"Missing version %d with latest version %d. Should save last %d and every %d",
ver, step, numRecent, storeEvery)
}
for _, ver := range state.deleted {
require.False(t, iavlStore.VersionExists(ver),
"Unpruned version %d with latest version %d. Should prune all but last %d and every %d",
ver, step, numRecent, storeEvery)
}
nextVersion(iavlStore)
}
}
func TestIAVLNoPrune(t *testing.T) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
iavlStore := newIAVLStore(tree, numRecent, int64(1))
nextVersion(iavlStore)
for i := 1; i < 100; i++ {
for j := 1; j <= i; j++ {
require.True(t, iavlStore.VersionExists(int64(j)),
"Missing version %d with latest version %d. Should be storing all versions",
j, i)
}
nextVersion(iavlStore)
}
}
func TestIAVLPruneEverything(t *testing.T) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
iavlStore := newIAVLStore(tree, int64(0), int64(0))
nextVersion(iavlStore)
for i := 1; i < 100; i++ {
for j := 1; j < i; j++ {
require.False(t, iavlStore.VersionExists(int64(j)),
"Unpruned version %d with latest version %d. Should prune all old versions",
j, i)
}
require.True(t, iavlStore.VersionExists(int64(i)),
"Missing current version on step %d, should not prune current state tree",
i)
nextVersion(iavlStore)
}
}
func TestIAVLStoreQuery(t *testing.T) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
iavlStore := newIAVLStore(tree, numHistory)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
k1, v1 := []byte("key1"), []byte("val1")
k2, v2 := []byte("key2"), []byte("val2")

2
store/prefixstore_test.go
View File

@ -66,7 +66,7 @@ func testPrefixStore(t *testing.T, baseStore KVStore, prefix []byte) {
func TestIAVLStorePrefix(t *testing.T) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
iavlStore := newIAVLStore(tree, numHistory)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
testPrefixStore(t, iavlStore, []byte("test"))
}