certusone
/
cosmos-sdk
mirror of https://github.com/certusone/cosmos-sdk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge PR #1533: Pruning Cleanup

This commit is contained in:
Jeremiah Andrews 2018-07-12 18:20:26 -07:00 committed by Christopher Goes
parent 5983a07fb6
commit 43b9cc6df0
13 changed files with 146 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -12,6 +12,7 @@ BREAKING CHANGES
FEATURES
* [baseapp] NewBaseApp now takes option functions as parameters
* [store] Added support for tracing multi-store operations via `--trace-store`
* [store] Pruning strategy configurable with pruning flag on gaiad start
BUG FIXES
* \#1630 - redelegation nolonger removes tokens from the delegator liquid account

28
baseapp/options.go Normal file
View File

@ -0,0 +1,28 @@
package baseapp
import (
"fmt"
sdk "github.com/cosmos/cosmos-sdk/types"
)
// File for storing in-package BaseApp optional functions,
// for options that need access to non-exported fields of the BaseApp
// SetPruning sets a pruning option on the multistore associated with the app
func SetPruning(pruning string) func(*BaseApp) {
var pruningEnum sdk.PruningStrategy
switch pruning {
case "nothing":
pruningEnum = sdk.PruneNothing
case "everything":
pruningEnum = sdk.PruneEverything
case "syncable":
pruningEnum = sdk.PruneSyncable
default:
panic(fmt.Sprintf("Invalid pruning strategy: %s", pruning))
}
return func(bap *BaseApp) {
bap.cms.SetPruning(pruningEnum)
}
}

8
cmd/gaia/app/app.go
View File

@ -57,14 +57,10 @@ type GaiaApp struct {
}
// NewGaiaApp returns a reference to an initialized GaiaApp.
//
// TODO: Determine how to use a flexible and robust configuration paradigm that
// allows for sensible defaults while being highly configurable
// (e.g. functional options).
func NewGaiaApp(logger log.Logger, db dbm.DB, traceStore io.Writer) *GaiaApp {
func NewGaiaApp(logger log.Logger, db dbm.DB, traceStore io.Writer, baseAppOptions ...func(*bam.BaseApp)) *GaiaApp {
cdc := MakeCodec()
bApp := bam.NewBaseApp(appName, cdc, logger, db)
bApp := bam.NewBaseApp(appName, cdc, logger, db, baseAppOptions...)
bApp.SetCommitMultiStoreTracer(traceStore)
var app = &GaiaApp{

5
cmd/gaia/cmd/gaiad/main.go
View File

@ -4,7 +4,10 @@ import (
"encoding/json"
"io"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/spf13/cobra"
"github.com/spf13/viper"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/cli"
@ -40,7 +43,7 @@ func main() {
}
func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer) abci.Application {
return app.NewGaiaApp(logger, db, traceStore)
return app.NewGaiaApp(logger, db, traceStore, baseapp.SetPruning(viper.GetString("pruning")))
}
func exportAppStateAndTMValidators(

9
cmd/gaia/cmd/gaiadebug/hack.go
View File

@ -7,7 +7,10 @@ import (
"os"
"path"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/spf13/cobra"
"github.com/spf13/viper"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/crypto"
cmn "github.com/tendermint/tendermint/libs/common"
@ -44,7 +47,7 @@ func runHackCmd(cmd *cobra.Command, args []string) error {
fmt.Println(err)
os.Exit(1)
}
app := NewGaiaApp(logger, db)
app := NewGaiaApp(logger, db, baseapp.SetPruning(viper.GetString("pruning")))
// print some info
id := app.LastCommitID()
@ -140,10 +143,10 @@ type GaiaApp struct {
slashingKeeper slashing.Keeper
}
func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp {
func NewGaiaApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*bam.BaseApp)) *GaiaApp {
cdc := MakeCodec()
bApp := bam.NewBaseApp(appName, cdc, logger, db)
bApp := bam.NewBaseApp(appName, cdc, logger, db, baseAppOptions...)
bApp.SetCommitMultiStoreTracer(os.Stdout)
// create your application object

4
examples/basecoin/app/app.go
View File

@ -46,14 +46,14 @@ type BasecoinApp struct {
// In addition, all necessary mappers and keepers are created, routes
// registered, and finally the stores being mounted along with any necessary
// chain initialization.
func NewBasecoinApp(logger log.Logger, db dbm.DB) *BasecoinApp {
func NewBasecoinApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*bam.BaseApp)) *BasecoinApp {
// create and register app-level codec for TXs and accounts
cdc := MakeCodec()
// create your application type
var app = &BasecoinApp{
cdc: cdc,
BaseApp: bam.NewBaseApp(appName, cdc, logger, db),
BaseApp: bam.NewBaseApp(appName, cdc, logger, db, baseAppOptions...),
keyMain: sdk.NewKVStoreKey("main"),
keyAccount: sdk.NewKVStoreKey("acc"),
keyIBC: sdk.NewKVStoreKey("ibc"),

5
examples/basecoin/cmd/basecoind/main.go
View File

@ -5,9 +5,12 @@ import (
"io"
"os"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/examples/basecoin/app"
"github.com/cosmos/cosmos-sdk/server"
"github.com/spf13/cobra"
"github.com/spf13/viper"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/cli"
dbm "github.com/tendermint/tendermint/libs/db"
@ -41,7 +44,7 @@ func main() {
}
func newApp(logger log.Logger, db dbm.DB, storeTracer io.Writer) abci.Application {
return app.NewBasecoinApp(logger, db)
return app.NewBasecoinApp(logger, db, baseapp.SetPruning(viper.GetString("pruning")))
}
func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB, storeTracer io.Writer) (json.RawMessage, []tmtypes.GenesisValidator, error) {

4
server/mock/store.go
View File

@ -48,6 +48,10 @@ func (ms multiStore) LastCommitID() sdk.CommitID {
panic("not implemented")
}
func (ms multiStore) SetPruning(s sdk.PruningStrategy) {
panic("not implemented")
}
func (ms multiStore) GetCommitKVStore(key sdk.StoreKey) sdk.CommitKVStore {
panic("not implemented")
}

2
server/start.go
View File

@ -18,6 +18,7 @@ const (
flagWithTendermint = "with-tendermint"
flagAddress = "address"
flagTraceStore = "trace-store"
flagPruning = "pruning"
)
// StartCmd runs the service passed in, either stand-alone or in-process with
@ -43,6 +44,7 @@ func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command {
cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint")
cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address")
cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file")
cmd.Flags().String(flagPruning, "syncable", "Pruning strategy: syncable, nothing, everything")
// add support for all Tendermint-specific command line options
tcmd.AddNodeFlags(cmd)

46
store/iavlstore.go
View File

@ -15,20 +15,19 @@ import (
)
const (
defaultIAVLCacheSize = 10000
defaultIAVLNumRecent = 100
defaultIAVLStoreEvery = 1
defaultIAVLCacheSize = 10000
)
// load the iavl store
func LoadIAVLStore(db dbm.DB, id CommitID) (CommitStore, error) {
func LoadIAVLStore(db dbm.DB, id CommitID, pruning sdk.PruningStrategy) (CommitStore, error) {
tree := iavl.NewVersionedTree(db, defaultIAVLCacheSize)
_, err := tree.LoadVersion(id.Version)
if err != nil {
return nil, err
}
store := newIAVLStore(tree, defaultIAVLNumRecent, defaultIAVLStoreEvery)
return store, nil
iavl := newIAVLStore(tree, int64(0), int64(0))
iavl.SetPruning(pruning)
return iavl, nil
}
//----------------------------------------
@ -44,16 +43,15 @@ type iavlStore struct {
tree *iavl.VersionedTree
// How many old versions we hold onto.
// A value of 0 means keep no recent states
// A value of 0 means keep no recent states.
numRecent int64
// Distance between state-sync waypoint states to be stored
// This is the distance between state-sync waypoint states to be stored.
// See https://github.com/tendermint/tendermint/issues/828
// A value of 1 means store every state
// A value of 0 means store no waypoints (node cannot assist in state-sync)
// A value of 1 means store every state.
// A value of 0 means store no waypoints. (node cannot assist in state-sync)
// By default this value should be set the same across all nodes,
// so that nodes can know the waypoints their peers store
// TODO if set to non-default, signal to peers that the node is not suitable as a state sync source
// so that nodes can know the waypoints their peers store.
storeEvery int64
}
@ -77,13 +75,13 @@ func (st *iavlStore) Commit() CommitID {
panic(err)
}
// Release an old version of history, if not a sync waypoint
// Release an old version of history, if not a sync waypoint.
previous := version - 1
if st.numRecent < previous {
toRelease := previous - st.numRecent
if st.storeEvery == 0 || toRelease%st.storeEvery != 0 {
err := st.tree.DeleteVersion(toRelease)
if err != nil {
if err != nil && err.(cmn.Error).Data() != iavl.ErrVersionDoesNotExist {
panic(err)
}
}
@ -103,7 +101,21 @@ func (st *iavlStore) LastCommitID() CommitID {
}
}
// VersionExists returns whether or not a given version is stored
// Implements Committer.
func (st *iavlStore) SetPruning(pruning sdk.PruningStrategy) {
switch pruning {
case sdk.PruneEverything:
st.numRecent = 0
st.storeEvery = 0
case sdk.PruneNothing:
st.storeEvery = 1
case sdk.PruneSyncable:
st.numRecent = 100
st.storeEvery = 10000
}
}
// VersionExists returns whether or not a given version is stored.
func (st *iavlStore) VersionExists(version int64) bool {
return st.tree.VersionExists(version)
}
@ -196,6 +208,10 @@ func (st *iavlStore) Query(req abci.RequestQuery) (res abci.ResponseQuery) {
case "/store", "/key": // Get by key
key := req.Data // Data holds the key bytes
res.Key = key
if !st.VersionExists(res.Height) {
res.Log = cmn.ErrorWrap(iavl.ErrVersionDoesNotExist, "").Error()
break
}
if req.Prove {
value, proof, err := tree.GetVersionedWithProof(key, res.Height)
if err != nil {

41
store/iavlstore_test.go
View File

@ -266,13 +266,11 @@ func nextVersion(iavl *iavlStore) {
iavl.Set(key, value)
iavl.Commit()
}
func TestIAVLDefaultPruning(t *testing.T) {
//Expected stored / deleted version numbers for:
//numRecent = 5, storeEvery = 3
var states = []struct {
stored []int64
deleted []int64
}{
var states = []pruneState{
{[]int64{}, []int64{}},
{[]int64{1}, []int64{}},
{[]int64{1, 2}, []int64{}},
@ -290,6 +288,39 @@ func TestIAVLDefaultPruning(t *testing.T) {
{[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}},
{[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}},
}
testPruning(t, int64(5), int64(3), states)
}
func TestIAVLAlternativePruning(t *testing.T) {
//Expected stored / deleted version numbers for:
//numRecent = 3, storeEvery = 5
var states = []pruneState{
{[]int64{}, []int64{}},
{[]int64{1}, []int64{}},
{[]int64{1, 2}, []int64{}},
{[]int64{1, 2, 3}, []int64{}},
{[]int64{1, 2, 3, 4}, []int64{}},
{[]int64{2, 3, 4, 5}, []int64{1}},
{[]int64{3, 4, 5, 6}, []int64{1, 2}},
{[]int64{4, 5, 6, 7}, []int64{1, 2, 3}},
{[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}},
{[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}},
{[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}},
{[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}},
{[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}},
{[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
{[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
{[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}},
}
testPruning(t, int64(3), int64(5), states)
}
type pruneState struct {
stored []int64
deleted []int64
}
func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
iavlStore := newIAVLStore(tree, numRecent, storeEvery)
@ -307,6 +338,7 @@ func TestIAVLDefaultPruning(t *testing.T) {
nextVersion(iavlStore)
}
}
func TestIAVLNoPrune(t *testing.T) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)
@ -321,6 +353,7 @@ func TestIAVLNoPrune(t *testing.T) {
nextVersion(iavlStore)
}
}
func TestIAVLPruneEverything(t *testing.T) {
db := dbm.NewMemDB()
tree := iavl.NewVersionedTree(db, cacheSize)

11
store/rootmultistore.go
View File

@ -25,6 +25,7 @@ const (
type rootMultiStore struct {
db dbm.DB
lastCommitID CommitID
pruning sdk.PruningStrategy
storesParams map[StoreKey]storeParams
stores map[StoreKey]CommitStore
keysByName map[string]StoreKey
@ -46,6 +47,14 @@ func NewCommitMultiStore(db dbm.DB) *rootMultiStore {
}
}
// Implements CommitMultiStore
func (rs *rootMultiStore) SetPruning(pruning sdk.PruningStrategy) {
rs.pruning = pruning
for _, substore := range rs.stores {
substore.SetPruning(pruning)
}
}
// Implements Store.
func (rs *rootMultiStore) GetStoreType() StoreType {
return sdk.StoreTypeMulti
@ -313,7 +322,7 @@ func (rs *rootMultiStore) loadCommitStoreFromParams(id CommitID, params storePar
// TODO: id?
// return NewCommitMultiStore(db, id)
case sdk.StoreTypeIAVL:
store, err = LoadIAVLStore(db, id)
store, err = LoadIAVLStore(db, id, rs.pruning)
return
case sdk.StoreTypeDB:
panic("dbm.DB is not a CommitStore")

15
types/store.go
View File

@ -11,6 +11,20 @@ import (
// NOTE: These are implemented in cosmos-sdk/store.
// PruningStrategy specfies how old states will be deleted over time
type PruningStrategy uint8
const (
// PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th)
PruneSyncable PruningStrategy = iota
// PruneEverything means all saved states will be deleted, storing only the current state
PruneEverything PruningStrategy = iota
// PruneNothing means all historic states will be saved, nothing will be deleted
PruneNothing PruningStrategy = iota
)
type Store interface { //nolint
GetStoreType() StoreType
CacheWrapper
@ -20,6 +34,7 @@ type Store interface { //nolint
type Committer interface {
Commit() CommitID
LastCommitID() CommitID
SetPruning(PruningStrategy)
}
// Stores of MultiStore must implement CommitStore.