Merge PR #7265: Tendermint Block Pruning
This commit is contained in:
parent
56e3bc1945
commit
7ae84898de
|
@ -162,6 +162,7 @@ be used to retrieve the actual proposal `Content`. Also the `NewMsgSubmitProposa
|
|||
|
||||
### Features
|
||||
|
||||
* [\#7265](https://github.com/cosmos/cosmos-sdk/pull/7265) Support Tendermint block pruning through a new `min-retain-blocks` configuration that can be set in either `app.toml` or via the CLI. This parameter is used in conjunction with other criteria to determine the height at which Tendermint should prune blocks.
|
||||
* (vesting) [\#7209](https://github.com/cosmos/cosmos-sdk/pull/7209) Create new `MsgCreateVestingAccount` message type along with CLI handler that allows for the creation of delayed and continuous vesting types.
|
||||
* (events) [\#7121](https://github.com/cosmos/cosmos-sdk/pull/7121) The application now drives what events are indexed by Tendermint via the `index-events` configuration in `app.toml`, which is a list of events taking the form `{eventType}.{attributeKey}`.
|
||||
* [\#6089](https://github.com/cosmos/cosmos-sdk/pull/6089) Transactions can now have a `TimeoutHeight` set which allows the transaction to be rejected if it's committed at a height greater than the timeout.
|
||||
|
|
|
@ -294,6 +294,7 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) {
|
|||
defer telemetry.MeasureSince(time.Now(), "abci", "commit")
|
||||
|
||||
header := app.deliverState.ctx.BlockHeader()
|
||||
retainHeight := app.GetBlockRetentionHeight(header.Height)
|
||||
|
||||
// Write the DeliverTx state which is cache-wrapped and commit the MultiStore.
|
||||
// The write to the DeliverTx state writes all state transitions to the root
|
||||
|
@ -334,7 +335,8 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) {
|
|||
}
|
||||
|
||||
return abci.ResponseCommit{
|
||||
Data: commitID.Hash,
|
||||
Data: commitID.Hash,
|
||||
RetainHeight: retainHeight,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -578,6 +580,93 @@ func (app *BaseApp) createQueryContext(height int64, prove bool) (sdk.Context, e
|
|||
return ctx, nil
|
||||
}
|
||||
|
||||
// GetBlockRetentionHeight returns the height for which all blocks below this height
|
||||
// are pruned from Tendermint. Given a commitment height and a non-zero local
|
||||
// minRetainBlocks configuration, the retentionHeight is the smallest height that
|
||||
// satisfies:
|
||||
//
|
||||
// - Unbonding (safety threshold) time: The block interval in which validators
|
||||
// can be economically punished for misbehavior. Blocks in this interval must be
|
||||
// auditable e.g. by the light client.
|
||||
//
|
||||
// - Logical store snapshot interval: The block interval at which the underlying
|
||||
// logical store database is persisted to disk, e.g. every 10000 heights. Blocks
|
||||
// since the last IAVL snapshot must be available for replay on application restart.
|
||||
//
|
||||
// - State sync snapshots: Blocks since the oldest available snapshot must be
|
||||
// available for state sync nodes to catch up (oldest because a node may be
|
||||
// restoring an old snapshot while a new snapshot was taken).
|
||||
//
|
||||
// - Local (minRetainBlocks) config: Archive nodes may want to retain more or
|
||||
// all blocks, e.g. via a local config option min-retain-blocks. There may also
|
||||
// be a need to vary retention for other nodes, e.g. sentry nodes which do not
|
||||
// need historical blocks.
|
||||
func (app *BaseApp) GetBlockRetentionHeight(commitHeight int64) int64 {
|
||||
// pruning is disabled if minRetainBlocks is zero
|
||||
if app.minRetainBlocks == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
minNonZero := func(x, y int64) int64 {
|
||||
switch {
|
||||
case x == 0:
|
||||
return y
|
||||
case y == 0:
|
||||
return x
|
||||
case x < y:
|
||||
return x
|
||||
default:
|
||||
return y
|
||||
}
|
||||
}
|
||||
|
||||
// Define retentionHeight as the minimum value that satisfies all non-zero
|
||||
// constraints. All blocks below (commitHeight-retentionHeight) are pruned
|
||||
// from Tendermint.
|
||||
var retentionHeight int64
|
||||
|
||||
// Define the number of blocks needed to protect against misbehaving validators
|
||||
// which allows light clients to operate safely. Note, we piggy back of the
|
||||
// evidence parameters instead of computing an estimated nubmer of blocks based
|
||||
// on the unbonding period and block commitment time as the two should be
|
||||
// equivalent.
|
||||
cp := app.GetConsensusParams(app.deliverState.ctx)
|
||||
if cp != nil && cp.Evidence != nil && cp.Evidence.MaxAgeNumBlocks > 0 {
|
||||
retentionHeight = commitHeight - cp.Evidence.MaxAgeNumBlocks
|
||||
}
|
||||
|
||||
// Define the state pruning offset, i.e. the block offset at which the
|
||||
// underlying logical database is persisted to disk.
|
||||
statePruningOffset := int64(app.cms.GetPruning().KeepEvery)
|
||||
if statePruningOffset > 0 {
|
||||
if commitHeight > statePruningOffset {
|
||||
v := commitHeight - (commitHeight % statePruningOffset)
|
||||
retentionHeight = minNonZero(retentionHeight, v)
|
||||
} else {
|
||||
// Hitting this case means we have persisting enabled but have yet to reach
|
||||
// a height in which we persist state, so we return zero regardless of other
|
||||
// conditions. Otherwise, we could end up pruning blocks without having
|
||||
// any state committed to disk.
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
if app.snapshotInterval > 0 && app.snapshotKeepRecent > 0 {
|
||||
v := commitHeight - int64((app.snapshotInterval * uint64(app.snapshotKeepRecent)))
|
||||
retentionHeight = minNonZero(retentionHeight, v)
|
||||
}
|
||||
|
||||
v := commitHeight - int64(app.minRetainBlocks)
|
||||
retentionHeight = minNonZero(retentionHeight, v)
|
||||
|
||||
if retentionHeight <= 0 {
|
||||
// prune nothing in the case of a non-positive height
|
||||
return 0
|
||||
}
|
||||
|
||||
return retentionHeight
|
||||
}
|
||||
|
||||
func handleQueryApp(app *BaseApp, path []string, req abci.RequestQuery) abci.ResponseQuery {
|
||||
if len(path) >= 2 {
|
||||
switch path[1] {
|
||||
|
|
|
@ -0,0 +1,118 @@
|
|||
package baseapp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
abci "github.com/tendermint/tendermint/abci/types"
|
||||
tmprototypes "github.com/tendermint/tendermint/proto/tendermint/types"
|
||||
dbm "github.com/tendermint/tm-db"
|
||||
|
||||
sdk "github.com/cosmos/cosmos-sdk/types"
|
||||
)
|
||||
|
||||
func TestGetBlockRentionHeight(t *testing.T) {
|
||||
logger := defaultLogger()
|
||||
db := dbm.NewMemDB()
|
||||
name := t.Name()
|
||||
|
||||
testCases := map[string]struct {
|
||||
bapp *BaseApp
|
||||
maxAgeBlocks int64
|
||||
commitHeight int64
|
||||
expected int64
|
||||
}{
|
||||
"defaults": {
|
||||
bapp: NewBaseApp(name, logger, db, nil),
|
||||
maxAgeBlocks: 0,
|
||||
commitHeight: 499000,
|
||||
expected: 0,
|
||||
},
|
||||
"pruning unbonding time only": {
|
||||
bapp: NewBaseApp(name, logger, db, nil, SetMinRetainBlocks(1)),
|
||||
maxAgeBlocks: 362880,
|
||||
commitHeight: 499000,
|
||||
expected: 136120,
|
||||
},
|
||||
"pruning iavl snapshot only": {
|
||||
bapp: NewBaseApp(
|
||||
name, logger, db, nil,
|
||||
SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
|
||||
SetMinRetainBlocks(1),
|
||||
),
|
||||
maxAgeBlocks: 0,
|
||||
commitHeight: 499000,
|
||||
expected: 490000,
|
||||
},
|
||||
"pruning state sync snapshot only": {
|
||||
bapp: NewBaseApp(
|
||||
name, logger, db, nil,
|
||||
SetSnapshotInterval(50000),
|
||||
SetSnapshotKeepRecent(3),
|
||||
SetMinRetainBlocks(1),
|
||||
),
|
||||
maxAgeBlocks: 0,
|
||||
commitHeight: 499000,
|
||||
expected: 349000,
|
||||
},
|
||||
"pruning min retention only": {
|
||||
bapp: NewBaseApp(
|
||||
name, logger, db, nil,
|
||||
SetMinRetainBlocks(400000),
|
||||
),
|
||||
maxAgeBlocks: 0,
|
||||
commitHeight: 499000,
|
||||
expected: 99000,
|
||||
},
|
||||
"pruning all conditions": {
|
||||
bapp: NewBaseApp(
|
||||
name, logger, db, nil,
|
||||
SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
|
||||
SetMinRetainBlocks(400000),
|
||||
SetSnapshotInterval(50000), SetSnapshotKeepRecent(3),
|
||||
),
|
||||
maxAgeBlocks: 362880,
|
||||
commitHeight: 499000,
|
||||
expected: 99000,
|
||||
},
|
||||
"no pruning due to no persisted state": {
|
||||
bapp: NewBaseApp(
|
||||
name, logger, db, nil,
|
||||
SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
|
||||
SetMinRetainBlocks(400000),
|
||||
SetSnapshotInterval(50000), SetSnapshotKeepRecent(3),
|
||||
),
|
||||
maxAgeBlocks: 362880,
|
||||
commitHeight: 10000,
|
||||
expected: 0,
|
||||
},
|
||||
"disable pruning": {
|
||||
bapp: NewBaseApp(
|
||||
name, logger, db, nil,
|
||||
SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
|
||||
SetMinRetainBlocks(0),
|
||||
SetSnapshotInterval(50000), SetSnapshotKeepRecent(3),
|
||||
),
|
||||
maxAgeBlocks: 362880,
|
||||
commitHeight: 499000,
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
tc := tc
|
||||
|
||||
tc.bapp.SetParamStore(¶mStore{db: dbm.NewMemDB()})
|
||||
tc.bapp.InitChain(abci.RequestInitChain{
|
||||
ConsensusParams: &abci.ConsensusParams{
|
||||
Evidence: &tmprototypes.EvidenceParams{
|
||||
MaxAgeNumBlocks: tc.maxAgeBlocks,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
t.Run(name, func(t *testing.T) {
|
||||
require.Equal(t, tc.expected, tc.bapp.GetBlockRetentionHeight(tc.commitHeight))
|
||||
})
|
||||
}
|
||||
}
|
|
@ -101,6 +101,18 @@ type BaseApp struct { // nolint: maligned
|
|||
// minimum block time (in Unix seconds) at which to halt the chain and gracefully shutdown
|
||||
haltTime uint64
|
||||
|
||||
// minRetainBlocks defines the minimum block height offset from the current
|
||||
// block being committed, such that all blocks past this offset are pruned
|
||||
// from Tendermint. It is used as part of the process of determining the
|
||||
// ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
|
||||
// that no blocks should be pruned.
|
||||
//
|
||||
// Note: Tendermint block pruning is dependant on this parameter in conunction
|
||||
// with the unbonding (safety threshold) period, state pruning and state sync
|
||||
// snapshot parameters to determine the correct minimum value of
|
||||
// ResponseCommit.RetainHeight.
|
||||
minRetainBlocks uint64
|
||||
|
||||
// application's version string
|
||||
appVersion string
|
||||
|
||||
|
@ -298,6 +310,10 @@ func (app *BaseApp) setHaltTime(haltTime uint64) {
|
|||
app.haltTime = haltTime
|
||||
}
|
||||
|
||||
func (app *BaseApp) setMinRetainBlocks(minRetainBlocks uint64) {
|
||||
app.minRetainBlocks = minRetainBlocks
|
||||
}
|
||||
|
||||
func (app *BaseApp) setInterBlockCache(cache sdk.MultiStorePersistentCache) {
|
||||
app.interBlockCache = cache
|
||||
}
|
||||
|
|
|
@ -39,6 +39,13 @@ func SetHaltTime(haltTime uint64) func(*BaseApp) {
|
|||
return func(bap *BaseApp) { bap.setHaltTime(haltTime) }
|
||||
}
|
||||
|
||||
// SetMinRetainBlocks returns a BaseApp option function that sets the minimum
|
||||
// block retention height value when determining which heights to prune during
|
||||
// ABCI Commit.
|
||||
func SetMinRetainBlocks(minRetainBlocks uint64) func(*BaseApp) {
|
||||
return func(bapp *BaseApp) { bapp.setMinRetainBlocks(minRetainBlocks) }
|
||||
}
|
||||
|
||||
// SetTrace will turn on or off trace flag
|
||||
func SetTrace(trace bool) func(*BaseApp) {
|
||||
return func(app *BaseApp) { app.setTrace(trace) }
|
||||
|
|
|
@ -43,6 +43,22 @@ type BaseConfig struct {
|
|||
// Note: Commitment of state will be attempted on the corresponding block.
|
||||
HaltTime uint64 `mapstructure:"halt-time"`
|
||||
|
||||
// MinRetainBlocks defines the minimum block height offset from the current
|
||||
// block being committed, such that blocks past this offset may be pruned
|
||||
// from Tendermint. It is used as part of the process of determining the
|
||||
// ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
|
||||
// that no blocks should be pruned.
|
||||
//
|
||||
// This configuration value is only responsible for pruning Tendermint blocks.
|
||||
// It has no bearing on application state pruning which is determined by the
|
||||
// "pruning-*" configurations.
|
||||
//
|
||||
// Note: Tendermint block pruning is dependant on this parameter in conunction
|
||||
// with the unbonding (safety threshold) period, state pruning and state sync
|
||||
// snapshot parameters to determine the correct minimum value of
|
||||
// ResponseCommit.RetainHeight.
|
||||
MinRetainBlocks uint64 `mapstructure:"min-retain-blocks"`
|
||||
|
||||
// InterBlockCache enables inter-block caching.
|
||||
InterBlockCache bool `mapstructure:"inter-block-cache"`
|
||||
|
||||
|
@ -150,6 +166,7 @@ func DefaultConfig() *Config {
|
|||
PruningKeepRecent: "0",
|
||||
PruningKeepEvery: "0",
|
||||
PruningInterval: "0",
|
||||
MinRetainBlocks: 0,
|
||||
IndexEvents: make([]string, 0),
|
||||
},
|
||||
Telemetry: telemetry.Config{
|
||||
|
@ -197,6 +214,7 @@ func GetConfig(v *viper.Viper) Config {
|
|||
HaltHeight: v.GetUint64("halt-height"),
|
||||
HaltTime: v.GetUint64("halt-time"),
|
||||
IndexEvents: v.GetStringSlice("index-events"),
|
||||
MinRetainBlocks: v.GetUint64("min-retain-blocks"),
|
||||
},
|
||||
Telemetry: telemetry.Config{
|
||||
ServiceName: v.GetString("telemetry.service-name"),
|
||||
|
|
|
@ -44,6 +44,22 @@ halt-height = {{ .BaseConfig.HaltHeight }}
|
|||
# Note: Commitment of state will be attempted on the corresponding block.
|
||||
halt-time = {{ .BaseConfig.HaltTime }}
|
||||
|
||||
# MinRetainBlocks defines the minimum block height offset from the current
|
||||
# block being committed, such that all blocks past this offset are pruned
|
||||
# from Tendermint. It is used as part of the process of determining the
|
||||
# ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
|
||||
# that no blocks should be pruned.
|
||||
#
|
||||
# This configuration value is only responsible for pruning Tendermint blocks.
|
||||
# It has no bearing on application state pruning which is determined by the
|
||||
# "pruning-*" configurations.
|
||||
#
|
||||
# Note: Tendermint block pruning is dependant on this parameter in conunction
|
||||
# with the unbonding (safety threshold) period, state pruning and state sync
|
||||
# snapshot parameters to determine the correct minimum value of
|
||||
# ResponseCommit.RetainHeight.
|
||||
min-retain-blocks = {{ .BaseConfig.MinRetainBlocks }}
|
||||
|
||||
# InterBlockCache enables inter-block caching.
|
||||
inter-block-cache = {{ .BaseConfig.InterBlockCache }}
|
||||
|
||||
|
|
|
@ -55,6 +55,10 @@ func (ms multiStore) SetPruning(opts sdk.PruningOptions) {
|
|||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (ms multiStore) GetPruning() sdk.PruningOptions {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (ms multiStore) GetCommitKVStore(key sdk.StoreKey) sdk.CommitKVStore {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
|
|
@ -48,6 +48,7 @@ const (
|
|||
FlagPruningKeepEvery = "pruning-keep-every"
|
||||
FlagPruningInterval = "pruning-interval"
|
||||
FlagIndexEvents = "index-events"
|
||||
FlagMinRetainBlocks = "min-retain-blocks"
|
||||
)
|
||||
|
||||
// GRPC-related flags.
|
||||
|
@ -135,6 +136,7 @@ which accepts a path for the resulting pprof file.
|
|||
cmd.Flags().Uint64(FlagPruningKeepEvery, 0, "Offset heights to keep on disk after 'keep-every' (ignored if pruning is not 'custom')")
|
||||
cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')")
|
||||
cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks")
|
||||
cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks")
|
||||
|
||||
cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled")
|
||||
cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on")
|
||||
|
|
|
@ -196,6 +196,7 @@ func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, appOpts serverty
|
|||
baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))),
|
||||
baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))),
|
||||
baseapp.SetHaltTime(cast.ToUint64(appOpts.Get(server.FlagHaltTime))),
|
||||
baseapp.SetMinRetainBlocks(cast.ToUint64(appOpts.Get(server.FlagMinRetainBlocks))),
|
||||
baseapp.SetInterBlockCache(cache),
|
||||
baseapp.SetTrace(cast.ToBool(appOpts.Get(server.FlagTrace))),
|
||||
baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),
|
||||
|
|
|
@ -125,6 +125,12 @@ func (st *Store) SetPruning(_ types.PruningOptions) {
|
|||
panic("cannot set pruning options on an initialized IAVL store")
|
||||
}
|
||||
|
||||
// SetPruning panics as pruning options should be provided at initialization
|
||||
// since IAVl accepts pruning options directly.
|
||||
func (st *Store) GetPruning() types.PruningOptions {
|
||||
panic("cannot get pruning options on an initialized IAVL store")
|
||||
}
|
||||
|
||||
// VersionExists returns whether or not a given version is stored.
|
||||
func (st *Store) VersionExists(version int64) bool {
|
||||
return st.tree.VersionExists(version)
|
||||
|
|
|
@ -49,4 +49,9 @@ func (s Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.Cach
|
|||
func (s *Store) Commit() (id types.CommitID) { return }
|
||||
|
||||
func (s *Store) SetPruning(pruning types.PruningOptions) {}
|
||||
func (s Store) LastCommitID() (id types.CommitID) { return }
|
||||
|
||||
// GetPruning is a no-op as pruning options cannot be directly set on this store.
|
||||
// They must be set on the root commit multi-store.
|
||||
func (s *Store) GetPruning() types.PruningOptions { return types.PruningOptions{} }
|
||||
|
||||
func (s Store) LastCommitID() (id types.CommitID) { return }
|
||||
|
|
|
@ -31,3 +31,7 @@ func (cdsa commitDBStoreAdapter) LastCommitID() types.CommitID {
|
|||
}
|
||||
|
||||
func (cdsa commitDBStoreAdapter) SetPruning(_ types.PruningOptions) {}
|
||||
|
||||
// GetPruning is a no-op as pruning options cannot be directly set on this store.
|
||||
// They must be set on the root commit multi-store.
|
||||
func (cdsa commitDBStoreAdapter) GetPruning() types.PruningOptions { return types.PruningOptions{} }
|
||||
|
|
|
@ -27,9 +27,11 @@ func (ts *Store) Commit() (id types.CommitID) {
|
|||
return
|
||||
}
|
||||
|
||||
// Implements CommitStore
|
||||
func (ts *Store) SetPruning(pruning types.PruningOptions) {
|
||||
}
|
||||
func (ts *Store) SetPruning(_ types.PruningOptions) {}
|
||||
|
||||
// GetPruning is a no-op as pruning options cannot be directly set on this store.
|
||||
// They must be set on the root commit multi-store.
|
||||
func (ts *Store) GetPruning() types.PruningOptions { return types.PruningOptions{} }
|
||||
|
||||
// Implements CommitStore
|
||||
func (ts *Store) LastCommitID() (id types.CommitID) {
|
||||
|
|
|
@ -21,8 +21,8 @@ type Committer interface {
|
|||
Commit() CommitID
|
||||
LastCommitID() CommitID
|
||||
|
||||
// TODO: Deprecate after 0.38.5
|
||||
SetPruning(PruningOptions)
|
||||
GetPruning() PruningOptions
|
||||
}
|
||||
|
||||
// Stores of MultiStore must implement CommitStore.
|
||||
|
|
Loading…
Reference in New Issue