Merge PR #5709: Add granularity options to pruning state

CHANGELOG.md

@@ -148,6 +148,8 @@ Buffers for state serialization instead of Amino.
 * (rest) [\#5648](https://github.com/cosmos/cosmos-sdk/pull/5648) Enhance /txs usability:
   * Add `tx.minheight` key to filter transaction with an inclusive minimum block height
   * Add `tx.maxheight` key to filter transaction with an inclusive maximum block height
+* (server) [\#5709](https://github.com/cosmos/cosmos-sdk/pull/5709) There are two new flags for pruning, `--pruning-keep-every` 
+and `--pruning-snapshot-every` as an alternative to `--pruning`. They allow to fine tune the strategy for pruning the state.
 
 ## [v0.38.1] - 2020-02-11
 

server/pruning.go

@@ -0,0 +1,24 @@
+package server
+
+import (
+	"github.com/cosmos/cosmos-sdk/store"
+	"github.com/spf13/viper"
+)
+
+// GetPruningOptionsFromFlags parses start command flags and returns the correct PruningOptions.
+// flagPruning prevails over flagPruningKeepEvery and flagPruningSnapshotEvery.
+// Default option is PruneSyncable.
+func GetPruningOptionsFromFlags() store.PruningOptions {
+	if viper.IsSet(flagPruning) {
+		return store.NewPruningOptionsFromString(viper.GetString(flagPruning))
+	}
+
+	if viper.IsSet(flagPruningKeepEvery) && viper.IsSet(flagPruningSnapshotEvery) {
+		return store.PruningOptions{
+			KeepEvery:     viper.GetInt64(flagPruningKeepEvery),
+			SnapshotEvery: viper.GetInt64(flagPruningSnapshotEvery),
+		}
+	}
+
+	return store.PruneSyncable
+}

server/pruning_test.go

@@ -0,0 +1,51 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cosmos/cosmos-sdk/store"
+)
+
+func TestGetPruningOptionsFromFlags(t *testing.T) {
+	tests := []struct {
+		name            string
+		initParams      func()
+		expectedOptions store.PruningOptions
+	}{
+		{
+			name: "pruning",
+			initParams: func() {
+				viper.Set(flagPruning, store.PruningStrategyNothing)
+			},
+			expectedOptions: store.PruneNothing,
+		},
+		{
+			name: "granular pruning",
+			initParams: func() {
+				viper.Set(flagPruningSnapshotEvery, 1234)
+				viper.Set(flagPruningKeepEvery, 4321)
+			},
+			expectedOptions: store.PruningOptions{
+				SnapshotEvery: 1234,
+				KeepEvery:     4321,
+			},
+		},
+		{
+			name:            "default",
+			initParams:      func() {},
+			expectedOptions: store.PruneSyncable,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(j *testing.T) {
+			viper.Reset()
+			tt.initParams()
+			require.Equal(t, tt.expectedOptions, GetPruningOptionsFromFlags())
+		})
+	}
+}

server/start.go

@@ -20,16 +20,29 @@ import (
 
 // Tendermint full-node start flags
 const (
-	flagWithTendermint     = "with-tendermint"
+	flagWithTendermint       = "with-tendermint"
-	flagAddress            = "address"
+	flagAddress              = "address"
-	flagTraceStore         = "trace-store"
+	flagTraceStore           = "trace-store"
-	flagPruning            = "pruning"
+	flagPruning              = "pruning"
-	flagCPUProfile         = "cpu-profile"
+	flagPruningKeepEvery     = "pruning-keep-every"
-	FlagMinGasPrices       = "minimum-gas-prices"
+	flagPruningSnapshotEvery = "pruning-snapshot-every"
-	FlagHaltHeight         = "halt-height"
+	flagCPUProfile           = "cpu-profile"
-	FlagHaltTime           = "halt-time"
+	FlagMinGasPrices         = "minimum-gas-prices"
-	FlagInterBlockCache    = "inter-block-cache"
+	FlagHaltHeight           = "halt-height"
-	FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades"
+	FlagHaltTime             = "halt-time"
+	FlagInterBlockCache      = "inter-block-cache"
+	FlagUnsafeSkipUpgrades   = "unsafe-skip-upgrades"
+)
+
+var (
+	errPruningWithGranularOptions = fmt.Errorf(
+		"'--%s' flag is not compatible with granular options  '--%s' or '--%s'",
+		flagPruning, flagPruningKeepEvery, flagPruningSnapshotEvery,
+	)
+	errPruningGranularOptions = fmt.Errorf(
+		"'--%s' and '--%s' must be set together",
+		flagPruningSnapshotEvery, flagPruningKeepEvery,
+	)
 )
 
 // StartCmd runs the service passed in, either stand-alone or in-process with
@@ -41,7 +54,9 @@ func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command {
 		Long: `Run the full node application with Tendermint in or out of process. By
 default, the application will run with Tendermint in process.
 
-Pruning options can be provided via the '--pruning' flag. The options are as follows:
+Pruning options can be provided via the '--pruning' flag or alternatively with '--pruning-snapshot-every' and 'pruning-keep-every' together.
+
+For '--pruning' the options are as follows:
 
 syncable: only those states not needed for state syncing will be deleted (flushes every 100th to disk and keeps every 10000th)
 nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)
@@ -56,6 +71,9 @@ will not be able to commit subsequent blocks.
 For profiling and benchmarking purposes, CPU profiling can be enabled via the '--cpu-profile' flag
 which accepts a path for the resulting pprof file.
 `,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return checkPruningParams()
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if !viper.GetBool(flagWithTendermint) {
 				ctx.Logger.Info("starting ABCI without Tendermint")
@@ -74,6 +92,8 @@ which accepts a path for the resulting pprof file.
 	cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address")
 	cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file")
 	cmd.Flags().String(flagPruning, "syncable", "Pruning strategy: syncable, nothing, everything")
+	cmd.Flags().Int64(flagPruningKeepEvery, 0, "Define the state number that will be kept")
+	cmd.Flags().Int64(flagPruningSnapshotEvery, 0, "Defines the state that will be snapshot for pruning")
 	cmd.Flags().String(
 		FlagMinGasPrices, "",
 		"Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)",
@@ -89,6 +109,27 @@ which accepts a path for the resulting pprof file.
 	return cmd
 }
 
+// checkPruningParams checks that the provided pruning params are correct
+func checkPruningParams() error {
+	if !viper.IsSet(flagPruning) && !viper.IsSet(flagPruningKeepEvery) && !viper.IsSet(flagPruningSnapshotEvery) {
+		return nil
+	}
+
+	if viper.IsSet(flagPruning) {
+		if viper.IsSet(flagPruningKeepEvery) || viper.IsSet(flagPruningSnapshotEvery) {
+			return errPruningWithGranularOptions
+		}
+
+		return nil
+	}
+
+	if !(viper.IsSet(flagPruningKeepEvery) && viper.IsSet(flagPruningSnapshotEvery)) {
+		return errPruningGranularOptions
+	}
+
+	return nil
+}
+
 func startStandAlone(ctx *Context, appCreator AppCreator) error {
 	addr := viper.GetString(flagAddress)
 	home := viper.GetString("home")

server/start_test.go

@@ -0,0 +1,104 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPruningOptions(t *testing.T) {
+	startCommand := StartCmd(nil, nil)
+
+	tests := []struct {
+		name        string
+		paramInit   func()
+		returnsErr  bool
+		expectedErr error
+	}{
+		{
+			name:        "none set, returns nil and will use default from flags",
+			paramInit:   func() {},
+			returnsErr:  false,
+			expectedErr: nil,
+		},
+		{
+			name: "only keep-every provided",
+			paramInit: func() {
+				viper.Set(flagPruningKeepEvery, 12345)
+			},
+			returnsErr:  true,
+			expectedErr: errPruningGranularOptions,
+		},
+		{
+			name: "only snapshot-every provided",
+			paramInit: func() {
+				viper.Set(flagPruningSnapshotEvery, 12345)
+			},
+			returnsErr:  true,
+			expectedErr: errPruningGranularOptions,
+		},
+		{
+			name: "pruning flag with other granular options 1",
+			paramInit: func() {
+				viper.Set(flagPruning, "set")
+				viper.Set(flagPruningSnapshotEvery, 1234)
+			},
+			returnsErr:  true,
+			expectedErr: errPruningWithGranularOptions,
+		},
+		{
+			name: "pruning flag with other granular options 2",
+			paramInit: func() {
+				viper.Set(flagPruning, "set")
+				viper.Set(flagPruningKeepEvery, 1234)
+			},
+			returnsErr:  true,
+			expectedErr: errPruningWithGranularOptions,
+		},
+		{
+			name: "pruning flag with other granular options 3",
+			paramInit: func() {
+				viper.Set(flagPruning, "set")
+				viper.Set(flagPruningKeepEvery, 1234)
+				viper.Set(flagPruningSnapshotEvery, 1234)
+			},
+			returnsErr:  true,
+			expectedErr: errPruningWithGranularOptions,
+		},
+		{
+			name: "only prunning set",
+			paramInit: func() {
+				viper.Set(flagPruning, "set")
+			},
+			returnsErr:  false,
+			expectedErr: nil,
+		},
+		{
+			name: "only granular set",
+			paramInit: func() {
+				viper.Set(flagPruningSnapshotEvery, 12345)
+				viper.Set(flagPruningKeepEvery, 12345)
+			},
+			returnsErr:  false,
+			expectedErr: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			viper.Reset()
+			tt.paramInit()
+
+			err := startCommand.PreRunE(nil, nil)
+
+			if tt.returnsErr {
+				require.EqualError(t, err, tt.expectedErr.Error())
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}