Merge PR #3333: F1 storage efficiency improvements

PENDING.md

@@ -45,6 +45,7 @@ BREAKING CHANGES
     * `Delegation` -> `Value` in `MsgCreateValidator` and `MsgDelegate`
     * `MsgBeginUnbonding` -> `MsgUndelegate`
   * [\#3315] Increase decimal precision to 18
+  * \#3333 - F1 storage efficiency improvements - automatic withdrawals when unbonded, historical reward reference counting
   * \#3323 Update to Tendermint 0.29.0
   * [\#3328](https://github.com/cosmos/cosmos-sdk/issues/3328) [x/gov] Remove redundant action tag
 

cmd/gaia/app/export.go

@@ -74,10 +74,10 @@ func (app *GaiaApp) prepForZeroHeightGenesis(ctx sdk.Context) {
 	}
 
 	// clear validator slash events
-	app.distrKeeper.DeleteValidatorSlashEvents(ctx)
+	app.distrKeeper.DeleteAllValidatorSlashEvents(ctx)
 
 	// clear validator historical rewards
-	app.distrKeeper.DeleteValidatorHistoricalRewards(ctx)
+	app.distrKeeper.DeleteAllValidatorHistoricalRewards(ctx)
 
 	// set context height to zero
 	height := ctx.BlockHeight()

docs/spec/distribution/f1_reference_counting.md

@@ -0,0 +1,18 @@
+## Reference Counting in F1 Fee Distribution
+
+In F1 fee distribution, in order to calculate the rewards a delegator ought to receive when they
+withdraw their delegation, we must read the terms of the summation of rewards divided by tokens from
+the period which they ended when they delegated, and the final period (created when they withdraw).
+
+Additionally, as slashes change the amount of tokens a delegation will have (but we calculate this lazily,
+only when a delegator un-delegates), we must calculate rewards in separate periods before / after any slashes
+which occurred in between when a delegator delegated and when they withdrew their rewards. Thus slashes, like
+delegations, reference the period which was ended by the slash event.
+
+All stored historical rewards records for periods which are no longer referenced by any delegations
+or any slashes can thus be safely removed, as they will never be read (future delegations and future
+slashes will always reference future periods). This is implemented by tracking a `ReferenceCount`
+along with each historical reward storage entry. Each time a new object (delegation or slash)
+is created which might need to reference the historical record, the reference count is incremented.
+Each time one object which previously needed to reference the historical record is deleted, the reference
+count is decremented. If the reference count hits zero, the historical record is deleted.

x/distribution/keeper/delegation.go

@@ -8,14 +8,16 @@ import (
 
 // initialize starting info for a new delegation
 func (k Keeper) initializeDelegation(ctx sdk.Context, val sdk.ValAddress, del sdk.AccAddress) {
-	// period has already been incremented
+	// period has already been incremented - we want to store the period ended by this delegation action
-	period := k.GetValidatorCurrentRewards(ctx, val).Period
+	previousPeriod := k.GetValidatorCurrentRewards(ctx, val).Period - 1
+
 	validator := k.stakingKeeper.Validator(ctx, val)
 	delegation := k.stakingKeeper.Delegation(ctx, del, val)
+
 	// calculate delegation stake in tokens
 	// we don't store directly, so multiply delegation shares * (tokens per share)
 	stake := delegation.GetShares().Mul(validator.GetDelegatorShareExRate())
-	k.SetDelegatorStartingInfo(ctx, val, del, types.NewDelegatorStartingInfo(period-1, stake, uint64(ctx.BlockHeight())))
+	k.SetDelegatorStartingInfo(ctx, val, del, types.NewDelegatorStartingInfo(previousPeriod, stake, uint64(ctx.BlockHeight())))
 }
 
 // calculate the rewards accrued by a delegation between two periods
@@ -29,7 +31,7 @@ func (k Keeper) calculateDelegationRewardsBetween(ctx sdk.Context, val sdk.Valid
 	// return staking * (ending - starting)
 	starting := k.GetValidatorHistoricalRewards(ctx, val.GetOperator(), startingPeriod)
 	ending := k.GetValidatorHistoricalRewards(ctx, val.GetOperator(), endingPeriod)
-	difference := ending.Minus(starting)
+	difference := ending.CumulativeRewardRatio.Minus(starting.CumulativeRewardRatio)
 	rewards = difference.MulDec(staking)
 	return
 }
@@ -50,7 +52,7 @@ func (k Keeper) calculateDelegationRewards(ctx sdk.Context, val sdk.Validator, d
 	if endingHeight >= startingHeight {
 		k.IterateValidatorSlashEventsBetween(ctx, del.GetValidatorAddr(), startingHeight, endingHeight,
 			func(height uint64, event types.ValidatorSlashEvent) (stop bool) {
-				endingPeriod := event.ValidatorPeriod - 1
+				endingPeriod := event.ValidatorPeriod
 				rewards = rewards.Plus(k.calculateDelegationRewardsBetween(ctx, val, startingPeriod, endingPeriod, stake))
 				stake = stake.Mul(sdk.OneDec().Sub(event.Fraction))
 				startingPeriod = endingPeriod
@@ -67,10 +69,20 @@ func (k Keeper) calculateDelegationRewards(ctx sdk.Context, val sdk.Validator, d
 
 func (k Keeper) withdrawDelegationRewards(ctx sdk.Context, val sdk.Validator, del sdk.Delegation) sdk.Error {
 
+	// check existence of delegator starting info
+	if !k.HasDelegatorStartingInfo(ctx, del.GetValidatorAddr(), del.GetDelegatorAddr()) {
+		return types.ErrNoDelegationDistInfo(k.codespace)
+	}
+
 	// end current period and calculate rewards
 	endingPeriod := k.incrementValidatorPeriod(ctx, val)
 	rewards := k.calculateDelegationRewards(ctx, val, del, endingPeriod)
 
+	// decrement reference count of starting period
+	startingInfo := k.GetDelegatorStartingInfo(ctx, del.GetValidatorAddr(), del.GetDelegatorAddr())
+	startingPeriod := startingInfo.PreviousPeriod
+	k.decrementReferenceCount(ctx, del.GetValidatorAddr(), startingPeriod)
+
 	// truncate coins, return remainder to community pool
 	coins, remainder := rewards.TruncateDecimal()
 	outstanding := k.GetOutstandingRewards(ctx)
@@ -85,5 +97,8 @@ func (k Keeper) withdrawDelegationRewards(ctx sdk.Context, val sdk.Validator, de
 		return err
 	}
 
+	// remove delegator starting info
+	k.DeleteDelegatorStartingInfo(ctx, del.GetValidatorAddr(), del.GetDelegatorAddr())
+
 	return nil
 }

x/distribution/keeper/delegation_test.go

@@ -29,9 +29,15 @@ func TestCalculateRewardsBasic(t *testing.T) {
 	val := sk.Validator(ctx, valOpAddr1)
 	del := sk.Delegation(ctx, sdk.AccAddress(valOpAddr1), valOpAddr1)
 
+	// historical count should be 2 (once for validator init, once for delegation init)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// end period
 	endingPeriod := k.incrementValidatorPeriod(ctx, val)
 
+	// historical count should be 3 (since we ended the period, and haven't yet decremented a reference)
+	require.Equal(t, uint64(3), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// calculate delegation rewards
 	rewards := k.calculateDelegationRewards(ctx, val, del, endingPeriod)
 
@@ -276,9 +282,15 @@ func TestWithdrawDelegationRewardsBasic(t *testing.T) {
 	tokens := sdk.DecCoins{{staking.DefaultBondDenom, sdk.NewDec(initial)}}
 	k.AllocateTokensToValidator(ctx, val, tokens)
 
+	// historical count should be 2 (initial + latest for delegation)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// withdraw rewards
 	require.Nil(t, k.WithdrawDelegationRewards(ctx, sdk.AccAddress(valOpAddr1), valOpAddr1))
 
+	// historical count should still be 2 (added one record, cleared one)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// assert correct balance
 	require.Equal(t, sdk.Coins{{staking.DefaultBondDenom, sdk.NewInt(balance - bond + (initial / 2))}}, ak.GetAccount(ctx, sdk.AccAddress(valOpAddr1)).GetCoins())
 
@@ -447,10 +459,16 @@ func TestCalculateRewardsMultiDelegatorMultWithdraw(t *testing.T) {
 	tokens := sdk.DecCoins{{staking.DefaultBondDenom, sdk.NewDec(initial)}}
 	k.AllocateTokensToValidator(ctx, val, tokens)
 
+	// historical count should be 2 (validator init, delegation init)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// second delegation
 	msg2 := staking.NewMsgDelegate(sdk.AccAddress(valOpAddr2), valOpAddr1, sdk.NewCoin(staking.DefaultBondDenom, sdk.NewInt(100)))
 	require.True(t, sh(ctx, msg2).IsOK())
 
+	// historical count should be 3 (second delegation init)
+	require.Equal(t, uint64(3), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// fetch updated validator
 	val = sk.Validator(ctx, valOpAddr1)
 	del2 := sk.Delegation(ctx, sdk.AccAddress(valOpAddr2), valOpAddr1)
@@ -467,6 +485,9 @@ func TestCalculateRewardsMultiDelegatorMultWithdraw(t *testing.T) {
 	// second delegator withdraws
 	k.WithdrawDelegationRewards(ctx, sdk.AccAddress(valOpAddr2), valOpAddr1)
 
+	// historical count should be 3 (validator init + two delegations)
+	require.Equal(t, uint64(3), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// validator withdraws commission
 	k.WithdrawValidatorCommission(ctx, valOpAddr1)
 

x/distribution/keeper/hooks.go

@@ -20,11 +20,41 @@ func (h Hooks) AfterValidatorCreated(ctx sdk.Context, valAddr sdk.ValAddress) {
 	h.k.initializeValidator(ctx, val)
 }
 func (h Hooks) BeforeValidatorModified(ctx sdk.Context, valAddr sdk.ValAddress) {
-	val := h.k.stakingKeeper.Validator(ctx, valAddr)
-	// increment period
-	h.k.incrementValidatorPeriod(ctx, val)
 }
 func (h Hooks) AfterValidatorRemoved(ctx sdk.Context, _ sdk.ConsAddress, valAddr sdk.ValAddress) {
+	// force-withdraw commission
+	commission := h.k.GetValidatorAccumulatedCommission(ctx, valAddr)
+	if !commission.IsZero() {
+		coins, remainder := commission.TruncateDecimal()
+
+		// remainder to community pool
+		feePool := h.k.GetFeePool(ctx)
+		feePool.CommunityPool = feePool.CommunityPool.Plus(remainder)
+		h.k.SetFeePool(ctx, feePool)
+
+		// update outstanding
+		outstanding := h.k.GetOutstandingRewards(ctx)
+		h.k.SetOutstandingRewards(ctx, outstanding.Minus(commission))
+
+		// add to validator account
+		accAddr := sdk.AccAddress(valAddr)
+		withdrawAddr := h.k.GetDelegatorWithdrawAddr(ctx, accAddr)
+
+		if _, _, err := h.k.bankKeeper.AddCoins(ctx, withdrawAddr, coins); err != nil {
+			panic(err)
+		}
+	}
+	// remove commission record
+	h.k.DeleteValidatorAccumulatedCommission(ctx, valAddr)
+
+	// clear slashes
+	h.k.DeleteValidatorSlashEvents(ctx, valAddr)
+
+	// clear historical rewards
+	h.k.DeleteValidatorHistoricalRewards(ctx, valAddr)
+
+	// clear current rewards
+	h.k.DeleteValidatorCurrentRewards(ctx, valAddr)
 }
 func (h Hooks) BeforeDelegationCreated(ctx sdk.Context, delAddr sdk.AccAddress, valAddr sdk.ValAddress) {
 	val := h.k.stakingKeeper.Validator(ctx, valAddr)
@@ -42,7 +72,7 @@ func (h Hooks) BeforeDelegationSharesModified(ctx sdk.Context, delAddr sdk.AccAd
 	}
 }
 func (h Hooks) BeforeDelegationRemoved(ctx sdk.Context, delAddr sdk.AccAddress, valAddr sdk.ValAddress) {
-	// nothing needed here since OnDelegationSharesModified will always also be called
+	// nothing needed here since BeforeDelegationSharesModified will always also be called
 }
 func (h Hooks) AfterDelegationModified(ctx sdk.Context, delAddr sdk.AccAddress, valAddr sdk.ValAddress) {
 	// create new delegation period record

x/distribution/keeper/key.go

@@ -112,6 +112,11 @@ func GetDelegatorStartingInfoKey(v sdk.ValAddress, d sdk.AccAddress) []byte {
 	return append(append(DelegatorStartingInfoPrefix, v.Bytes()...), d.Bytes()...)
 }
 
+// gets the prefix key for a validator's historical rewards
+func GetValidatorHistoricalRewardsPrefix(v sdk.ValAddress) []byte {
+	return append(ValidatorHistoricalRewardsPrefix, v.Bytes()...)
+}
+
 // gets the key for a validator's historical rewards
 func GetValidatorHistoricalRewardsKey(v sdk.ValAddress, k uint64) []byte {
 	b := make([]byte, 8)
@@ -129,6 +134,11 @@ func GetValidatorAccumulatedCommissionKey(v sdk.ValAddress) []byte {
 	return append(ValidatorAccumulatedCommissionPrefix, v.Bytes()...)
 }
 
+// gets the prefix key for a validator's slash fractions
+func GetValidatorSlashEventPrefix(v sdk.ValAddress) []byte {
+	return append(ValidatorSlashEventPrefix, v.Bytes()...)
+}
+
 // gets the key for a validator's slash fraction
 func GetValidatorSlashEventKey(v sdk.ValAddress, height uint64) []byte {
 	b := make([]byte, 8)

x/distribution/keeper/querier.go

@@ -3,10 +3,11 @@ package keeper
 import (
 	"fmt"
 
+	abci "github.com/tendermint/tendermint/abci/types"
+
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/distribution/types"
-	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 // nolint

x/distribution/keeper/querier_test.go

@@ -6,11 +6,12 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	abci "github.com/tendermint/tendermint/abci/types"
+
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/distribution/types"
 	"github.com/cosmos/cosmos-sdk/x/staking"
-	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 const custom = "custom"

x/distribution/keeper/store.go

@@ -21,8 +21,8 @@ func (k Keeper) SetDelegatorWithdrawAddr(ctx sdk.Context, delAddr, withdrawAddr
 	store.Set(GetDelegatorWithdrawAddrKey(delAddr), withdrawAddr.Bytes())
 }
 
-// remove a delegator withdraw addr
+// delete a delegator withdraw addr
-func (k Keeper) RemoveDelegatorWithdrawAddr(ctx sdk.Context, delAddr, withdrawAddr sdk.AccAddress) {
+func (k Keeper) DeleteDelegatorWithdrawAddr(ctx sdk.Context, delAddr, withdrawAddr sdk.AccAddress) {
 	store := ctx.KVStore(k.storeKey)
 	store.Delete(GetDelegatorWithdrawAddrKey(delAddr))
 }
@@ -77,7 +77,7 @@ func (k Keeper) SetPreviousProposerConsAddr(ctx sdk.Context, consAddr sdk.ConsAd
 	store.Set(ProposerKey, b)
 }
 
-// get the starting period associated with a delegator
+// get the starting info associated with a delegator
 func (k Keeper) GetDelegatorStartingInfo(ctx sdk.Context, val sdk.ValAddress, del sdk.AccAddress) (period types.DelegatorStartingInfo) {
 	store := ctx.KVStore(k.storeKey)
 	b := store.Get(GetDelegatorStartingInfoKey(val, del))
@@ -85,13 +85,25 @@ func (k Keeper) GetDelegatorStartingInfo(ctx sdk.Context, val sdk.ValAddress, de
 	return
 }
 
-// set the starting period associated with a delegator
+// set the starting info associated with a delegator
 func (k Keeper) SetDelegatorStartingInfo(ctx sdk.Context, val sdk.ValAddress, del sdk.AccAddress, period types.DelegatorStartingInfo) {
 	store := ctx.KVStore(k.storeKey)
 	b := k.cdc.MustMarshalBinaryLengthPrefixed(period)
 	store.Set(GetDelegatorStartingInfoKey(val, del), b)
 }
 
+// check existence of the starting info associated with a delegator
+func (k Keeper) HasDelegatorStartingInfo(ctx sdk.Context, val sdk.ValAddress, del sdk.AccAddress) bool {
+	store := ctx.KVStore(k.storeKey)
+	return store.Has(GetDelegatorStartingInfoKey(val, del))
+}
+
+// delete the starting info associated with a delegator
+func (k Keeper) DeleteDelegatorStartingInfo(ctx sdk.Context, val sdk.ValAddress, del sdk.AccAddress) {
+	store := ctx.KVStore(k.storeKey)
+	store.Delete(GetDelegatorStartingInfoKey(val, del))
+}
+
 // iterate over delegator starting infos
 func (k Keeper) IterateDelegatorStartingInfos(ctx sdk.Context, handler func(val sdk.ValAddress, del sdk.AccAddress, info types.DelegatorStartingInfo) (stop bool)) {
 	store := ctx.KVStore(k.storeKey)
@@ -137,8 +149,24 @@ func (k Keeper) IterateValidatorHistoricalRewards(ctx sdk.Context, handler func(
 	}
 }
 
-// delete historical rewards
+// delete a historical reward
-func (k Keeper) DeleteValidatorHistoricalRewards(ctx sdk.Context) {
+func (k Keeper) DeleteValidatorHistoricalReward(ctx sdk.Context, val sdk.ValAddress, period uint64) {
+	store := ctx.KVStore(k.storeKey)
+	store.Delete(GetValidatorHistoricalRewardsKey(val, period))
+}
+
+// delete historical rewards for a validator
+func (k Keeper) DeleteValidatorHistoricalRewards(ctx sdk.Context, val sdk.ValAddress) {
+	store := ctx.KVStore(k.storeKey)
+	iter := sdk.KVStorePrefixIterator(store, GetValidatorHistoricalRewardsPrefix(val))
+	defer iter.Close()
+	for ; iter.Valid(); iter.Next() {
+		store.Delete(iter.Key())
+	}
+}
+
+// delete all historical rewards
+func (k Keeper) DeleteAllValidatorHistoricalRewards(ctx sdk.Context) {
 	store := ctx.KVStore(k.storeKey)
 	iter := sdk.KVStorePrefixIterator(store, ValidatorHistoricalRewardsPrefix)
 	defer iter.Close()
@@ -147,6 +175,17 @@ func (k Keeper) DeleteValidatorHistoricalRewards(ctx sdk.Context) {
 	}
 }
 
+// historical record count (used for testcases)
+func (k Keeper) GetValidatorHistoricalRewardCount(ctx sdk.Context) (count uint64) {
+	store := ctx.KVStore(k.storeKey)
+	iter := sdk.KVStorePrefixIterator(store, ValidatorHistoricalRewardsPrefix)
+	defer iter.Close()
+	for ; iter.Valid(); iter.Next() {
+		count++
+	}
+	return
+}
+
 // get current rewards for a validator
 func (k Keeper) GetValidatorCurrentRewards(ctx sdk.Context, val sdk.ValAddress) (rewards types.ValidatorCurrentRewards) {
 	store := ctx.KVStore(k.storeKey)
@@ -162,6 +201,12 @@ func (k Keeper) SetValidatorCurrentRewards(ctx sdk.Context, val sdk.ValAddress,
 	store.Set(GetValidatorCurrentRewardsKey(val), b)
 }
 
+// delete current rewards for a validator
+func (k Keeper) DeleteValidatorCurrentRewards(ctx sdk.Context, val sdk.ValAddress) {
+	store := ctx.KVStore(k.storeKey)
+	store.Delete(GetValidatorCurrentRewardsKey(val))
+}
+
 // iterate over current rewards
 func (k Keeper) IterateValidatorCurrentRewards(ctx sdk.Context, handler func(val sdk.ValAddress, rewards types.ValidatorCurrentRewards) (stop bool)) {
 	store := ctx.KVStore(k.storeKey)
@@ -195,6 +240,12 @@ func (k Keeper) SetValidatorAccumulatedCommission(ctx sdk.Context, val sdk.ValAd
 	store.Set(GetValidatorAccumulatedCommissionKey(val), b)
 }
 
+// delete accumulated commission for a validator
+func (k Keeper) DeleteValidatorAccumulatedCommission(ctx sdk.Context, val sdk.ValAddress) {
+	store := ctx.KVStore(k.storeKey)
+	store.Delete(GetValidatorAccumulatedCommissionKey(val))
+}
+
 // iterate over accumulated commissions
 func (k Keeper) IterateValidatorAccumulatedCommissions(ctx sdk.Context, handler func(val sdk.ValAddress, commission types.ValidatorAccumulatedCommission) (stop bool)) {
 	store := ctx.KVStore(k.storeKey)
@@ -277,8 +328,18 @@ func (k Keeper) IterateValidatorSlashEvents(ctx sdk.Context, handler func(val sd
 	}
 }
 
+// delete slash events for a particular validator
+func (k Keeper) DeleteValidatorSlashEvents(ctx sdk.Context, val sdk.ValAddress) {
+	store := ctx.KVStore(k.storeKey)
+	iter := sdk.KVStorePrefixIterator(store, GetValidatorSlashEventPrefix(val))
+	defer iter.Close()
+	for ; iter.Valid(); iter.Next() {
+		store.Delete(iter.Key())
+	}
+}
+
 // delete all slash events
-func (k Keeper) DeleteValidatorSlashEvents(ctx sdk.Context) {
+func (k Keeper) DeleteAllValidatorSlashEvents(ctx sdk.Context) {
 	store := ctx.KVStore(k.storeKey)
 	iter := sdk.KVStorePrefixIterator(store, ValidatorSlashEventPrefix)
 	defer iter.Close()

x/distribution/keeper/validator.go

@@ -8,8 +8,8 @@ import (
 
 // initialize rewards for a new validator
 func (k Keeper) initializeValidator(ctx sdk.Context, val sdk.Validator) {
-	// set initial historical rewards (period 0)
+	// set initial historical rewards (period 0) with reference count of 1
-	k.SetValidatorHistoricalRewards(ctx, val.GetOperator(), 0, types.ValidatorHistoricalRewards{})
+	k.SetValidatorHistoricalRewards(ctx, val.GetOperator(), 0, types.NewValidatorHistoricalRewards(sdk.DecCoins{}, 1))
 
 	// set current rewards (starting at period 1)
 	k.SetValidatorCurrentRewards(ctx, val.GetOperator(), types.NewValidatorCurrentRewards(sdk.DecCoins{}, 1))
@@ -42,10 +42,10 @@ func (k Keeper) incrementValidatorPeriod(ctx sdk.Context, val sdk.Validator) uin
 	}
 
 	// fetch historical rewards for last period
-	historical := k.GetValidatorHistoricalRewards(ctx, val.GetOperator(), rewards.Period-1)
+	historical := k.GetValidatorHistoricalRewards(ctx, val.GetOperator(), rewards.Period-1).CumulativeRewardRatio
 
-	// fet new historical rewards
+	// set new historical rewards with reference count of 1
-	k.SetValidatorHistoricalRewards(ctx, val.GetOperator(), rewards.Period, historical.Plus(current))
+	k.SetValidatorHistoricalRewards(ctx, val.GetOperator(), rewards.Period, types.NewValidatorHistoricalRewards(historical.Plus(current), 1))
 
 	// set current rewards, incrementing period by 1
 	k.SetValidatorCurrentRewards(ctx, val.GetOperator(), types.NewValidatorCurrentRewards(sdk.DecCoins{}, rewards.Period+1))
@@ -53,19 +53,47 @@ func (k Keeper) incrementValidatorPeriod(ctx sdk.Context, val sdk.Validator) uin
 	return rewards.Period
 }
 
+// increment the reference count for a historical rewards value
+func (k Keeper) incrementReferenceCount(ctx sdk.Context, valAddr sdk.ValAddress, period uint64) {
+	historical := k.GetValidatorHistoricalRewards(ctx, valAddr, period)
+	if historical.ReferenceCount > 1 {
+		panic("reference count should never exceed 1")
+	}
+	historical.ReferenceCount++
+	k.SetValidatorHistoricalRewards(ctx, valAddr, period, historical)
+}
+
+// decrement the reference count for a historical rewards value, and delete if zero references remain
+func (k Keeper) decrementReferenceCount(ctx sdk.Context, valAddr sdk.ValAddress, period uint64) {
+	historical := k.GetValidatorHistoricalRewards(ctx, valAddr, period)
+	if historical.ReferenceCount == 0 {
+		panic("cannot set negative reference count")
+	}
+	historical.ReferenceCount--
+	if historical.ReferenceCount == 0 {
+		k.DeleteValidatorHistoricalReward(ctx, valAddr, period)
+	} else {
+		k.SetValidatorHistoricalRewards(ctx, valAddr, period, historical)
+	}
+}
+
 func (k Keeper) updateValidatorSlashFraction(ctx sdk.Context, valAddr sdk.ValAddress, fraction sdk.Dec) {
 	height := uint64(ctx.BlockHeight())
 	currentFraction := sdk.ZeroDec()
-	currentPeriod := k.GetValidatorCurrentRewards(ctx, valAddr).Period
+	endedPeriod := k.GetValidatorCurrentRewards(ctx, valAddr).Period - 1
 	current, found := k.GetValidatorSlashEvent(ctx, valAddr, height)
 	if found {
 		// there has already been a slash event this height,
 		// and we don't need to store more than one,
 		// so just update the current slash fraction
 		currentFraction = current.Fraction
+	} else {
+		val := k.stakingKeeper.Validator(ctx, valAddr)
+		// increment current period
+		endedPeriod = k.incrementValidatorPeriod(ctx, val)
 	}
 	currentMultiplicand := sdk.OneDec().Sub(currentFraction)
 	newMultiplicand := sdk.OneDec().Sub(fraction)
 	updatedFraction := sdk.OneDec().Sub(currentMultiplicand.Mul(newMultiplicand))
-	k.SetValidatorSlashEvent(ctx, valAddr, height, types.NewValidatorSlashEvent(currentPeriod, updatedFraction))
+	k.SetValidatorSlashEvent(ctx, valAddr, height, types.NewValidatorSlashEvent(endedPeriod, updatedFraction))
 }

x/distribution/simulation/invariants.go

@@ -5,6 +5,7 @@ import (
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	distr "github.com/cosmos/cosmos-sdk/x/distribution"
+	"github.com/cosmos/cosmos-sdk/x/distribution/types"
 	"github.com/cosmos/cosmos-sdk/x/mock/simulation"
 	"github.com/cosmos/cosmos-sdk/x/staking"
 )
@@ -20,6 +21,10 @@ func AllInvariants(d distr.Keeper, stk staking.Keeper) simulation.Invariant {
 		if err != nil {
 			return err
 		}
+		err = ReferenceCountInvariant(d, stk)(ctx)
+		if err != nil {
+			return err
+		}
 		return nil
 	}
 }
@@ -29,7 +34,7 @@ func NonNegativeOutstandingInvariant(k distr.Keeper) simulation.Invariant {
 	return func(ctx sdk.Context) error {
 		outstanding := k.GetOutstandingRewards(ctx)
 		if outstanding.HasNegative() {
-			return fmt.Errorf("Negative outstanding coins: %v", outstanding)
+			return fmt.Errorf("negative outstanding coins: %v", outstanding)
 		}
 		return nil
 	}
@@ -57,7 +62,35 @@ func CanWithdrawInvariant(k distr.Keeper, sk staking.Keeper) simulation.Invarian
 		remaining := k.GetOutstandingRewards(ctx)
 
 		if len(remaining) > 0 && remaining[0].Amount.LT(sdk.ZeroDec()) {
-			return fmt.Errorf("Negative remaining coins: %v", remaining)
+			return fmt.Errorf("negative remaining coins: %v", remaining)
+		}
+
+		return nil
+	}
+}
+
+// ReferenceCountInvariant checks that the number of historical rewards records is correct
+func ReferenceCountInvariant(k distr.Keeper, sk staking.Keeper) simulation.Invariant {
+	return func(ctx sdk.Context) error {
+
+		valCount := uint64(0)
+		sk.IterateValidators(ctx, func(_ int64, val sdk.Validator) (stop bool) {
+			valCount++
+			return false
+		})
+		dels := sk.GetAllDelegations(ctx)
+		slashCount := uint64(0)
+		k.IterateValidatorSlashEvents(ctx, func(_ sdk.ValAddress, _ uint64, _ types.ValidatorSlashEvent) (stop bool) {
+			slashCount++
+			return false
+		})
+
+		// one record per validator (zeroeth period), one record per delegation (previous period), one record per slash (previous period)
+		expected := valCount + uint64(len(dels)) + slashCount
+
+		count := k.GetValidatorHistoricalRewardCount(ctx)
+		if count != expected {
+			return fmt.Errorf("unexpected number of historical rewards records: expected %v, got %v", expected, count)
 		}
 
 		return nil

x/distribution/types/validator.go

@@ -8,9 +8,28 @@ import (
 )
 
 // historical rewards for a validator
-// TODO add reference counter, ref https://github.com/cosmos/cosmos-sdk/pull/3099#discussion_r245747051
 // height is implicit within the store key
-type ValidatorHistoricalRewards = sdk.DecCoins
+// cumulative reward ratio is the sum from the zeroeth period
+// until this period of rewards / tokens, per the spec
+// The reference count indicates the number of objects
+// which might need to reference this historical entry
+// at any point.
+// ReferenceCount =
+//    number of outstanding delegations which ended the associated period (and might need to read that record)
+//  + number of slashes which ended the associated period (and might need to read that record)
+//  + one per validator for the zeroeth period, set on initialization
+type ValidatorHistoricalRewards struct {
+	CumulativeRewardRatio sdk.DecCoins `json:"cumulative_reward_ratio"`
+	ReferenceCount        uint16       `json:"reference_count"`
+}
+
+// create a new ValidatorHistoricalRewards
+func NewValidatorHistoricalRewards(cumulativeRewardRatio sdk.DecCoins, referenceCount uint16) ValidatorHistoricalRewards {
+	return ValidatorHistoricalRewards{
+		CumulativeRewardRatio: cumulativeRewardRatio,
+		ReferenceCount:        referenceCount,
+	}
+}
 
 // current rewards and current period for a validator
 // kept as a running counter and incremented each block

x/staking/types/validator_test.go

@@ -184,8 +184,8 @@ func TestRemoveDelShares(t *testing.T) {
 		DelegatorShares: delShares,
 	}
 	pool := Pool{
-		BondedTokens: sdk.NewInt(248305),
+		BondedTokens:    sdk.NewInt(248305),
-		NotBondedTokens:  sdk.NewInt(232147),
+		NotBondedTokens: sdk.NewInt(232147),
 	}
 	shares := sdk.NewDec(29)
 	_, newPool, tokens := validator.RemoveDelShares(pool, shares)
@@ -232,8 +232,8 @@ func TestPossibleOverflow(t *testing.T) {
 		DelegatorShares: delShares,
 	}
 	pool := Pool{
-		NotBondedTokens:  sdk.NewInt(100),
+		NotBondedTokens: sdk.NewInt(100),
-		BondedTokens: poolTokens,
+		BondedTokens:    poolTokens,
 	}
 	tokens := int64(71)
 	msg := fmt.Sprintf("validator %#v", validator)