diff --git a/blockchain/store.go b/blockchain/store.go index c77f67ed..cff26c42 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -35,6 +35,9 @@ type BlockStore struct { height int64 } +// NewBlockStore loads a blockStore's JSON serialized form from the +// database, db to retrieve the starting height of the blockstore +// and backs db as the internal database of the blockstore. func NewBlockStore(db dbm.DB) *BlockStore { bsjson := LoadBlockStoreStateJSON(db) return &BlockStore{ @@ -50,6 +53,10 @@ func (bs *BlockStore) Height() int64 { return bs.height } +// GetReader conveniently wraps the result of the database +// lookup for key key into an io.Reader. If no result is found, +// it returns nil otherwise it creates an io.Reader. +// Its utility is mainly for use with wire.ReadBinary. func (bs *BlockStore) GetReader(key []byte) io.Reader { bytez := bs.db.Get(key) if bytez == nil { @@ -58,6 +65,13 @@ func (bs *BlockStore) GetReader(key []byte) io.Reader { return bytes.NewReader(bytez) } +// LoadBlock retrieves the serialized block, keyed by height in the +// store's database. If the data at the requested height is not found, +// it returns nil. However, if the block meta data is found but +// cannot be deserialized by wire.ReadBinary, it panics. +// The serialized data consists of the BlockMeta data and different +// parts that are reassembled by their internal Data. If the final +// reassembled data cannot be deserialized by wire.ReadBinary, it panics. func (bs *BlockStore) LoadBlock(height int64) *types.Block { var n int var err error @@ -81,6 +95,11 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { return block } +// LoadBlockPart tries to load a blockPart from the +// backing database, keyed by height and index. +// If it doesn't find the requested blockPart, it +// returns nil. Otherwise, If the found part is +// corrupted/not deserializable by wire.ReadBinary, it panics. func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { var n int var err error @@ -95,6 +114,10 @@ func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { return part } +// LoadBlockMeta tries to load a block meta from the backing database, +// keyed by height. The block meta must have been wire.Binary serialized. +// If it doesn't find the requested meta, it returns nil. Otherwise, +// if the found data cannot be deserialized by wire.ReadBinary, it panics. func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { var n int var err error @@ -109,6 +132,11 @@ func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { return blockMeta } +// LoadBlockCommit tries to load a commit from the backing database, +// keyed by height. The commit must have been wire.Binary serialized. +// If it doesn't find the requested commit in the database, it returns nil. +// Otherwise, if the found data cannot be deserialized by wire.ReadBinary, it panics. +// // The +2/3 and other Precommit-votes for block at `height`. // This Commit comes from block.LastCommit for `height+1`. func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { @@ -125,6 +153,11 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { return commit } +// LoadSeenCommit tries to load the seen commit from the backing database, +// keyed by height. The commit must have been wire.Binary serialized. +// If it doesn't find the requested commit in the database, it returns nil. +// Otherwise, if the found data cannot be deserialized by wire.ReadBinary, it panics. +// // NOTE: the Precommit-vote heights are for the block at `height` func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { var n int @@ -146,6 +179,9 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { // we need this to reload the precommits to catch-up nodes to the // most recent height. Otherwise they'd stall at H-1. func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { + if block == nil { + PanicSanity("BlockStore can only save a non-nil block") + } height := block.Height if height != bs.Height()+1 { cmn.PanicSanity(cmn.Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) @@ -219,6 +255,7 @@ type BlockStoreStateJSON struct { Height int64 } +// Save JSON marshals the blockStore state to the database, saving it synchronously. func (bsj BlockStoreStateJSON) Save(db dbm.DB) { bytes, err := json.Marshal(bsj) if err != nil { @@ -227,6 +264,10 @@ func (bsj BlockStoreStateJSON) Save(db dbm.DB) { db.SetSync(blockStoreKey, bytes) } +// LoadBlockStoreStateJSON JSON unmarshals the +// blockStore state from the database, keyed by +// key "blockStore". If it cannot lookup the state, +// it returns the zero value BlockStoreStateJSON. func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON { bytes := db.Get(blockStoreKey) if bytes == nil { diff --git a/blockchain/store_test.go b/blockchain/store_test.go new file mode 100644 index 00000000..65832e22 --- /dev/null +++ b/blockchain/store_test.go @@ -0,0 +1,459 @@ +package blockchain + +import ( + "bytes" + "fmt" + "io/ioutil" + "runtime/debug" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/go-wire" + "github.com/tendermint/go-wire/data" + "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmlibs/db" +) + +func TestLoadBlockStoreStateJSON(t *testing.T) { + db := db.NewMemDB() + + bsj := &BlockStoreStateJSON{Height: 1000} + bsj.Save(db) + + retrBSJ := LoadBlockStoreStateJSON(db) + + assert.Equal(t, *bsj, retrBSJ, "expected the retrieved DBs to match") +} + +func TestNewBlockStore(t *testing.T) { + db := db.NewMemDB() + db.Set(blockStoreKey, []byte(`{"height": 10000}`)) + bs := NewBlockStore(db) + assert.Equal(t, bs.Height(), 10000, "failed to properly parse blockstore") + + panicCausers := []struct { + data []byte + wantErr string + }{ + {[]byte("artful-doger"), "not unmarshal bytes"}, + {[]byte(""), "unmarshal bytes"}, + {[]byte(" "), "unmarshal bytes"}, + } + + for i, tt := range panicCausers { + // Expecting a panic here on trying to parse an invalid blockStore + _, _, panicErr := doFn(func() (interface{}, error) { + db.Set(blockStoreKey, tt.data) + _ = NewBlockStore(db) + return nil, nil + }) + require.NotNil(t, panicErr, "#%d panicCauser: %q expected a panic", i, tt.data) + assert.Contains(t, panicErr.Error(), tt.wantErr, "#%d data: %q", i, tt.data) + } + + db.Set(blockStoreKey, nil) + bs = NewBlockStore(db) + assert.Equal(t, bs.Height(), 0, "expecting nil bytes to be unmarshaled alright") +} + +func TestBlockStoreGetReader(t *testing.T) { + db := db.NewMemDB() + // Initial setup + db.Set([]byte("Foo"), []byte("Bar")) + db.Set([]byte("Foo1"), nil) + + bs := NewBlockStore(db) + + tests := [...]struct { + key []byte + want []byte + }{ + 0: {key: []byte("Foo"), want: []byte("Bar")}, + 1: {key: []byte("KnoxNonExistent"), want: nil}, + 2: {key: []byte("Foo1"), want: nil}, + } + + for i, tt := range tests { + r := bs.GetReader(tt.key) + if r == nil { + assert.Nil(t, tt.want, "#%d: expected a non-nil reader", i) + continue + } + slurp, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("#%d: unexpected Read err: %v", i, err) + } else { + assert.Equal(t, slurp, tt.want, "#%d: mismatch", i) + } + } +} + +func freshBlockStore() (*BlockStore, db.DB) { + db := db.NewMemDB() + return NewBlockStore(db), db +} + +var ( + // Setup, test data + // If needed, the parts' data can be generated by running + // the code at https://gist.github.com/odeke-em/9ffac2b5df44595fad7084ece4c9bd98 + part1 = &types.Part{Index: 0, Bytes: data.Bytes([]byte{ + 0x01, 0x01, 0x01, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x65, + 0x73, 0x74, 0x01, 0x01, 0xa1, 0xb2, 0x03, 0xeb, 0x3d, 0x1f, 0x44, 0x40, 0x01, 0x64, 0x00, + })} + part2 = &types.Part{Index: 1, Bytes: data.Bytes([]byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, + 0x00, 0x01, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + })} + seenCommit1 = &types.Commit{Precommits: []*types.Vote{{Height: 10}}} +) + +func TestBlockStoreSaveLoadBlock(t *testing.T) { + bs, _ := freshBlockStore() + noBlockHeights := []int{0, -1, 100, 1000, 2} + for i, height := range noBlockHeights { + if g := bs.LoadBlock(height); g != nil { + t.Errorf("#%d: height(%d) got a block; want nil", i, height) + } + } + validPartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 2}) + validPartSet.AddPart(part1, false) + validPartSet.AddPart(part2, false) + + incompletePartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 2}) + + uncontiguousPartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 0}) + uncontiguousPartSet.AddPart(part2, false) + + // End of setup, test data + + tuples := []struct { + block *types.Block + parts *types.PartSet + seenCommit *types.Commit + wantErr bool + wantPanic string + + corruptBlockInDB bool + corruptCommitInDB bool + corruptSeenCommitInDB bool + eraseCommitInDB bool + eraseSeenCommitInDB bool + }{ + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: validPartSet, + seenCommit: seenCommit1, + }, + + { + block: nil, + wantPanic: "only save a non-nil block", + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 4, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: uncontiguousPartSet, + wantPanic: "only save contiguous blocks", // and incomplete and uncontiguous parts + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: incompletePartSet, + wantPanic: "only save complete block", // incomplete parts + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: validPartSet, + seenCommit: seenCommit1, + corruptCommitInDB: true, // Corrupt the DB's commit entry + wantPanic: "rror reading commit", + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: validPartSet, + seenCommit: seenCommit1, + wantPanic: "rror reading block", + corruptBlockInDB: true, // Corrupt the DB's block entry + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: validPartSet, + seenCommit: seenCommit1, + + // Expecting no error and we want a nil back + eraseSeenCommitInDB: true, + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: validPartSet, + seenCommit: seenCommit1, + + corruptSeenCommitInDB: true, + wantPanic: "rror reading commit", + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + }, + parts: validPartSet, + seenCommit: seenCommit1, + + // Expecting no error and we want a nil back + eraseCommitInDB: true, + }, + } + + type quad struct { + block *types.Block + commit *types.Commit + meta *types.BlockMeta + + seenCommit *types.Commit + } + + for i, tuple := range tuples { + bs, db := freshBlockStore() + // SaveBlock + res, err, panicErr := doFn(func() (interface{}, error) { + bs.SaveBlock(tuple.block, tuple.parts, tuple.seenCommit) + if tuple.block == nil { + return nil, nil + } + + if tuple.corruptBlockInDB { + db.Set(calcBlockMetaKey(tuple.block.Height), []byte("block-bogus")) + } + bBlock := bs.LoadBlock(tuple.block.Height) + bBlockMeta := bs.LoadBlockMeta(tuple.block.Height) + + if tuple.eraseSeenCommitInDB { + db.Delete(calcSeenCommitKey(tuple.block.Height)) + } + if tuple.corruptSeenCommitInDB { + db.Set(calcSeenCommitKey(tuple.block.Height), []byte("bogus-seen-commit")) + } + bSeenCommit := bs.LoadSeenCommit(tuple.block.Height) + + commitHeight := tuple.block.Height - 1 + if tuple.eraseCommitInDB { + db.Delete(calcBlockCommitKey(commitHeight)) + } + if tuple.corruptCommitInDB { + db.Set(calcBlockCommitKey(commitHeight), []byte("foo-bogus")) + } + bCommit := bs.LoadBlockCommit(commitHeight) + return &quad{block: bBlock, seenCommit: bSeenCommit, commit: bCommit, meta: bBlockMeta}, nil + }) + + if subStr := tuple.wantPanic; subStr != "" { + if panicErr == nil { + t.Errorf("#%d: want a non-nil panic", i) + } else if got := panicErr.Error(); !strings.Contains(got, subStr) { + t.Errorf("#%d:\n\tgotErr: %q\nwant substring: %q", i, got, subStr) + } + continue + } + + if tuple.wantErr { + if err == nil { + t.Errorf("#%d: got nil error", i) + } + continue + } + + assert.Nil(t, panicErr, "#%d: unexpected panic", i) + assert.Nil(t, err, "#%d: expecting a non-nil error", i) + qua, ok := res.(*quad) + if !ok || qua == nil { + t.Errorf("#%d: got nil quad back; gotType=%T", i, res) + continue + } + if tuple.eraseSeenCommitInDB { + assert.Nil(t, qua.seenCommit, "erased the seenCommit in the DB hence we should get back a nil seenCommit") + } + if tuple.eraseCommitInDB { + assert.Nil(t, qua.commit, "erased the commit in the DB hence we should get back a nil commit") + } + } +} + +func binarySerializeIt(v interface{}) []byte { + var n int + var err error + buf := new(bytes.Buffer) + wire.WriteBinary(v, buf, &n, &err) + return buf.Bytes() +} + +func TestLoadBlockPart(t *testing.T) { + bs, db := freshBlockStore() + height, index := 10, 1 + loadPart := func() (interface{}, error) { + part := bs.LoadBlockPart(height, index) + return part, nil + } + + // Initially no contents. + // 1. Requesting for a non-existent block shouldn't fail + res, _, panicErr := doFn(loadPart) + require.Nil(t, panicErr, "a non-existent block part shouldn't cause a panic") + require.Nil(t, res, "a non-existent block part should return nil") + + // 2. Next save a corrupted block then try to load it + db.Set(calcBlockPartKey(height, index), []byte("Tendermint")) + res, _, panicErr = doFn(loadPart) + require.NotNil(t, panicErr, "expecting a non-nil panic") + require.Contains(t, panicErr.Error(), "Error reading block part") + + // 3. A good block serialized and saved to the DB should be retrievable + db.Set(calcBlockPartKey(height, index), binarySerializeIt(part1)) + gotPart, _, panicErr := doFn(loadPart) + require.Nil(t, panicErr, "an existent and proper block should not panic") + require.Nil(t, res, "a properly saved block should return a proper block") + require.Equal(t, gotPart.(*types.Part).Hash(), part1.Hash(), "expecting successful retrieval of previously saved block") +} + +func TestLoadBlockMeta(t *testing.T) { + bs, db := freshBlockStore() + height := 10 + loadMeta := func() (interface{}, error) { + meta := bs.LoadBlockMeta(height) + return meta, nil + } + + // Initially no contents. + // 1. Requesting for a non-existent blockMeta shouldn't fail + res, _, panicErr := doFn(loadMeta) + require.Nil(t, panicErr, "a non-existent blockMeta shouldn't cause a panic") + require.Nil(t, res, "a non-existent blockMeta should return nil") + + // 2. Next save a corrupted blockMeta then try to load it + db.Set(calcBlockMetaKey(height), []byte("Tendermint-Meta")) + res, _, panicErr = doFn(loadMeta) + require.NotNil(t, panicErr, "expecting a non-nil panic") + require.Contains(t, panicErr.Error(), "Error reading block meta") + + // 3. A good blockMeta serialized and saved to the DB should be retrievable + meta := &types.BlockMeta{} + db.Set(calcBlockMetaKey(height), binarySerializeIt(meta)) + gotMeta, _, panicErr := doFn(loadMeta) + require.Nil(t, panicErr, "an existent and proper block should not panic") + require.Nil(t, res, "a properly saved blockMeta should return a proper blocMeta ") + require.Equal(t, binarySerializeIt(meta), binarySerializeIt(gotMeta), "expecting successful retrieval of previously saved blockMeta") +} + +func TestBlockFetchAtHeight(t *testing.T) { + bs, _ := freshBlockStore() + block := &types.Block{ + Header: &types.Header{ + Height: 1, + NumTxs: 100, + ChainID: "block_test", + }, + LastCommit: &types.Commit{Precommits: []*types.Vote{{Height: 10}}}, + } + seenCommit := seenCommit1 + validPartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 2}) + validPartSet.AddPart(part1, false) + validPartSet.AddPart(part2, false) + parts := validPartSet + + require.Equal(t, bs.Height(), 0, "initially the height should be zero") + require.NotEqual(t, bs.Height(), block.Header.Height, "expecting different heights initially") + + bs.SaveBlock(block, parts, seenCommit) + require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed") + + blockAtHeight := bs.LoadBlock(bs.Height()) + require.Equal(t, block.Hash(), blockAtHeight.Hash(), "expecting a successful load of the last saved block") + + blockAtHeightPlus1 := bs.LoadBlock(bs.Height() + 1) + require.Nil(t, blockAtHeightPlus1, "expecting an unsuccessful load of Height()+1") + blockAtHeightPlus2 := bs.LoadBlock(bs.Height() + 2) + require.Nil(t, blockAtHeightPlus2, "expecting an unsuccessful load of Height()+2") +} + +func doFn(fn func() (interface{}, error)) (res interface{}, err error, panicErr error) { + defer func() { + if r := recover(); r != nil { + switch e := r.(type) { + case error: + panicErr = e + case string: + panicErr = fmt.Errorf("%s", e) + default: + if st, ok := r.(fmt.Stringer); ok { + panicErr = fmt.Errorf("%s", st) + } else { + panicErr = fmt.Errorf("%s", debug.Stack()) + } + } + } + }() + + res, err = fn() + return res, err, panicErr +}