Merge pull request #254 from cosmos/feature/historical-queries

Historical queries support

app/app.go

@@ -59,6 +59,9 @@ func (app *Basecoin) GetState() sm.SimpleDB {
 // Info - ABCI
 func (app *Basecoin) Info(req abci.RequestInfo) abci.ResponseInfo {
 	resp := app.state.Info()
+	app.logger.Debug("Info",
+		"height", resp.LastBlockHeight,
+		"hash", fmt.Sprintf("%X", resp.LastBlockAppHash))
 	app.height = resp.LastBlockHeight
 	return abci.ResponseInfo{
 		Data:             fmt.Sprintf("Basecoin v%v", version.Version),
@@ -70,7 +73,6 @@ func (app *Basecoin) Info(req abci.RequestInfo) abci.ResponseInfo {
 // InitState - used to setup state (was SetOption)
 // to be used by InitChain later
 func (app *Basecoin) InitState(key string, value string) string {
-
 	module, key := splitKey(key)
 	state := app.state.Append()
 

app/app_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	abci "github.com/tendermint/abci/types"
 	sdk "github.com/cosmos/cosmos-sdk"
 	"github.com/cosmos/cosmos-sdk/modules/auth"
 	"github.com/cosmos/cosmos-sdk/modules/base"
@@ -18,6 +17,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/modules/roles"
 	"github.com/cosmos/cosmos-sdk/stack"
 	"github.com/cosmos/cosmos-sdk/state"
+	abci "github.com/tendermint/abci/types"
 	wire "github.com/tendermint/go-wire"
 	"github.com/tendermint/tmlibs/log"
 )
@@ -294,9 +294,10 @@ func TestQuery(t *testing.T) {
 	res = at.app.Commit()
 	assert.True(res.IsOK(), res)
 
+	key := stack.PrefixedKey(coin.NameCoin, at.acctIn.Address())
 	resQueryPostCommit := at.app.Query(abci.RequestQuery{
-		Path: "/account",
-		Data: at.acctIn.Address(),
+		Path: "/key",
+		Data: key,
 	})
 	assert.NotEqual(resQueryPreCommit, resQueryPostCommit, "Query should change before/after commit")
 }

app/store.go

@@ -1,7 +1,6 @@
 package app
 
 import (
-	"bytes"
 	"fmt"
 	"path"
 	"path/filepath"
@@ -9,7 +8,6 @@ import (
 
 	"github.com/pkg/errors"
 	abci "github.com/tendermint/abci/types"
-	"github.com/tendermint/go-wire"
 	"github.com/tendermint/iavl"
 	cmn "github.com/tendermint/tmlibs/common"
 	dbm "github.com/tendermint/tmlibs/db"
@@ -21,21 +19,10 @@ import (
 // Store contains the merkle tree, and all info to handle abci requests
 type Store struct {
 	state.State
-	height    uint64
-	hash      []byte
-	persisted bool
-
+	height uint64
 	logger log.Logger
 }
 
-var stateKey = []byte("merkle:state") // Database key for merkle tree save value db values
-
-// ChainState contains the latest Merkle root hash and the number of times `Commit` has been called
-type ChainState struct {
-	Hash   []byte
-	Height uint64
-}
-
 // MockStore returns an in-memory store only intended for testing
 func MockStore() *Store {
 	res, err := NewStore("", 0, log.NewNopLogger())
@@ -46,22 +33,17 @@ func MockStore() *Store {
 	return res
 }
 
-// NewStore initializes an in-memory IAVLTree, or attempts to load a persistant
-// tree from disk
+// NewStore initializes an in-memory iavl.VersionedTree, or attempts to load a
+// persistant tree from disk
 func NewStore(dbName string, cacheSize int, logger log.Logger) (*Store, error) {
-	// start at 1 so the height returned by query is for the
-	// next block, ie. the one that includes the AppHash for our current state
-	initialHeight := uint64(1)
-
-	// Non-persistent case
+	// memory backed case, just for testing
 	if dbName == "" {
-		tree := iavl.NewIAVLTree(
+		tree := iavl.NewVersionedTree(
 			0,
-			nil,
+			dbm.NewMemDB(),
 		)
 		store := &Store{
-			State:  state.NewState(tree, false),
-			height: initialHeight,
+			State:  state.NewState(tree),
 			logger: logger,
 		}
 		return store, nil
@@ -85,102 +67,88 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) (*Store, error) {
 
 	// Open database called "dir/name.db", if it doesn't exist it will be created
 	db := dbm.NewDB(name, dbm.LevelDBBackendStr, dir)
-	tree := iavl.NewIAVLTree(cacheSize, db)
+	tree := iavl.NewVersionedTree(cacheSize, db)
 
-	var chainState ChainState
 	if empty {
 		logger.Info("no existing db, creating new db")
-		chainState = ChainState{
-			Hash:   tree.Save(),
-			Height: initialHeight,
-		}
-		db.Set(stateKey, wire.BinaryBytes(chainState))
 	} else {
 		logger.Info("loading existing db")
-		eyesStateBytes := db.Get(stateKey)
-		err = wire.ReadBinaryBytes(eyesStateBytes, &chainState)
-		if err != nil {
-			return nil, errors.Wrap(err, "Reading MerkleEyesState")
+		if err = tree.Load(); err != nil {
+			return nil, errors.Wrap(err, "Loading tree")
 		}
-		tree.Load(chainState.Hash)
 	}
 
 	res := &Store{
-		State:     state.NewState(tree, true),
-		height:    chainState.Height,
-		hash:      chainState.Hash,
-		persisted: true,
-		logger:    logger,
+		State:  state.NewState(tree),
+		logger: logger,
 	}
+	res.height = res.State.LatestHeight()
 	return res, nil
 }
 
+// Hash gets the last hash stored in the database
+func (s *Store) Hash() []byte {
+	return s.State.LatestHash()
+}
+
 // Info implements abci.Application. It returns the height, hash and size (in the data).
 // The height is the block that holds the transactions, not the apphash itself.
 func (s *Store) Info() abci.ResponseInfo {
 	s.logger.Info("Info synced",
 		"height", s.height,
-		"hash", fmt.Sprintf("%X", s.hash))
+		"hash", fmt.Sprintf("%X", s.Hash()))
 	return abci.ResponseInfo{
 		Data:             cmn.Fmt("size:%v", s.State.Size()),
-		LastBlockHeight:  s.height - 1,
-		LastBlockAppHash: s.hash,
+		LastBlockHeight:  s.height,
+		LastBlockAppHash: s.Hash(),
 	}
 }
 
 // Commit implements abci.Application
 func (s *Store) Commit() abci.Result {
-	var err error
 	s.height++
-	s.hash, err = s.State.Hash()
+
+	hash, err := s.State.Commit(s.height)
 	if err != nil {
 		return abci.NewError(abci.CodeType_InternalError, err.Error())
 	}
-
 	s.logger.Debug("Commit synced",
 		"height", s.height,
-		"hash", fmt.Sprintf("%X", s.hash))
-
-	s.State.BatchSet(stateKey, wire.BinaryBytes(ChainState{
-		Hash:   s.hash,
-		Height: s.height,
-	}))
-
-	hash, err := s.State.Commit()
-	if err != nil {
-		return abci.NewError(abci.CodeType_InternalError, err.Error())
-	}
-	if !bytes.Equal(hash, s.hash) {
-		return abci.NewError(abci.CodeType_InternalError, "AppHash is incorrect")
-	}
+		"hash", fmt.Sprintf("%X", hash),
+	)
 
 	if s.State.Size() == 0 {
 		return abci.NewResultOK(nil, "Empty hash for empty tree")
 	}
-	return abci.NewResultOK(s.hash, "")
+	return abci.NewResultOK(hash, "")
 }
 
 // Query implements abci.Application
 func (s *Store) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
-
-	if reqQuery.Height != 0 {
-		// TODO: support older commits
-		resQuery.Code = abci.CodeType_InternalError
-		resQuery.Log = "merkleeyes only supports queries on latest commit"
-		return
-	}
-
 	// set the query response height to current
-	resQuery.Height = s.height
-
 	tree := s.State.Committed()
 
+	height := reqQuery.Height
+	if height == 0 {
+		// TODO: once the rpc actually passes in non-zero
+		// heights we can use to query right after a tx
+		// we must retrun most recent, even if apphash
+		// is not yet in the blockchain
+
+		// if tree.Tree.VersionExists(s.height - 1) {
+		// 	height = s.height - 1
+		// } else {
+		height = s.height
+		// }
+	}
+	resQuery.Height = height
+
 	switch reqQuery.Path {
 	case "/store", "/key": // Get by key
 		key := reqQuery.Data // Data holds the key bytes
 		resQuery.Key = key
 		if reqQuery.Prove {
-			value, proof, err := tree.GetWithProof(key)
+			value, proof, err := tree.GetVersionedWithProof(key, height)
 			if err != nil {
 				resQuery.Log = err.Error()
 				break

client/query.go

@@ -39,7 +39,9 @@ func GetWithProof(key []byte, node client.Client, cert certifiers.Certifier) (
 		return
 	}
 
-	check, err := GetCertifiedCheckpoint(int(resp.Height), node, cert)
+	// AppHash for height H is in header H+1
+	var check lc.Checkpoint
+	check, err = GetCertifiedCheckpoint(int(resp.Height+1), node, cert)
 	if err != nil {
 		return
 	}
@@ -69,7 +71,6 @@ func GetWithProof(key []byte, node client.Client, cert certifiers.Certifier) (
 			err = errors.Wrap(err, "Error reading proof")
 			return
 		}
-
 		// Validate the proof against the certified header to ensure data integrity.
 		err = aproof.Verify(resp.Key, nil, check.Header.AppHash)
 		if err != nil {

client/query_test.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"fmt"
 	"os"
 	"testing"
 
@@ -103,13 +104,14 @@ func TestTxProofs(t *testing.T) {
 	cl := client.NewLocal(node)
 	client.WaitForHeight(cl, 1, nil)
 
-	tx := eyes.SetTx{Key: []byte("key-a"), Value: []byte("value-a")}.Wrap()
+	tx := eyes.NewSetTx([]byte("key-a"), []byte("value-a"))
 
 	btx := types.Tx(wire.BinaryBytes(tx))
 	br, err := cl.BroadcastTxCommit(btx)
 	require.NoError(err, "%+v", err)
 	require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx)
 	require.EqualValues(0, br.DeliverTx.Code)
+	fmt.Printf("tx height: %d\n", br.Height)
 
 	source := certclient.New(cl)
 	seed, err := source.GetByHeight(br.Height - 2)
@@ -118,18 +120,20 @@ func TestTxProofs(t *testing.T) {
 
 	// First let's make sure a bogus transaction hash returns a valid non-existence proof.
 	key := types.Tx([]byte("bogus")).Hash()
-	bs, _, proof, err := GetWithProof(key, cl, cert)
-	assert.Nil(bs, "value should be nil")
-	require.True(lc.IsNoDataErr(err), "error should signal 'no data'")
-	require.NotNil(proof, "proof shouldn't be nil")
-	err = proof.Verify(key, nil, proof.Root())
-	require.NoError(err, "%+v", err)
+	res, err := cl.Tx(key, true)
+	require.NotNil(err)
+	require.Contains(err.Error(), "not found")
 
 	// Now let's check with the real tx hash.
 	key = btx.Hash()
-	res, err := cl.Tx(key, true)
+	res, err = cl.Tx(key, true)
 	require.NoError(err, "%+v", err)
 	require.NotNil(res)
 	err = res.Proof.Validate(key)
 	assert.NoError(err, "%+v", err)
+
+	check, err := GetCertifiedCheckpoint(int(br.Height), cl, cert)
+	require.Nil(err, "%+v", err)
+	require.Equal(res.Proof.RootHash, check.Header.DataHash)
+
 }

glide.lock

@@ -1,5 +1,5 @@
-hash: 647d25291b8e9a85cb4a49abc972a41537e8a286514bf41180845785e3b180a4
-updated: 2017-10-02T23:59:53.784455453-04:00
+hash: fbfdd03c0367bb0785ceb81ed34059df219e55d5a9c71c12597e505fbce14165
+updated: 2017-10-10T17:14:33.612302321+02:00
 imports:
 - name: github.com/bgentry/speakeasy
   version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd
@@ -108,7 +108,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: 191c4b6d176169ffc7f9972d490fa362a3b7d940
+  version: 15cd7fb1e3b75c436b6dee89a44db35f3d265bd0
   subpackages:
   - client
   - example/dummy
@@ -128,12 +128,12 @@ imports:
   - keys/storage/memstorage
   - keys/wordlist
 - name: github.com/tendermint/go-wire
-  version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb
+  version: ddbcd39cf68f7026d12f81c66a3cb45fc38ac48b
   subpackages:
   - data
   - data/base58
 - name: github.com/tendermint/iavl
-  version: 2d3ca4f466c32953641d4c49cad3d93eb7876a5e
+  version: 372f484952449aae18cce33b82c13329a9009acf
 - name: github.com/tendermint/light-client
   version: ac2e4bf47b31aaf5d3d336691ac786ec751bfc32
   subpackages:
@@ -146,7 +146,7 @@ imports:
   subpackages:
   - iavl
 - name: github.com/tendermint/tendermint
-  version: 97e980225530133c7b7e2b45e5e65c1f78ace89b
+  version: 49653d3e31e34a5da83e16e9ffdcd95a68acd9be
   subpackages:
   - blockchain
   - cmd/tendermint/commands
@@ -173,7 +173,7 @@ imports:
   - types
   - version
 - name: github.com/tendermint/tmlibs
-  version: 096dcb90e60aa00b748b3fe49a4b95e48ebf1e13
+  version: 7dd6b3d3f8a7a998a79bdd0d8222252b309570f3
   subpackages:
   - autofile
   - cli

glide.yaml

@@ -8,7 +8,6 @@ import:
 - package: github.com/spf13/viper
 - package: github.com/tendermint/abci
   version: develop
-  version:
   subpackages:
   - server
   - types

stack/checkpoint_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/tendermint/iavl"
+	db "github.com/tendermint/tmlibs/db"
 	"github.com/tendermint/tmlibs/log"
 
 	sdk "github.com/cosmos/cosmos-sdk"
@@ -17,7 +18,7 @@ import (
 func makeState() state.SimpleDB {
 	// return state.NewMemKVStore()
 
-	return state.NewBonsai(iavl.NewIAVLTree(0, nil))
+	return state.NewBonsai(iavl.NewVersionedTree(0, db.NewMemDB()))
 
 	// tree with persistence....
 	// tmpDir, err := ioutil.TempDir("", "state-tests")

state/bonsai.go

@@ -12,7 +12,7 @@ type nonce int64
 // Bonsai is a deformed tree forced to fit in a small pot
 type Bonsai struct {
 	id   nonce
-	Tree *iavl.IAVLTree
+	Tree *iavl.VersionedTree
 }
 
 func (b *Bonsai) String() string {
@@ -22,7 +22,7 @@ func (b *Bonsai) String() string {
 var _ SimpleDB = &Bonsai{}
 
 // NewBonsai wraps a merkle tree and tags it to track children
-func NewBonsai(tree *iavl.IAVLTree) *Bonsai {
+func NewBonsai(tree *iavl.VersionedTree) *Bonsai {
 	return &Bonsai{
 		id:   nonce(rand.Int63()),
 		Tree: tree,
@@ -54,6 +54,10 @@ func (b *Bonsai) GetWithProof(key []byte) ([]byte, iavl.KeyProof, error) {
 	return b.Tree.GetWithProof(key)
 }
 
+func (b *Bonsai) GetVersionedWithProof(key []byte, version uint64) ([]byte, iavl.KeyProof, error) {
+	return b.Tree.GetVersionedWithProof(key, version)
+}
+
 func (b *Bonsai) List(start, end []byte, limit int) []Model {
 	res := []Model{}
 	stopAtCount := func(key []byte, value []byte) (stop bool) {

state/merkle.go

@@ -11,13 +11,13 @@ type State struct {
 	persistent bool
 }
 
-func NewState(tree *iavl.IAVLTree, persistent bool) State {
+func NewState(tree *iavl.VersionedTree) State {
 	base := NewBonsai(tree)
 	return State{
 		committed:  base,
 		deliverTx:  base.Checkpoint(),
 		checkTx:    base.Checkpoint(),
-		persistent: persistent,
+		persistent: true,
 	}
 }
 
@@ -37,19 +37,12 @@ func (s State) Check() SimpleDB {
 	return s.checkTx
 }
 
-// Hash applies deliverTx to committed and calculates hash
-//
-// Note the usage:
-// Hash -> gets the calculated hash but doesn't save
-// BatchSet -> adds some out-of-bounds data
-// Commit -> Save everything to disk and the same hash
-func (s *State) Hash() ([]byte, error) {
-	err := s.committed.Commit(s.deliverTx)
-	if err != nil {
-		return nil, err
-	}
-	s.deliverTx = s.committed.Checkpoint()
-	return s.committed.Tree.Hash(), nil
+func (s State) LatestHeight() uint64 {
+	return s.committed.Tree.LatestVersion()
+}
+
+func (s State) LatestHash() []byte {
+	return s.committed.Tree.Hash()
 }
 
 // BatchSet is used for some weird magic in storing the new height
@@ -61,7 +54,7 @@ func (s *State) BatchSet(key, value []byte) {
 }
 
 // Commit save persistent nodes to the database and re-copies the trees
-func (s *State) Commit() ([]byte, error) {
+func (s *State) Commit(version uint64) ([]byte, error) {
 	// commit (if we didn't do hash earlier)
 	err := s.committed.Commit(s.deliverTx)
 	if err != nil {
@@ -70,7 +63,12 @@ func (s *State) Commit() ([]byte, error) {
 
 	var hash []byte
 	if s.persistent {
-		hash = s.committed.Tree.Save()
+		if s.committed.Tree.Size() > 0 || s.committed.Tree.LatestVersion() > 0 {
+			hash, err = s.committed.Tree.SaveVersion(version)
+			if err != nil {
+				return nil, err
+			}
+		}
 	} else {
 		hash = s.committed.Tree.Hash()
 	}

state/merkle_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/tendermint/iavl"
+	db "github.com/tendermint/tmlibs/db"
 )
 
 type keyVal struct {
@@ -64,8 +65,8 @@ func TestStateCommitHash(t *testing.T) {
 			result := make([][]byte, len(tc.rounds))
 
 			// make the store...
-			tree := iavl.NewIAVLTree(0, nil)
-			store := NewState(tree, false)
+			tree := iavl.NewVersionedTree(0, db.NewMemDB())
+			store := NewState(tree)
 
 			for n, r := range tc.rounds {
 				// start the cache
@@ -76,7 +77,7 @@ func TestStateCommitHash(t *testing.T) {
 					deliver.Set(k, v)
 				}
 				// commit and add hash to result
-				hash, err := store.Commit()
+				hash, err := store.Commit(uint64(n + 1))
 				require.Nil(err, "tc:%d / rnd:%d - %+v", i, n, err)
 				result[n] = hash
 			}

state/store_test.go

@@ -17,11 +17,11 @@ func GetDBs() []SimpleDB {
 		panic(err)
 	}
 	db := dbm.NewDB("test-get-dbs", dbm.LevelDBBackendStr, tmpDir)
-	persist := iavl.NewIAVLTree(500, db)
+	persist := iavl.NewVersionedTree(500, db)
 
 	return []SimpleDB{
 		NewMemKVStore(),
-		NewBonsai(iavl.NewIAVLTree(0, nil)),
+		NewBonsai(iavl.NewVersionedTree(0, dbm.NewMemDB())),
 		NewBonsai(persist),
 	}
 }