From a52cdbfe435155d39b04b970850bb15f253fb227 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 9 Nov 2017 17:35:46 -0500 Subject: [PATCH 01/27] extract tags from DeliverTx/Result and send them along with predefined --- consensus/replay.go | 1 + glide.lock | 2 +- glide.yaml | 2 +- state/execution.go | 1 + state/state_test.go | 4 ++-- state/txindex/kv/kv_test.go | 4 ++-- types/event_bus.go | 14 +++++++++++++- types/events.go | 13 +++++++------ 8 files changed, 28 insertions(+), 13 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index fb1c49a1..38a5eef3 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -392,6 +392,7 @@ func (mock *mockProxyApp) DeliverTx(tx []byte) abci.Result { r.Code, r.Data, r.Log, + r.Tags, } } diff --git a/glide.lock b/glide.lock index e12ddb4e..ccb74759 100644 --- a/glide.lock +++ b/glide.lock @@ -98,7 +98,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 76ef8a0697c6179220a74c479b36c27a5b53008a + version: 6b47155e08732f46dafdcef185d23f0ff9ff24a5 subpackages: - client - example/counter diff --git a/glide.yaml b/glide.yaml index 0f07dc2d..19485fb6 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: ~0.7.0 + version: 6b47155e08732f46dafdcef185d23f0ff9ff24a5 subpackages: - client - example/dummy diff --git a/state/execution.go b/state/execution.go index 6c74f7a9..aa4cd9c8 100644 --- a/state/execution.go +++ b/state/execution.go @@ -75,6 +75,7 @@ func execBlockOnProxyApp(txEventPublisher types.TxEventPublisher, proxyAppConn p Data: txResult.Data, Code: txResult.Code, Log: txResult.Log, + Tags: txResult.Tags, Error: txError, } txEventPublisher.PublishEventTx(event) diff --git a/state/state_test.go b/state/state_test.go index 7bb43afa..b60f1546 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -78,8 +78,8 @@ func TestABCIResponsesSaveLoad(t *testing.T) { // build mock responses block := makeBlock(2, state) abciResponses := NewABCIResponses(block) - abciResponses.DeliverTx[0] = &abci.ResponseDeliverTx{Data: []byte("foo")} - abciResponses.DeliverTx[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok"} + abciResponses.DeliverTx[0] = &abci.ResponseDeliverTx{Data: []byte("foo"), Tags: []*abci.KVPair{}} + abciResponses.DeliverTx[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok", Tags: []*abci.KVPair{}} abciResponses.EndBlock = abci.ResponseEndBlock{Diffs: []*abci.Validator{ { PubKey: crypto.GenPrivKeyEd25519().PubKey().Bytes(), diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 673674b3..c0f1403e 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -17,7 +17,7 @@ func TestTxIndex(t *testing.T) { indexer := &TxIndex{store: db.NewMemDB()} tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} hash := tx.Hash() batch := txindex.NewBatch(1) @@ -34,7 +34,7 @@ func TestTxIndex(t *testing.T) { func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { diff --git a/types/event_bus.go b/types/event_bus.go index 85ef1448..479ae735 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -82,7 +82,19 @@ func (b *EventBus) PublishEventVote(vote EventDataVote) error { func (b *EventBus) PublishEventTx(tx EventDataTx) error { // no explicit deadline for publishing events ctx := context.Background() - b.pubsub.PublishWithTags(ctx, TMEventData{tx}, map[string]interface{}{EventTypeKey: EventTx, TxHashKey: fmt.Sprintf("%X", tx.Tx.Hash())}) + tags := make(map[string]interface{}) + for _, t := range tx.Tags { + // TODO [@melekes]: validate, but where? + if t.ValueString != "" { + tags[t.Key] = t.ValueString + } else { + tags[t.Key] = t.ValueInt + } + } + // predefined tags should come last + tags[EventTypeKey] = EventTx + tags[TxHashKey] = fmt.Sprintf("%X", tx.Tx.Hash()) + b.pubsub.PublishWithTags(ctx, TMEventData{tx}, tags) return nil } diff --git a/types/events.go b/types/events.go index 64b83ec9..c9de20af 100644 --- a/types/events.go +++ b/types/events.go @@ -110,12 +110,13 @@ type EventDataNewBlockHeader struct { // All txs fire EventDataTx type EventDataTx struct { - Height int `json:"height"` - Tx Tx `json:"tx"` - Data data.Bytes `json:"data"` - Log string `json:"log"` - Code abci.CodeType `json:"code"` - Error string `json:"error"` // this is redundant information for now + Height int `json:"height"` + Tx Tx `json:"tx"` + Data data.Bytes `json:"data"` + Log string `json:"log"` + Code abci.CodeType `json:"code"` + Tags []*abci.KVPair `json:"tags"` + Error string `json:"error"` // this is redundant information for now } type EventDataProposalHeartbeat struct { From acae38ab9e2f226cdd8aa714e3f775e42bb8f837 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 15 Nov 2017 14:10:54 -0600 Subject: [PATCH 02/27] validate tags --- state/execution.go | 26 +++++++++++++++++++++----- types/event_bus.go | 12 ++---------- types/events.go | 14 +++++++------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/state/execution.go b/state/execution.go index aa4cd9c8..0033e7f3 100644 --- a/state/execution.go +++ b/state/execution.go @@ -69,16 +69,32 @@ func execBlockOnProxyApp(txEventPublisher types.TxEventPublisher, proxyAppConn p // NOTE: if we count we can access the tx from the block instead of // pulling it from the req - event := types.EventDataTx{ + tx := types.Tx(req.GetDeliverTx().Tx) + + tags := make(map[string]interface{}) + for _, t := range txResult.Tags { + // basic validation + if t.Key == "" { + logger.Info("Got tag with an empty key (skipping)", "tag", t, "tx", tx) + continue + } + + if t.ValueString != "" { + tags[t.Key] = t.ValueString + } else { + tags[t.Key] = t.ValueInt + } + } + + txEventPublisher.PublishEventTx(types.EventDataTx{ Height: block.Height, - Tx: types.Tx(req.GetDeliverTx().Tx), + Tx: tx, Data: txResult.Data, Code: txResult.Code, Log: txResult.Log, - Tags: txResult.Tags, + Tags: tags, Error: txError, - } - txEventPublisher.PublishEventTx(event) + }) } } proxyAppConn.SetResponseCallback(proxyCb) diff --git a/types/event_bus.go b/types/event_bus.go index 479ae735..6091538e 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -82,16 +82,8 @@ func (b *EventBus) PublishEventVote(vote EventDataVote) error { func (b *EventBus) PublishEventTx(tx EventDataTx) error { // no explicit deadline for publishing events ctx := context.Background() - tags := make(map[string]interface{}) - for _, t := range tx.Tags { - // TODO [@melekes]: validate, but where? - if t.ValueString != "" { - tags[t.Key] = t.ValueString - } else { - tags[t.Key] = t.ValueInt - } - } - // predefined tags should come last + tags := tx.Tags + // add predefined tags (they should overwrite any existing tags) tags[EventTypeKey] = EventTx tags[TxHashKey] = fmt.Sprintf("%X", tx.Tx.Hash()) b.pubsub.PublishWithTags(ctx, TMEventData{tx}, tags) diff --git a/types/events.go b/types/events.go index c9de20af..f20297d6 100644 --- a/types/events.go +++ b/types/events.go @@ -110,13 +110,13 @@ type EventDataNewBlockHeader struct { // All txs fire EventDataTx type EventDataTx struct { - Height int `json:"height"` - Tx Tx `json:"tx"` - Data data.Bytes `json:"data"` - Log string `json:"log"` - Code abci.CodeType `json:"code"` - Tags []*abci.KVPair `json:"tags"` - Error string `json:"error"` // this is redundant information for now + Height int `json:"height"` + Tx Tx `json:"tx"` + Data data.Bytes `json:"data"` + Log string `json:"log"` + Code abci.CodeType `json:"code"` + Tags map[string]interface{} `json:"tags"` + Error string `json:"error"` // this is redundant information for now } type EventDataProposalHeartbeat struct { From cd4be1f30896a3afb51991f765d11f9abbaa127b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 15 Nov 2017 14:11:13 -0600 Subject: [PATCH 03/27] add tx_index config --- config/config.go | 28 ++++++++++++++++++++++++---- node/node.go | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 25d6c44a..97d55ff8 100644 --- a/config/config.go +++ b/config/config.go @@ -16,6 +16,7 @@ type Config struct { P2P *P2PConfig `mapstructure:"p2p"` Mempool *MempoolConfig `mapstructure:"mempool"` Consensus *ConsensusConfig `mapstructure:"consensus"` + TxIndex *TxIndexConfig `mapstructure:"tx_index"` } // DefaultConfig returns a default configuration for a Tendermint node @@ -26,6 +27,7 @@ func DefaultConfig() *Config { P2P: DefaultP2PConfig(), Mempool: DefaultMempoolConfig(), Consensus: DefaultConsensusConfig(), + TxIndex: DefaultTxIndexConfig(), } } @@ -37,6 +39,7 @@ func TestConfig() *Config { P2P: TestP2PConfig(), Mempool: DefaultMempoolConfig(), Consensus: TestConsensusConfig(), + TxIndex: DefaultTxIndexConfig(), } } @@ -93,9 +96,6 @@ type BaseConfig struct { // so the app can decide if we should keep the connection or not FilterPeers bool `mapstructure:"filter_peers"` // false - // What indexer to use for transactions - TxIndex string `mapstructure:"tx_index"` - // Database backend: leveldb | memdb DBBackend string `mapstructure:"db_backend"` @@ -115,7 +115,6 @@ func DefaultBaseConfig() BaseConfig { ProfListenAddress: "", FastSync: true, FilterPeers: false, - TxIndex: "kv", DBBackend: "leveldb", DBPath: "data", } @@ -412,6 +411,27 @@ func (c *ConsensusConfig) SetWalFile(walFile string) { c.walFile = walFile } +//----------------------------------------------------------------------------- +// TxIndexConfig + +// TxIndexConfig defines the confuguration for the transaction +// indexer, including tags to index. +type TxIndexConfig struct { + // What indexer to use for transactions + Indexer string `mapstructure:"indexer"` + + // Comma-separated list of tags to index (by default only by tx hash) + IndexTags string `mapstructure:"index_tags"` +} + +// DefaultTxIndexConfig returns a default configuration for the transaction indexer. +func DefaultTxIndexConfig() *TxIndexConfig { + return &TxIndexConfig{ + Indexer: "kv", + IndexTags: "tx.hash", // types.TxHashKey + } +} + //----------------------------------------------------------------------------- // Utils diff --git a/node/node.go b/node/node.go index 5b8ab994..c0e4197b 100644 --- a/node/node.go +++ b/node/node.go @@ -175,7 +175,7 @@ func NewNode(config *cfg.Config, // Transaction indexing var txIndexer txindex.TxIndexer - switch config.TxIndex { + switch config.TxIndex.Indexer { case "kv": store, err := dbProvider(&DBContext{"tx_index", config}) if err != nil { From 29cd1a1b8f3ba1e71b5fb7be96adbf064abc510e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 15 Nov 2017 15:07:08 -0600 Subject: [PATCH 04/27] rewrite indexer to be a listener of eventBus --- node/node.go | 38 ++++++++++------ rpc/client/event_test.go | 4 +- rpc/client/rpc_test.go | 10 ++--- rpc/core/mempool.go | 8 ++-- rpc/core/tx.go | 8 ++-- rpc/core/types/responses.go | 6 +-- state/execution.go | 63 ++++---------------------- state/execution_test.go | 18 -------- state/state.go | 12 +---- state/txindex/indexer.go | 8 ++-- state/txindex/kv/kv.go | 9 +++- state/txindex/kv/kv_test.go | 11 +++++ state/txindex/null/null.go | 5 +++ types/event_bus.go | 90 ++++++++++++++++++++++++------------- types/events.go | 12 ++--- 15 files changed, 141 insertions(+), 161 deletions(-) diff --git a/node/node.go b/node/node.go index c0e4197b..5efe39b9 100644 --- a/node/node.go +++ b/node/node.go @@ -173,20 +173,6 @@ func NewNode(config *cfg.Config, state = sm.LoadState(stateDB) state.SetLogger(stateLogger) - // Transaction indexing - var txIndexer txindex.TxIndexer - switch config.TxIndex.Indexer { - case "kv": - store, err := dbProvider(&DBContext{"tx_index", config}) - if err != nil { - return nil, err - } - txIndexer = kv.NewTxIndex(store) - default: - txIndexer = &null.TxIndex{} - } - state.TxIndexer = txIndexer - // Generate node PrivKey privKey := crypto.GenPrivKeyEd25519() @@ -293,6 +279,30 @@ func NewNode(config *cfg.Config, bcReactor.SetEventBus(eventBus) consensusReactor.SetEventBus(eventBus) + // Transaction indexing + var txIndexer txindex.TxIndexer + switch config.TxIndex.Indexer { + case "kv": + store, err := dbProvider(&DBContext{"tx_index", config}) + if err != nil { + return nil, err + } + txIndexer = kv.NewTxIndex(store) + default: + txIndexer = &null.TxIndex{} + } + + // subscribe for all transactions and index them by tags + ch := make(chan interface{}) + eventBus.Subscribe(context.Background(), "tx_index", types.EventQueryTx, ch) + go func() { + for event := range ch { + // XXX: may be not perfomant to write one event at a time + txResult := event.(types.TMEventData).Unwrap().(types.EventDataTx).TxResult + txIndexer.Index(&txResult) + } + }() + // run the profile server profileHost := config.ProfListenAddress if profileHost != "" { diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go index 9619e5c0..96328229 100644 --- a/rpc/client/event_test.go +++ b/rpc/client/event_test.go @@ -100,7 +100,7 @@ func TestTxEventsSentWithBroadcastTxAsync(t *testing.T) { require.True(ok, "%d: %#v", i, evt) // make sure this is the proper tx require.EqualValues(tx, txe.Tx) - require.True(txe.Code.IsOK()) + require.True(txe.Result.Code.IsOK()) } } @@ -132,6 +132,6 @@ func TestTxEventsSentWithBroadcastTxSync(t *testing.T) { require.True(ok, "%d: %#v", i, evt) // make sure this is the proper tx require.EqualValues(tx, txe.Tx) - require.True(txe.Code.IsOK()) + require.True(txe.Result.Code.IsOK()) } } diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index c6827635..b6b3d9e2 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -104,7 +104,7 @@ func TestABCIQuery(t *testing.T) { k, v, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) require.Nil(t, err, "%d: %+v", i, err) - apph := bres.Height + 1 // this is where the tx will be applied to the state + apph := int(bres.Height) + 1 // this is where the tx will be applied to the state // wait before querying client.WaitForHeight(c, apph, nil) @@ -136,7 +136,7 @@ func TestAppCalls(t *testing.T) { bres, err := c.BroadcastTxCommit(tx) require.Nil(err, "%d: %+v", i, err) require.True(bres.DeliverTx.Code.IsOK()) - txh := bres.Height + txh := int(bres.Height) apph := txh + 1 // this is where the tx will be applied to the state // wait before querying @@ -153,7 +153,7 @@ func TestAppCalls(t *testing.T) { // ptx, err := c.Tx(bres.Hash, true) ptx, err := c.Tx(bres.Hash, true) require.Nil(err, "%d: %+v", i, err) - assert.Equal(txh, ptx.Height) + assert.EqualValues(txh, ptx.Height) assert.EqualValues(tx, ptx.Tx) // and we can even check the block is added @@ -280,9 +280,9 @@ func TestTx(t *testing.T) { require.NotNil(err) } else { require.Nil(err, "%+v", err) - assert.Equal(txHeight, ptx.Height) + assert.EqualValues(txHeight, ptx.Height) assert.EqualValues(tx, ptx.Tx) - assert.Equal(0, ptx.Index) + assert.Zero(ptx.Index) assert.True(ptx.TxResult.Code.IsOK()) // time to verify the proof diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 382b2f55..88c5bd2b 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -154,7 +154,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { ctx, cancel := context.WithTimeout(context.Background(), subscribeTimeout) defer cancel() deliverTxResCh := make(chan interface{}) - q := types.EventQueryTx(tx) + q := types.EventQueryTxFor(tx) err := eventBus.Subscribe(ctx, "mempool", q, deliverTxResCh) if err != nil { err = errors.Wrap(err, "failed to subscribe to tx") @@ -192,9 +192,9 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { deliverTxRes := deliverTxResMsg.(types.TMEventData).Unwrap().(types.EventDataTx) // The tx was included in a block. deliverTxR := &abci.ResponseDeliverTx{ - Code: deliverTxRes.Code, - Data: deliverTxRes.Data, - Log: deliverTxRes.Log, + Code: deliverTxRes.Result.Code, + Data: deliverTxRes.Result.Data, + Log: deliverTxRes.Result.Log, } logger.Info("DeliverTx passed ", "tx", data.Bytes(tx), "response", deliverTxR) return &ctypes.ResultBroadcastTxCommit{ diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 03a911e2..dc842e62 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -82,13 +82,13 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return nil, fmt.Errorf("Tx (%X) not found", hash) } - height := int(r.Height) // XXX - index := int(r.Index) + height := r.Height + index := r.Index var proof types.TxProof if prove { - block := blockStore.LoadBlock(height) - proof = block.Data.Txs.Proof(index) + block := blockStore.LoadBlock(int(height)) + proof = block.Data.Txs.Proof(int(index)) } return &ctypes.ResultTx{ diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 8aa904fe..e4c5d8fc 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -107,12 +107,12 @@ type ResultBroadcastTxCommit struct { CheckTx abci.Result `json:"check_tx"` DeliverTx abci.Result `json:"deliver_tx"` Hash data.Bytes `json:"hash"` - Height int `json:"height"` + Height uint64 `json:"height"` } type ResultTx struct { - Height int `json:"height"` - Index int `json:"index"` + Height uint64 `json:"height"` + Index uint32 `json:"index"` TxResult abci.Result `json:"tx_result"` Tx types.Tx `json:"tx"` Proof types.TxProof `json:"proof,omitempty"` diff --git a/state/execution.go b/state/execution.go index 0033e7f3..be09b2b2 100644 --- a/state/execution.go +++ b/state/execution.go @@ -8,7 +8,6 @@ import ( abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -54,47 +53,25 @@ func execBlockOnProxyApp(txEventPublisher types.TxEventPublisher, proxyAppConn p // TODO: make use of this info // Blocks may include invalid txs. // reqDeliverTx := req.(abci.RequestDeliverTx) - txError := "" txResult := r.DeliverTx if txResult.Code == abci.CodeType_OK { validTxs++ } else { logger.Debug("Invalid tx", "code", txResult.Code, "log", txResult.Log) invalidTxs++ - txError = txResult.Code.String() } - abciResponses.DeliverTx[txIndex] = txResult - txIndex++ - // NOTE: if we count we can access the tx from the block instead of // pulling it from the req - tx := types.Tx(req.GetDeliverTx().Tx) + txEventPublisher.PublishEventTx(types.EventDataTx{types.TxResult{ + Height: uint64(block.Height), + Index: uint32(txIndex), + Tx: types.Tx(req.GetDeliverTx().Tx), + Result: *txResult, + }}) - tags := make(map[string]interface{}) - for _, t := range txResult.Tags { - // basic validation - if t.Key == "" { - logger.Info("Got tag with an empty key (skipping)", "tag", t, "tx", tx) - continue - } - - if t.ValueString != "" { - tags[t.Key] = t.ValueString - } else { - tags[t.Key] = t.ValueInt - } - } - - txEventPublisher.PublishEventTx(types.EventDataTx{ - Height: block.Height, - Tx: tx, - Data: txResult.Data, - Code: txResult.Code, - Log: txResult.Log, - Tags: tags, - Error: txError, - }) + abciResponses.DeliverTx[txIndex] = txResult + txIndex++ } } proxyAppConn.SetResponseCallback(proxyCb) @@ -227,7 +204,6 @@ func (s *State) validateBlock(block *types.Block) error { //----------------------------------------------------------------------------- // ApplyBlock validates & executes the block, updates state w/ ABCI responses, // then commits and updates the mempool atomically, then saves state. -// Transaction results are optionally indexed. // ApplyBlock validates the block against the state, executes it against the app, // commits it, and saves the block and state. It's the only function that needs to be called @@ -242,9 +218,6 @@ func (s *State) ApplyBlock(txEventPublisher types.TxEventPublisher, proxyAppConn fail.Fail() // XXX - // index txs. This could run in the background - s.indexTxs(abciResponses) - // save the results before we commit s.SaveABCIResponses(abciResponses) @@ -293,26 +266,6 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl return mempool.Update(block.Height, block.Txs) } -func (s *State) indexTxs(abciResponses *ABCIResponses) { - // save the tx results using the TxIndexer - // NOTE: these may be overwriting, but the values should be the same. - batch := txindex.NewBatch(len(abciResponses.DeliverTx)) - for i, d := range abciResponses.DeliverTx { - tx := abciResponses.txs[i] - if err := batch.Add(types.TxResult{ - Height: uint64(abciResponses.Height), - Index: uint32(i), - Tx: tx, - Result: *d, - }); err != nil { - s.logger.Error("Error with batch.Add", "err", err) - } - } - if err := s.TxIndexer.AddBatch(batch); err != nil { - s.logger.Error("Error adding batch", "err", err) - } -} - // ExecCommitBlock executes and commits a block on the proxyApp without validating or mutating the state. // It returns the application root hash (result of abci.Commit). func ExecCommitBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block, logger log.Logger) ([]byte, error) { diff --git a/state/execution_test.go b/state/execution_test.go index 5b9bf168..e54d983d 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -3,13 +3,11 @@ package state import ( "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tendermint/abci/example/dummy" crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -31,8 +29,6 @@ func TestApplyBlock(t *testing.T) { state := state() state.SetLogger(log.TestingLogger()) - indexer := &dummyIndexer{0} - state.TxIndexer = indexer // make block block := makeBlock(1, state) @@ -40,7 +36,6 @@ func TestApplyBlock(t *testing.T) { err = state.ApplyBlock(types.NopEventBus{}, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), types.MockMempool{}) require.Nil(t, err) - assert.Equal(t, nTxsPerBlock, indexer.Indexed) // test indexing works // TODO check state and mempool } @@ -75,16 +70,3 @@ func makeBlock(num int, state *State) *types.Block { prevBlockID, valHash, state.AppHash, testPartSize) return block } - -// dummyIndexer increments counter every time we index transaction. -type dummyIndexer struct { - Indexed int -} - -func (indexer *dummyIndexer) Get(hash []byte) (*types.TxResult, error) { - return nil, nil -} -func (indexer *dummyIndexer) AddBatch(batch *txindex.Batch) error { - indexer.Indexed += batch.Size() - return nil -} diff --git a/state/state.go b/state/state.go index 4241f9de..1c2b3efe 100644 --- a/state/state.go +++ b/state/state.go @@ -15,8 +15,6 @@ import ( wire "github.com/tendermint/go-wire" - "github.com/tendermint/tendermint/state/txindex" - "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" ) @@ -61,9 +59,6 @@ type State struct { // AppHash is updated after Commit AppHash []byte - // TxIndexer indexes transactions - TxIndexer txindex.TxIndexer `json:"-"` - logger log.Logger } @@ -95,7 +90,7 @@ func loadState(db dbm.DB, key []byte) *State { return nil } - s := &State{db: db, TxIndexer: &null.TxIndex{}} + s := &State{db: db} r, n, err := bytes.NewReader(buf), new(int), new(error) wire.ReadBinaryPtr(&s, r, 0, n, err) if *err != nil { @@ -114,8 +109,6 @@ func (s *State) SetLogger(l log.Logger) { } // Copy makes a copy of the State for mutating. -// NOTE: Does not create a copy of TxIndexer. It creates a new pointer that points to the same -// underlying TxIndexer. func (s *State) Copy() *State { return &State{ db: s.db, @@ -125,7 +118,6 @@ func (s *State) Copy() *State { Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), AppHash: s.AppHash, - TxIndexer: s.TxIndexer, LastHeightValidatorsChanged: s.LastHeightValidatorsChanged, logger: s.logger, ChainID: s.ChainID, @@ -368,7 +360,6 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) (*State, error) { } } - // we do not need indexer during replay and in tests return &State{ db: db, @@ -381,7 +372,6 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) (*State, error) { Validators: types.NewValidatorSet(validators), LastValidators: types.NewValidatorSet(nil), AppHash: genDoc.AppHash, - TxIndexer: &null.TxIndex{}, LastHeightValidatorsChanged: 1, }, nil } diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index 039460a1..2c37283c 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -9,12 +9,12 @@ import ( // TxIndexer interface defines methods to index and search transactions. type TxIndexer interface { - // AddBatch analyzes, indexes or stores a batch of transactions. - // NOTE: We do not specify Index method for analyzing a single transaction - // here because it bears heavy performance losses. Almost all advanced indexers - // support batching. + // AddBatch analyzes, indexes and stores a batch of transactions. AddBatch(b *Batch) error + // Index analyzes, indexes and stores a single transaction. + Index(result *types.TxResult) error + // Get returns the transaction specified by hash or nil if the transaction is not indexed // or stored. Get(hash []byte) (*types.TxResult, error) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index db075e54..a3826c8b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -4,7 +4,7 @@ import ( "bytes" "fmt" - "github.com/tendermint/go-wire" + wire "github.com/tendermint/go-wire" db "github.com/tendermint/tmlibs/db" @@ -56,3 +56,10 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { storeBatch.Write() return nil } + +// Index writes a single transaction into the TxIndex storage. +func (txi *TxIndex) Index(result *types.TxResult) error { + rawBytes := wire.BinaryBytes(result) + txi.store.Set(result.Tx.Hash(), rawBytes) + return nil +} diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index c0f1403e..f814fabe 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -30,6 +30,17 @@ func TestTxIndex(t *testing.T) { loadedTxResult, err := indexer.Get(hash) require.Nil(t, err) assert.Equal(t, txResult, loadedTxResult) + + tx2 := types.Tx("BYE BYE WORLD") + txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} + hash2 := tx2.Hash() + + err = indexer.Index(txResult2) + require.Nil(t, err) + + loadedTxResult2, err := indexer.Get(hash2) + require.Nil(t, err) + assert.Equal(t, txResult2, loadedTxResult2) } func benchmarkTxIndex(txsCount int, b *testing.B) { diff --git a/state/txindex/null/null.go b/state/txindex/null/null.go index 4939d6d8..27e81d73 100644 --- a/state/txindex/null/null.go +++ b/state/txindex/null/null.go @@ -19,3 +19,8 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { func (txi *TxIndex) AddBatch(batch *txindex.Batch) error { return nil } + +// Index is a noop and always returns nil. +func (txi *TxIndex) Index(result *types.TxResult) error { + return nil +} diff --git a/types/event_bus.go b/types/event_bus.go index 6091538e..a4daaa3c 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -67,67 +67,95 @@ func (b *EventBus) Publish(eventType string, eventData TMEventData) error { //--- block, tx, and vote events -func (b *EventBus) PublishEventNewBlock(block EventDataNewBlock) error { - return b.Publish(EventNewBlock, TMEventData{block}) +func (b *EventBus) PublishEventNewBlock(event EventDataNewBlock) error { + return b.Publish(EventNewBlock, TMEventData{event}) } -func (b *EventBus) PublishEventNewBlockHeader(header EventDataNewBlockHeader) error { - return b.Publish(EventNewBlockHeader, TMEventData{header}) +func (b *EventBus) PublishEventNewBlockHeader(event EventDataNewBlockHeader) error { + return b.Publish(EventNewBlockHeader, TMEventData{event}) } -func (b *EventBus) PublishEventVote(vote EventDataVote) error { - return b.Publish(EventVote, TMEventData{vote}) +func (b *EventBus) PublishEventVote(event EventDataVote) error { + return b.Publish(EventVote, TMEventData{event}) } -func (b *EventBus) PublishEventTx(tx EventDataTx) error { +// PublishEventTx publishes tx event with tags from Result. Note it will add +// predefined tags (EventTypeKey, TxHashKey). Existing tags with the same names +// will be overwritten. +func (b *EventBus) PublishEventTx(event EventDataTx) error { // no explicit deadline for publishing events ctx := context.Background() - tags := tx.Tags - // add predefined tags (they should overwrite any existing tags) + + tags := make(map[string]interface{}) + + // validate and fill tags from tx result + for _, tag := range event.Result.Tags { + // basic validation + if tag.Key == "" { + b.Logger.Info("Got tag with an empty key (skipping)", "tag", tag, "tx", event.Tx) + continue + } + + if tag.ValueString != "" { + tags[tag.Key] = tag.ValueString + } else { + tags[tag.Key] = tag.ValueInt + } + } + + // add predefined tags + if tag, ok := tags[EventTypeKey]; ok { + b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) + } tags[EventTypeKey] = EventTx - tags[TxHashKey] = fmt.Sprintf("%X", tx.Tx.Hash()) - b.pubsub.PublishWithTags(ctx, TMEventData{tx}, tags) + + if tag, ok := tags[TxHashKey]; ok { + b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) + } + tags[TxHashKey] = fmt.Sprintf("%X", event.Tx.Hash()) + + b.pubsub.PublishWithTags(ctx, TMEventData{event}, tags) return nil } -func (b *EventBus) PublishEventProposalHeartbeat(ph EventDataProposalHeartbeat) error { - return b.Publish(EventProposalHeartbeat, TMEventData{ph}) +func (b *EventBus) PublishEventProposalHeartbeat(event EventDataProposalHeartbeat) error { + return b.Publish(EventProposalHeartbeat, TMEventData{event}) } //--- EventDataRoundState events -func (b *EventBus) PublishEventNewRoundStep(rs EventDataRoundState) error { - return b.Publish(EventNewRoundStep, TMEventData{rs}) +func (b *EventBus) PublishEventNewRoundStep(event EventDataRoundState) error { + return b.Publish(EventNewRoundStep, TMEventData{event}) } -func (b *EventBus) PublishEventTimeoutPropose(rs EventDataRoundState) error { - return b.Publish(EventTimeoutPropose, TMEventData{rs}) +func (b *EventBus) PublishEventTimeoutPropose(event EventDataRoundState) error { + return b.Publish(EventTimeoutPropose, TMEventData{event}) } -func (b *EventBus) PublishEventTimeoutWait(rs EventDataRoundState) error { - return b.Publish(EventTimeoutWait, TMEventData{rs}) +func (b *EventBus) PublishEventTimeoutWait(event EventDataRoundState) error { + return b.Publish(EventTimeoutWait, TMEventData{event}) } -func (b *EventBus) PublishEventNewRound(rs EventDataRoundState) error { - return b.Publish(EventNewRound, TMEventData{rs}) +func (b *EventBus) PublishEventNewRound(event EventDataRoundState) error { + return b.Publish(EventNewRound, TMEventData{event}) } -func (b *EventBus) PublishEventCompleteProposal(rs EventDataRoundState) error { - return b.Publish(EventCompleteProposal, TMEventData{rs}) +func (b *EventBus) PublishEventCompleteProposal(event EventDataRoundState) error { + return b.Publish(EventCompleteProposal, TMEventData{event}) } -func (b *EventBus) PublishEventPolka(rs EventDataRoundState) error { - return b.Publish(EventPolka, TMEventData{rs}) +func (b *EventBus) PublishEventPolka(event EventDataRoundState) error { + return b.Publish(EventPolka, TMEventData{event}) } -func (b *EventBus) PublishEventUnlock(rs EventDataRoundState) error { - return b.Publish(EventUnlock, TMEventData{rs}) +func (b *EventBus) PublishEventUnlock(event EventDataRoundState) error { + return b.Publish(EventUnlock, TMEventData{event}) } -func (b *EventBus) PublishEventRelock(rs EventDataRoundState) error { - return b.Publish(EventRelock, TMEventData{rs}) +func (b *EventBus) PublishEventRelock(event EventDataRoundState) error { + return b.Publish(EventRelock, TMEventData{event}) } -func (b *EventBus) PublishEventLock(rs EventDataRoundState) error { - return b.Publish(EventLock, TMEventData{rs}) +func (b *EventBus) PublishEventLock(event EventDataRoundState) error { + return b.Publish(EventLock, TMEventData{event}) } diff --git a/types/events.go b/types/events.go index f20297d6..03e5e795 100644 --- a/types/events.go +++ b/types/events.go @@ -3,7 +3,6 @@ package types import ( "fmt" - abci "github.com/tendermint/abci/types" "github.com/tendermint/go-wire/data" tmpubsub "github.com/tendermint/tmlibs/pubsub" tmquery "github.com/tendermint/tmlibs/pubsub/query" @@ -110,13 +109,7 @@ type EventDataNewBlockHeader struct { // All txs fire EventDataTx type EventDataTx struct { - Height int `json:"height"` - Tx Tx `json:"tx"` - Data data.Bytes `json:"data"` - Log string `json:"log"` - Code abci.CodeType `json:"code"` - Tags map[string]interface{} `json:"tags"` - Error string `json:"error"` // this is redundant information for now + TxResult } type EventDataProposalHeartbeat struct { @@ -168,9 +161,10 @@ var ( EventQueryTimeoutWait = queryForEvent(EventTimeoutWait) EventQueryVote = queryForEvent(EventVote) EventQueryProposalHeartbeat = queryForEvent(EventProposalHeartbeat) + EventQueryTx = queryForEvent(EventTx) ) -func EventQueryTx(tx Tx) tmpubsub.Query { +func EventQueryTxFor(tx Tx) tmpubsub.Query { return tmquery.MustParse(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTx, TxHashKey, tx.Hash())) } From 4a31532897276249a998357aa094c7ca4053c4df Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 22 Nov 2017 18:54:31 -0600 Subject: [PATCH 05/27] remove unreachable code --- rpc/core/mempool.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 88c5bd2b..72cf2865 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -211,8 +211,6 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { Hash: tx.Hash(), }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } - - panic("Should never happen!") } // Get unconfirmed transactions including their number. From f65e357d2b802a070254d3b7bbb12e7f285f961f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 22 Nov 2017 18:55:09 -0600 Subject: [PATCH 06/27] adapt Tendermint to new abci.Client interface which was introduced in https://github.com/tendermint/abci/pull/130 --- consensus/mempool_test.go | 39 ++++++++++++++++++++++-------------- consensus/replay.go | 15 +++++--------- glide.lock | 2 +- glide.yaml | 2 +- mempool/mempool_test.go | 12 ++++++++--- proxy/app_conn.go | 28 +++++++++++++------------- proxy/app_conn_test.go | 4 ++-- rpc/client/mock/abci.go | 6 +++--- rpc/client/mock/abci_test.go | 4 ++-- rpc/core/abci.go | 2 +- rpc/core/mempool.go | 14 ++++++------- rpc/core/tx.go | 2 +- rpc/core/types/responses.go | 18 ++++++++--------- state/execution.go | 12 +++++++++-- state/state.go | 2 +- state/state_test.go | 7 ++++--- 16 files changed, 94 insertions(+), 75 deletions(-) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index e4d09c95..089d7b3f 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -2,6 +2,7 @@ package consensus import ( "encoding/binary" + "fmt" "testing" "time" @@ -188,33 +189,41 @@ func (app *CounterApplication) Info(req abci.RequestInfo) abci.ResponseInfo { return abci.ResponseInfo{Data: cmn.Fmt("txs:%v", app.txCount)} } -func (app *CounterApplication) DeliverTx(tx []byte) abci.Result { - return runTx(tx, &app.txCount) +func (app *CounterApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { + txValue := txAsUint64(tx) + if txValue != uint64(app.txCount) { + return abci.ResponseDeliverTx{ + Code: abci.CodeType_BadNonce, + Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)} + } + app.txCount += 1 + return abci.ResponseDeliverTx{Code: abci.CodeType_OK} } -func (app *CounterApplication) CheckTx(tx []byte) abci.Result { - return runTx(tx, &app.mempoolTxCount) +func (app *CounterApplication) CheckTx(tx []byte) abci.ResponseCheckTx { + txValue := txAsUint64(tx) + if txValue != uint64(app.mempoolTxCount) { + return abci.ResponseCheckTx{ + Code: abci.CodeType_BadNonce, + Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)} + } + app.mempoolTxCount += 1 + return abci.ResponseCheckTx{Code: abci.CodeType_OK} } -func runTx(tx []byte, countPtr *int) abci.Result { - count := *countPtr +func txAsUint64(tx []byte) uint64 { tx8 := make([]byte, 8) copy(tx8[len(tx8)-len(tx):], tx) - txValue := binary.BigEndian.Uint64(tx8) - if txValue != uint64(count) { - return abci.ErrBadNonce.AppendLog(cmn.Fmt("Invalid nonce. Expected %v, got %v", count, txValue)) - } - *countPtr += 1 - return abci.OK + return binary.BigEndian.Uint64(tx8) } -func (app *CounterApplication) Commit() abci.Result { +func (app *CounterApplication) Commit() abci.ResponseCommit { app.mempoolTxCount = app.txCount if app.txCount == 0 { - return abci.OK + return abci.ResponseCommit{Code: abci.CodeType_OK} } else { hash := make([]byte, 8) binary.BigEndian.PutUint64(hash, uint64(app.txCount)) - return abci.NewResultOK(hash, "") + return abci.ResponseCommit{Code: abci.CodeType_OK, Data: hash} } } diff --git a/consensus/replay.go b/consensus/replay.go index 38a5eef3..853d3a8d 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -385,22 +385,17 @@ type mockProxyApp struct { abciResponses *sm.ABCIResponses } -func (mock *mockProxyApp) DeliverTx(tx []byte) abci.Result { +func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx { r := mock.abciResponses.DeliverTx[mock.txCount] mock.txCount += 1 - return abci.Result{ - r.Code, - r.Data, - r.Log, - r.Tags, - } + return *r } func (mock *mockProxyApp) EndBlock(height uint64) abci.ResponseEndBlock { mock.txCount = 0 - return mock.abciResponses.EndBlock + return *mock.abciResponses.EndBlock } -func (mock *mockProxyApp) Commit() abci.Result { - return abci.NewResultOK(mock.appHash, "") +func (mock *mockProxyApp) Commit() abci.ResponseCommit { + return abci.ResponseCommit{Code: abci.CodeType_OK, Data: mock.appHash} } diff --git a/glide.lock b/glide.lock index ccb74759..09f9ad2b 100644 --- a/glide.lock +++ b/glide.lock @@ -98,7 +98,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 6b47155e08732f46dafdcef185d23f0ff9ff24a5 + version: 2cfad8523a54d64271d7cbc69a39433eab918aa0 subpackages: - client - example/counter diff --git a/glide.yaml b/glide.yaml index 19485fb6..a20e76db 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: 6b47155e08732f46dafdcef185d23f0ff9ff24a5 + version: 2cfad8523a54d64271d7cbc69a39433eab918aa0 subpackages: - client - example/dummy diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 2bbf9944..aa19e380 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -172,13 +172,19 @@ func TestSerialReap(t *testing.T) { for i := start; i < end; i++ { txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(i)) - res := appConnCon.DeliverTxSync(txBytes) - if !res.IsOK() { + res, err := appConnCon.DeliverTxSync(txBytes) + if err != nil { + t.Errorf("Client error committing tx: %v", err) + } + if res.IsErr() { t.Errorf("Error committing tx. Code:%v result:%X log:%v", res.Code, res.Data, res.Log) } } - res := appConnCon.CommitSync() + res, err := appConnCon.CommitSync() + if err != nil { + t.Errorf("Client error committing: %v", err) + } if len(res.Data) != 8 { t.Errorf("Error committing. Hash:%X log:%v", res.Data, res.Log) } diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 9121e8db..49c88a37 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -12,12 +12,12 @@ type AppConnConsensus interface { SetResponseCallback(abcicli.Callback) Error() error - InitChainSync(types.RequestInitChain) (err error) + InitChainSync(types.RequestInitChain) error - BeginBlockSync(types.RequestBeginBlock) (err error) + BeginBlockSync(types.RequestBeginBlock) error DeliverTxAsync(tx []byte) *abcicli.ReqRes - EndBlockSync(height uint64) (types.ResponseEndBlock, error) - CommitSync() (res types.Result) + EndBlockSync(height uint64) (*types.ResponseEndBlock, error) + CommitSync() (*types.ResponseCommit, error) } type AppConnMempool interface { @@ -33,9 +33,9 @@ type AppConnMempool interface { type AppConnQuery interface { Error() error - EchoSync(string) (res types.Result) - InfoSync(types.RequestInfo) (types.ResponseInfo, error) - QuerySync(types.RequestQuery) (types.ResponseQuery, error) + EchoSync(string) (*types.ResponseEcho, error) + InfoSync(types.RequestInfo) (*types.ResponseInfo, error) + QuerySync(types.RequestQuery) (*types.ResponseQuery, error) // SetOptionSync(key string, value string) (res types.Result) } @@ -61,11 +61,11 @@ func (app *appConnConsensus) Error() error { return app.appConn.Error() } -func (app *appConnConsensus) InitChainSync(req types.RequestInitChain) (err error) { +func (app *appConnConsensus) InitChainSync(req types.RequestInitChain) error { return app.appConn.InitChainSync(req) } -func (app *appConnConsensus) BeginBlockSync(req types.RequestBeginBlock) (err error) { +func (app *appConnConsensus) BeginBlockSync(req types.RequestBeginBlock) error { return app.appConn.BeginBlockSync(req) } @@ -73,11 +73,11 @@ func (app *appConnConsensus) DeliverTxAsync(tx []byte) *abcicli.ReqRes { return app.appConn.DeliverTxAsync(tx) } -func (app *appConnConsensus) EndBlockSync(height uint64) (types.ResponseEndBlock, error) { +func (app *appConnConsensus) EndBlockSync(height uint64) (*types.ResponseEndBlock, error) { return app.appConn.EndBlockSync(height) } -func (app *appConnConsensus) CommitSync() (res types.Result) { +func (app *appConnConsensus) CommitSync() (*types.ResponseCommit, error) { return app.appConn.CommitSync() } @@ -131,14 +131,14 @@ func (app *appConnQuery) Error() error { return app.appConn.Error() } -func (app *appConnQuery) EchoSync(msg string) (res types.Result) { +func (app *appConnQuery) EchoSync(msg string) (*types.ResponseEcho, error) { return app.appConn.EchoSync(msg) } -func (app *appConnQuery) InfoSync(req types.RequestInfo) (types.ResponseInfo, error) { +func (app *appConnQuery) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { return app.appConn.InfoSync(req) } -func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (types.ResponseQuery, error) { +func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (*types.ResponseQuery, error) { return app.appConn.QuerySync(reqQuery) } diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index 3c00f1ae..0fbad602 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -17,7 +17,7 @@ import ( type AppConnTest interface { EchoAsync(string) *abcicli.ReqRes FlushSync() error - InfoSync(types.RequestInfo) (types.ResponseInfo, error) + InfoSync(types.RequestInfo) (*types.ResponseInfo, error) } type appConnTest struct { @@ -36,7 +36,7 @@ func (app *appConnTest) FlushSync() error { return app.appConn.FlushSync() } -func (app *appConnTest) InfoSync(req types.RequestInfo) (types.ResponseInfo, error) { +func (app *appConnTest) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { return app.appConn.InfoSync(req) } diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go index e935a282..2ffa9269 100644 --- a/rpc/client/mock/abci.go +++ b/rpc/client/mock/abci.go @@ -38,7 +38,7 @@ func (a ABCIApp) ABCIQueryWithOptions(path string, data data.Bytes, opts client. func (a ABCIApp) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { res := ctypes.ResultBroadcastTxCommit{} res.CheckTx = a.App.CheckTx(tx) - if !res.CheckTx.IsOK() { + if res.CheckTx.IsErr() { return &res, nil } res.DeliverTx = a.App.DeliverTx(tx) @@ -48,7 +48,7 @@ func (a ABCIApp) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit func (a ABCIApp) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { c := a.App.CheckTx(tx) // and this gets written in a background thread... - if c.IsOK() { + if !c.IsErr() { go func() { a.App.DeliverTx(tx) }() // nolint: errcheck } return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log, tx.Hash()}, nil @@ -57,7 +57,7 @@ func (a ABCIApp) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error func (a ABCIApp) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { c := a.App.CheckTx(tx) // and this gets written in a background thread... - if c.IsOK() { + if !c.IsErr() { go func() { a.App.DeliverTx(tx) }() // nolint: errcheck } return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log, tx.Hash()}, nil diff --git a/rpc/client/mock/abci_test.go b/rpc/client/mock/abci_test.go index 36a45791..216bd7c2 100644 --- a/rpc/client/mock/abci_test.go +++ b/rpc/client/mock/abci_test.go @@ -37,8 +37,8 @@ func TestABCIMock(t *testing.T) { BroadcastCommit: mock.Call{ Args: goodTx, Response: &ctypes.ResultBroadcastTxCommit{ - CheckTx: abci.Result{Data: data.Bytes("stand")}, - DeliverTx: abci.Result{Data: data.Bytes("deliver")}, + CheckTx: abci.ResponseCheckTx{Data: data.Bytes("stand")}, + DeliverTx: abci.ResponseDeliverTx{Data: data.Bytes("deliver")}, }, Error: errors.New("bad tx"), }, diff --git a/rpc/core/abci.go b/rpc/core/abci.go index 564c0bc6..a64c3d29 100644 --- a/rpc/core/abci.go +++ b/rpc/core/abci.go @@ -93,5 +93,5 @@ func ABCIInfo() (*ctypes.ResultABCIInfo, error) { if err != nil { return nil, err } - return &ctypes.ResultABCIInfo{resInfo}, nil + return &ctypes.ResultABCIInfo{*resInfo}, nil } diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 72cf2865..857ea75b 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -177,8 +177,8 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { if checkTxR.Code != abci.CodeType_OK { // CheckTx failed! return &ctypes.ResultBroadcastTxCommit{ - CheckTx: checkTxR.Result(), - DeliverTx: abci.Result{}, + CheckTx: *checkTxR, + DeliverTx: abci.ResponseDeliverTx{}, Hash: tx.Hash(), }, nil } @@ -191,23 +191,23 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { case deliverTxResMsg := <-deliverTxResCh: deliverTxRes := deliverTxResMsg.(types.TMEventData).Unwrap().(types.EventDataTx) // The tx was included in a block. - deliverTxR := &abci.ResponseDeliverTx{ + deliverTxR := abci.ResponseDeliverTx{ Code: deliverTxRes.Result.Code, Data: deliverTxRes.Result.Data, Log: deliverTxRes.Result.Log, } logger.Info("DeliverTx passed ", "tx", data.Bytes(tx), "response", deliverTxR) return &ctypes.ResultBroadcastTxCommit{ - CheckTx: checkTxR.Result(), - DeliverTx: deliverTxR.Result(), + CheckTx: *checkTxR, + DeliverTx: deliverTxR, Hash: tx.Hash(), Height: deliverTxRes.Height, }, nil case <-timer.C: logger.Error("failed to include tx") return &ctypes.ResultBroadcastTxCommit{ - CheckTx: checkTxR.Result(), - DeliverTx: abci.Result{}, + CheckTx: *checkTxR, + DeliverTx: abci.ResponseDeliverTx{}, Hash: tx.Hash(), }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } diff --git a/rpc/core/tx.go b/rpc/core/tx.go index dc842e62..80d1cb32 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -94,7 +94,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return &ctypes.ResultTx{ Height: height, Index: index, - TxResult: r.Result.Result(), + TxResult: r.Result, Tx: r.Tx, Proof: proof, }, nil diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index e4c5d8fc..a1b7e36f 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -104,18 +104,18 @@ type ResultBroadcastTx struct { } type ResultBroadcastTxCommit struct { - CheckTx abci.Result `json:"check_tx"` - DeliverTx abci.Result `json:"deliver_tx"` - Hash data.Bytes `json:"hash"` - Height uint64 `json:"height"` + CheckTx abci.ResponseCheckTx `json:"check_tx"` + DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` + Hash data.Bytes `json:"hash"` + Height uint64 `json:"height"` } type ResultTx struct { - Height uint64 `json:"height"` - Index uint32 `json:"index"` - TxResult abci.Result `json:"tx_result"` - Tx types.Tx `json:"tx"` - Proof types.TxProof `json:"proof,omitempty"` + Height uint64 `json:"height"` + Index uint32 `json:"index"` + TxResult abci.ResponseDeliverTx `json:"tx_result"` + Tx types.Tx `json:"tx"` + Proof types.TxProof `json:"proof,omitempty"` } type ResultUnconfirmedTxs struct { diff --git a/state/execution.go b/state/execution.go index be09b2b2..5b324eff 100644 --- a/state/execution.go +++ b/state/execution.go @@ -248,7 +248,11 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl defer mempool.Unlock() // Commit block, get hash back - res := proxyAppConn.CommitSync() + res, err := proxyAppConn.CommitSync() + if err != nil { + s.logger.Error("Client error during proxyAppConn.CommitSync", "err", err) + return err + } if res.IsErr() { s.logger.Error("Error in proxyAppConn.CommitSync", "err", res) return res @@ -275,7 +279,11 @@ func ExecCommitBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block return nil, err } // Commit block, get hash back - res := appConnConsensus.CommitSync() + res, err := appConnConsensus.CommitSync() + if err != nil { + logger.Error("Client error during proxyAppConn.CommitSync", "err", res) + return nil, err + } if res.IsErr() { logger.Error("Error in proxyAppConn.CommitSync", "err", res) return nil, res diff --git a/state/state.go b/state/state.go index 1c2b3efe..e1f16835 100644 --- a/state/state.go +++ b/state/state.go @@ -279,7 +279,7 @@ type ABCIResponses struct { Height int DeliverTx []*abci.ResponseDeliverTx - EndBlock abci.ResponseEndBlock + EndBlock *abci.ResponseEndBlock txs types.Txs // reference for indexing results by hash } diff --git a/state/state_test.go b/state/state_test.go index b60f1546..7fff0774 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -80,7 +80,7 @@ func TestABCIResponsesSaveLoad(t *testing.T) { abciResponses := NewABCIResponses(block) abciResponses.DeliverTx[0] = &abci.ResponseDeliverTx{Data: []byte("foo"), Tags: []*abci.KVPair{}} abciResponses.DeliverTx[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok", Tags: []*abci.KVPair{}} - abciResponses.EndBlock = abci.ResponseEndBlock{Diffs: []*abci.Validator{ + abciResponses.EndBlock = &abci.ResponseEndBlock{Diffs: []*abci.Validator{ { PubKey: crypto.GenPrivKeyEd25519().PubKey().Bytes(), Power: 10, @@ -198,12 +198,13 @@ func makeHeaderPartsResponses(state *State, height int, block := makeBlock(height, state) _, val := state.Validators.GetByIndex(0) abciResponses := &ABCIResponses{ - Height: height, + Height: height, + EndBlock: &abci.ResponseEndBlock{Diffs: []*abci.Validator{}}, } // if the pubkey is new, remove the old and add the new if !bytes.Equal(pubkey.Bytes(), val.PubKey.Bytes()) { - abciResponses.EndBlock = abci.ResponseEndBlock{ + abciResponses.EndBlock = &abci.ResponseEndBlock{ Diffs: []*abci.Validator{ {val.PubKey.Bytes(), 0}, {pubkey.Bytes(), 10}, From 461a143a2bccc446231229f8ec85e350e8ed62f5 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 24 Nov 2017 18:22:17 -0600 Subject: [PATCH 07/27] remove tx.hash tag from config because it's mandatory --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 97d55ff8..8b1c0e28 100644 --- a/config/config.go +++ b/config/config.go @@ -428,7 +428,7 @@ type TxIndexConfig struct { func DefaultTxIndexConfig() *TxIndexConfig { return &TxIndexConfig{ Indexer: "kv", - IndexTags: "tx.hash", // types.TxHashKey + IndexTags: "", } } From 56abea74276785fdd58f046ed08dc31dc9a6c786 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 24 Nov 2017 18:22:46 -0600 Subject: [PATCH 08/27] rename tm.events.type to just tm.event --- types/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/events.go b/types/events.go index 03e5e795..ed972e93 100644 --- a/types/events.go +++ b/types/events.go @@ -136,7 +136,7 @@ type EventDataVote struct { const ( // EventTypeKey is a reserved key, used to specify event type in tags. - EventTypeKey = "tm.events.type" + EventTypeKey = "tm.event" // TxHashKey is a reserved key, used to specify transaction's hash. // see EventBus#PublishEventTx TxHashKey = "tx.hash" From 16cf7a5e0a47ecead703f8159e6ed6fc028672df Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 24 Nov 2017 18:23:17 -0600 Subject: [PATCH 09/27] use a switch when validating tags --- types/event_bus.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/types/event_bus.go b/types/event_bus.go index a4daaa3c..2e31489c 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + abci "github.com/tendermint/abci/types" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" tmpubsub "github.com/tendermint/tmlibs/pubsub" @@ -96,9 +97,10 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { continue } - if tag.ValueString != "" { + switch tag.ValueType { + case abci.KVPair_STRING: tags[tag.Key] = tag.ValueString - } else { + case abci.KVPair_INT: tags[tag.Key] = tag.ValueInt } } From ea0b20545583c97173b9ce79b96b789336de5937 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sun, 26 Nov 2017 19:16:21 -0600 Subject: [PATCH 10/27] searching transaction results --- glide.lock | 4 +- node/node.go | 2 +- state/txindex/indexer.go | 10 +- state/txindex/kv/kv.go | 334 ++++++++++++++++++++++++++++++++++-- state/txindex/kv/kv_test.go | 81 ++++++++- state/txindex/null/null.go | 11 +- types/event_bus.go | 10 ++ types/events.go | 6 + 8 files changed, 427 insertions(+), 31 deletions(-) diff --git a/glide.lock b/glide.lock index 09f9ad2b..31f1aaa9 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 223d8e42a118e7861cb673ea58a035e99d3a98c94e4b71fb52998d320f9c3b49 -updated: 2017-11-25T22:00:24.612202481-08:00 +hash: e279cca35a5cc9a68bb266015dc6a57da749b28dabca3994b2c5dbe02309f470 +updated: 2017-11-28T00:53:04.816567531Z imports: - name: github.com/btcsuite/btcd version: 8cea3866d0f7fb12d567a20744942c0d078c7d15 diff --git a/node/node.go b/node/node.go index 5efe39b9..fff550bf 100644 --- a/node/node.go +++ b/node/node.go @@ -299,7 +299,7 @@ func NewNode(config *cfg.Config, for event := range ch { // XXX: may be not perfomant to write one event at a time txResult := event.(types.TMEventData).Unwrap().(types.EventDataTx).TxResult - txIndexer.Index(&txResult) + txIndexer.Index(&txResult, strings.Split(config.TxIndex.IndexTags, ",")) } }() diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index 2c37283c..f9908f32 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -10,10 +10,10 @@ import ( type TxIndexer interface { // AddBatch analyzes, indexes and stores a batch of transactions. - AddBatch(b *Batch) error + AddBatch(b *Batch, allowedTags []string) error // Index analyzes, indexes and stores a single transaction. - Index(result *types.TxResult) error + Index(result *types.TxResult, allowedTags []string) error // Get returns the transaction specified by hash or nil if the transaction is not indexed // or stored. @@ -26,18 +26,18 @@ type TxIndexer interface { // Batch groups together multiple Index operations to be performed at the same time. // NOTE: Batch is NOT thread-safe and must not be modified after starting its execution. type Batch struct { - Ops []types.TxResult + Ops []*types.TxResult } // NewBatch creates a new Batch. func NewBatch(n int) *Batch { return &Batch{ - Ops: make([]types.TxResult, n), + Ops: make([]*types.TxResult, n), } } // Add or update an entry for the given result.Index. -func (b *Batch) Add(result types.TxResult) error { +func (b *Batch) Add(result *types.TxResult) error { b.Ops[result.Index] = result return nil } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index a3826c8b..ee81674b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -2,16 +2,24 @@ package kv import ( "bytes" + "encoding/hex" "fmt" + "strconv" + "strings" + "time" + "github.com/pkg/errors" + + abci "github.com/tendermint/abci/types" wire "github.com/tendermint/go-wire" - - db "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" + db "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tmlibs/pubsub/query" ) +var _ txindex.TxIndexer = (*TxIndex)(nil) + // TxIndex is the simplest possible indexer, backed by Key-Value storage (levelDB). // It can only index transaction by its identifier. type TxIndex struct { @@ -46,20 +54,322 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { return txResult, nil } -// AddBatch writes a batch of transactions into the TxIndex storage. -func (txi *TxIndex) AddBatch(b *txindex.Batch) error { +// AddBatch indexes a batch of transactions using the given list of tags. +func (txi *TxIndex) AddBatch(b *txindex.Batch, allowedTags []string) error { storeBatch := txi.store.NewBatch() + for _, result := range b.Ops { - rawBytes := wire.BinaryBytes(&result) - storeBatch.Set(result.Tx.Hash(), rawBytes) + hash := result.Tx.Hash() + + // index tx by tags + for _, tag := range result.Result.Tags { + if stringInSlice(tag.Key, allowedTags) { + storeBatch.Set(keyForTag(tag, result), hash) + } + } + + // index tx by hash + rawBytes := wire.BinaryBytes(result) + storeBatch.Set(hash, rawBytes) } + storeBatch.Write() return nil } -// Index writes a single transaction into the TxIndex storage. -func (txi *TxIndex) Index(result *types.TxResult) error { - rawBytes := wire.BinaryBytes(result) - txi.store.Set(result.Tx.Hash(), rawBytes) - return nil +// Index indexes a single transaction using the given list of tags. +func (txi *TxIndex) Index(result *types.TxResult, allowedTags []string) error { + batch := txindex.NewBatch(1) + batch.Add(result) + return txi.AddBatch(batch, allowedTags) +} + +func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { + hashes := make(map[string][]byte) // key - (base 16, upper-case hash) + + // get a list of conditions (like "tx.height > 5") + conditions := q.Conditions() + + // if there is a hash condition, return the result immediately + hash, err, ok := lookForHash(conditions) + if err != nil { + return []*types.TxResult{}, errors.Wrap(err, "error during searching for a hash in the query") + } else if ok { + res, err := txi.Get(hash) + return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") + } + + // conditions to skip + skipIndexes := make([]int, 0) + + // if there is a height condition ("tx.height=3"), extract it for faster lookups + height, heightIndex := lookForHeight(conditions) + if heightIndex >= 0 { + skipIndexes = append(skipIndexes, heightIndex) + } + + var hashes2 [][]byte + + // extract ranges + // if both upper and lower bounds exist, it's better to get them in order not + // no iterate over kvs that are not within range. + ranges, rangeIndexes := lookForRanges(conditions) + if len(ranges) > 0 { + skipIndexes = append(skipIndexes, rangeIndexes...) + } + for _, r := range ranges { + hashes2 = txi.matchRange(r, startKeyForRange(r, height, heightIndex > 0)) + + // initialize hashes if we're running the first time + if len(hashes) == 0 { + for _, h := range hashes2 { + hashes[hashKey(h)] = h + } + continue + } + + // no matches + if len(hashes2) == 0 { + hashes = make(map[string][]byte) + } else { + // perform intersection as we go + for _, h := range hashes2 { + k := hashKey(h) + if _, ok := hashes[k]; !ok { + delete(hashes, k) + } + } + } + } + + // for all other conditions + for i, c := range conditions { + if intInSlice(i, skipIndexes) { + continue + } + + hashes2 = txi.match(c, startKey(c, height, heightIndex > 0)) + + // initialize hashes if we're running the first time + if len(hashes) == 0 { + for _, h := range hashes2 { + hashes[hashKey(h)] = h + } + continue + } + + // no matches + if len(hashes2) == 0 { + hashes = make(map[string][]byte) + } else { + // perform intersection as we go + for _, h := range hashes2 { + k := hashKey(h) + if _, ok := hashes[k]; !ok { + delete(hashes, k) + } + } + } + } + + results := make([]*types.TxResult, len(hashes)) + i := 0 + for _, h := range hashes { + results[i], err = txi.Get(h) + if err != nil { + return []*types.TxResult{}, errors.Wrapf(err, "failed to get Tx{%X}", h) + } + i++ + } + + return results, nil +} + +func lookForHash(conditions []query.Condition) (hash []byte, err error, ok bool) { + for _, c := range conditions { + if c.Tag == types.TxHashKey { + decoded, err := hex.DecodeString(c.Operand.(string)) + return decoded, err, true + } + } + return +} + +func lookForHeight(conditions []query.Condition) (height uint64, index int) { + for i, c := range conditions { + if c.Tag == types.TxHeightKey { + return uint64(c.Operand.(int64)), i + } + } + return 0, -1 +} + +type queryRanges map[string]queryRange + +type queryRange struct { + key string + lowerBound interface{} // int || time.Time + includeLowerBound bool + upperBound interface{} // int || time.Time + includeUpperBound bool +} + +func lookForRanges(conditions []query.Condition) (ranges queryRanges, indexes []int) { + ranges = make(queryRanges) + for i, c := range conditions { + if isRangeOperation(c.Op) { + r, ok := ranges[c.Tag] + if !ok { + r = queryRange{key: c.Tag} + } + switch c.Op { + case query.OpGreater: + r.lowerBound = c.Operand + case query.OpGreaterEqual: + r.includeLowerBound = true + r.lowerBound = c.Operand + case query.OpLess: + r.upperBound = c.Operand + case query.OpLessEqual: + r.includeUpperBound = true + r.upperBound = c.Operand + } + ranges[c.Tag] = r + indexes = append(indexes, i) + } + } + return ranges, indexes +} + +func isRangeOperation(op query.Operator) bool { + switch op { + case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: + return true + default: + return false + } +} + +func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) { + if c.Op == query.OpEqual { + it := txi.store.IteratorPrefix(startKey) + for it.Next() { + hashes = append(hashes, it.Value()) + } + } else if c.Op == query.OpContains { + // XXX: full scan + it := txi.store.Iterator() + for it.Next() { + // if it is a hash key, continue + if !strings.Contains(string(it.Key()), "/") { + continue + } + if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { + hashes = append(hashes, it.Value()) + } + } + } else { + panic("other operators should be handled already") + } + return +} + +func startKey(c query.Condition, height uint64, heightSpecified bool) []byte { + var key string + if heightSpecified { + key = fmt.Sprintf("%s/%v/%d", c.Tag, c.Operand, height) + } else { + key = fmt.Sprintf("%s/%v", c.Tag, c.Operand) + } + return []byte(key) +} + +func startKeyForRange(r queryRange, height uint64, heightSpecified bool) []byte { + var lowerBound interface{} + if r.includeLowerBound { + lowerBound = r.lowerBound + } else { + switch t := r.lowerBound.(type) { + case int64: + lowerBound = t + 1 + case time.Time: + lowerBound = t.Unix() + 1 + default: + panic("not implemented") + } + } + var key string + if heightSpecified { + key = fmt.Sprintf("%s/%v/%d", r.key, lowerBound, height) + } else { + key = fmt.Sprintf("%s/%v", r.key, lowerBound) + } + return []byte(key) +} + +func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { + it := txi.store.IteratorPrefix(startKey) + defer it.Release() + for it.Next() { + // no other way to stop iterator other than checking for upperBound + switch (r.upperBound).(type) { + case int64: + v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + if err == nil && v == r.upperBound { + if r.includeUpperBound { + hashes = append(hashes, it.Value()) + } + break + } + // XXX: passing time in a ABCI Tags is not yet implemented + // case time.Time: + // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + // if v == r.upperBound { + // break + // } + } + hashes = append(hashes, it.Value()) + } + return +} + +func extractValueFromKey(key []byte) string { + s := string(key) + parts := strings.SplitN(s, "/", 3) + return parts[1] +} + +func keyForTag(tag *abci.KVPair, result *types.TxResult) []byte { + switch tag.ValueType { + case abci.KVPair_STRING: + return []byte(fmt.Sprintf("%s/%v/%d/%d", tag.Key, tag.ValueString, result.Height, result.Index)) + case abci.KVPair_INT: + return []byte(fmt.Sprintf("%s/%v/%d/%d", tag.Key, tag.ValueInt, result.Height, result.Index)) + // case abci.KVPair_TIME: + // return []byte(fmt.Sprintf("%s/%d/%d/%d", tag.Key, tag.ValueTime.Unix(), result.Height, result.Index)) + default: + panic(fmt.Sprintf("Undefined value type: %v", tag.ValueType)) + } +} + +func hashKey(hash []byte) string { + return fmt.Sprintf("%X", hash) +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +func intInSlice(a int, list []int) bool { + for _, b := range list { + if b == a { + return true + } + } + return false } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index f814fabe..b1f9840e 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -1,6 +1,7 @@ package kv import ( + "fmt" "io/ioutil" "os" "testing" @@ -11,6 +12,7 @@ import ( "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" db "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tmlibs/pubsub/query" ) func TestTxIndex(t *testing.T) { @@ -21,28 +23,89 @@ func TestTxIndex(t *testing.T) { hash := tx.Hash() batch := txindex.NewBatch(1) - if err := batch.Add(*txResult); err != nil { + if err := batch.Add(txResult); err != nil { t.Error(err) } - err := indexer.AddBatch(batch) - require.Nil(t, err) + err := indexer.AddBatch(batch, []string{}) + require.NoError(t, err) loadedTxResult, err := indexer.Get(hash) - require.Nil(t, err) + require.NoError(t, err) assert.Equal(t, txResult, loadedTxResult) tx2 := types.Tx("BYE BYE WORLD") txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} hash2 := tx2.Hash() - err = indexer.Index(txResult2) - require.Nil(t, err) + err = indexer.Index(txResult2, []string{}) + require.NoError(t, err) loadedTxResult2, err := indexer.Get(hash2) - require.Nil(t, err) + require.NoError(t, err) assert.Equal(t, txResult2, loadedTxResult2) } +func TestTxSearch(t *testing.T) { + indexer := &TxIndex{store: db.NewMemDB()} + + tx := types.Tx("HELLO WORLD") + tags := []*abci.KVPair{ + &abci.KVPair{Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 1}, + &abci.KVPair{Key: "account.owner", ValueType: abci.KVPair_STRING, ValueString: "Ivan"}, + &abci.KVPair{Key: "not_allowed", ValueType: abci.KVPair_STRING, ValueString: "Vlad"}, + } + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: tags}} + hash := tx.Hash() + + allowedTags := []string{"account.number", "account.owner", "account.date"} + err := indexer.Index(txResult, allowedTags) + require.NoError(t, err) + + testCases := []struct { + q string + expectError bool + resultsLength int + results []*types.TxResult + }{ + // search by hash + {fmt.Sprintf("tx.hash = '%X'", hash), false, 1, []*types.TxResult{txResult}}, + // search by exact match (one tag) + {"account.number = 1", false, 1, []*types.TxResult{txResult}}, + // search by exact match (two tags) + {"account.number = 1 AND account.owner = 'Ivan'", false, 1, []*types.TxResult{txResult}}, + // search by exact match (two tags) + {"account.number = 1 AND account.owner = 'Vlad'", false, 0, []*types.TxResult{}}, + // search by range + {"account.number >= 1 AND account.number <= 5", false, 1, []*types.TxResult{txResult}}, + // search using not allowed tag + {"not_allowed = 'boom'", false, 0, []*types.TxResult{}}, + // search for not existing tx result + {"account.number >= 2 AND account.number <= 5", false, 0, []*types.TxResult{}}, + // search using not existing tag + {"account.date >= TIME 2013-05-03T14:45:00Z", false, 0, []*types.TxResult{}}, + // search using CONTAINS + {"account.owner CONTAINS 'an'", false, 1, []*types.TxResult{txResult}}, + // search using CONTAINS + {"account.owner CONTAINS 'Vlad'", false, 0, []*types.TxResult{}}, + } + + for _, tc := range testCases { + t.Run(tc.q, func(t *testing.T) { + results, err := indexer.Search(query.MustParse(tc.q)) + if tc.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + assert.Len(t, results, tc.resultsLength) + if tc.resultsLength > 0 { + assert.Equal(t, tc.results, results) + } + }) + } +} + func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} @@ -58,7 +121,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { batch := txindex.NewBatch(txsCount) for i := 0; i < txsCount; i++ { - if err := batch.Add(*txResult); err != nil { + if err := batch.Add(txResult); err != nil { b.Fatal(err) } txResult.Index += 1 @@ -67,7 +130,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { - err = indexer.AddBatch(batch) + err = indexer.AddBatch(batch, []string{}) } if err != nil { b.Fatal(err) diff --git a/state/txindex/null/null.go b/state/txindex/null/null.go index 27e81d73..12f5eb91 100644 --- a/state/txindex/null/null.go +++ b/state/txindex/null/null.go @@ -5,8 +5,11 @@ import ( "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmlibs/pubsub/query" ) +var _ txindex.TxIndexer = (*TxIndex)(nil) + // TxIndex acts as a /dev/null. type TxIndex struct{} @@ -16,11 +19,15 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { } // AddBatch is a noop and always returns nil. -func (txi *TxIndex) AddBatch(batch *txindex.Batch) error { +func (txi *TxIndex) AddBatch(batch *txindex.Batch, allowedTags []string) error { return nil } // Index is a noop and always returns nil. -func (txi *TxIndex) Index(result *types.TxResult) error { +func (txi *TxIndex) Index(result *types.TxResult, allowedTags []string) error { return nil } + +func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { + return []*types.TxResult{}, nil +} diff --git a/types/event_bus.go b/types/event_bus.go index 2e31489c..1a89ef29 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -116,6 +116,16 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { } tags[TxHashKey] = fmt.Sprintf("%X", event.Tx.Hash()) + if tag, ok := tags[TxHeightKey]; ok { + b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) + } + tags[TxHeightKey] = event.Height + + if tag, ok := tags[TxIndexKey]; ok { + b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) + } + tags[TxIndexKey] = event.Index + b.pubsub.PublishWithTags(ctx, TMEventData{event}, tags) return nil } diff --git a/types/events.go b/types/events.go index ed972e93..10df2643 100644 --- a/types/events.go +++ b/types/events.go @@ -140,6 +140,12 @@ const ( // TxHashKey is a reserved key, used to specify transaction's hash. // see EventBus#PublishEventTx TxHashKey = "tx.hash" + // TxHeightKey is a reserved key, used to specify transaction block's height. + // see EventBus#PublishEventTx + TxHeightKey = "tx.height" + // TxIndexKey is a reserved key, used to specify transaction's index within the block. + // see EventBus#PublishEventTx + TxIndexKey = "tx.index" ) var ( From 3e577ccf4f9125bfecdd78f7cf82e12d1af3a8ec Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 28 Nov 2017 14:12:04 -0600 Subject: [PATCH 11/27] add `tx_search` RPC endpoint --- rpc/core/routes.go | 1 + rpc/core/tx.go | 40 ++++++++++++++++++++++++++++++++++++++++ state/txindex/indexer.go | 4 ++++ 3 files changed, 45 insertions(+) diff --git a/rpc/core/routes.go b/rpc/core/routes.go index a4328f1d..2ae352c1 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -19,6 +19,7 @@ var Routes = map[string]*rpc.RPCFunc{ "block": rpc.NewRPCFunc(Block, "height"), "commit": rpc.NewRPCFunc(Commit, "height"), "tx": rpc.NewRPCFunc(Tx, "hash,prove"), + "tx_search": rpc.NewRPCFunc(Tx, "query,prove"), "validators": rpc.NewRPCFunc(Validators, "height"), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxs, ""), diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 80d1cb32..3609c05d 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -6,6 +6,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" + tmquery "github.com/tendermint/tmlibs/pubsub/query" ) // Tx allows you to query the transaction results. `nil` could mean the @@ -99,3 +100,42 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { Proof: proof, }, nil } + +func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { + // if index is disabled, return error + if _, ok := txIndexer.(*null.TxIndex); ok { + return nil, fmt.Errorf("Transaction indexing is disabled.") + } + + q, err := tmquery.New(query) + if err != nil { + return []*ctypes.ResultTx{}, err + } + + results, err := txIndexer.Search(q) + if err != nil { + return []*ctypes.ResultTx{}, err + } + + apiResults := make([]*ctypes.ResultTx, len(results)) + for i, r := range results { + height := r.Height + index := r.Index + + var proof types.TxProof + if prove { + block := blockStore.LoadBlock(int(height)) + proof = block.Data.Txs.Proof(int(index)) + } + + apiResults[i] = &ctypes.ResultTx{ + Height: height, + Index: index, + TxResult: r.Result, + Tx: r.Tx, + Proof: proof, + } + } + + return apiResults, nil +} diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index f9908f32..07a544bd 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmlibs/pubsub/query" ) // TxIndexer interface defines methods to index and search transactions. @@ -18,6 +19,9 @@ type TxIndexer interface { // Get returns the transaction specified by hash or nil if the transaction is not indexed // or stored. Get(hash []byte) (*types.TxResult, error) + + // Search allows you to query for transactions. + Search(q *query.Query) ([]*types.TxResult, error) } //---------------------------------------------------- From 91f218400356007176155c4d6b8180a9bf480c83 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 28 Nov 2017 18:58:39 -0600 Subject: [PATCH 12/27] fixes after bucky's review --- config/config.go | 10 +- node/node.go | 4 +- rpc/core/tx.go | 10 +- state/txindex/indexer.go | 4 +- state/txindex/kv/kv.go | 185 ++++++++++++++++++------------------ state/txindex/kv/kv_test.go | 52 +++++----- state/txindex/null/null.go | 4 +- types/event_bus.go | 23 ++--- types/events.go | 3 - 9 files changed, 149 insertions(+), 146 deletions(-) diff --git a/config/config.go b/config/config.go index 8b1c0e28..fc3671d8 100644 --- a/config/config.go +++ b/config/config.go @@ -418,9 +418,17 @@ func (c *ConsensusConfig) SetWalFile(walFile string) { // indexer, including tags to index. type TxIndexConfig struct { // What indexer to use for transactions + // + // Options: + // 1) "null" (default) + // 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). Indexer string `mapstructure:"indexer"` - // Comma-separated list of tags to index (by default only by tx hash) + // Comma-separated list of tags to index (by default the only tag is tx hash) + // + // It's recommended to index only a subset of tags due to possible memory + // bloat. This is, of course, depends on the indexer's DB and the volume of + // transactions. IndexTags string `mapstructure:"index_tags"` } diff --git a/node/node.go b/node/node.go index fff550bf..57fbfbf2 100644 --- a/node/node.go +++ b/node/node.go @@ -287,7 +287,7 @@ func NewNode(config *cfg.Config, if err != nil { return nil, err } - txIndexer = kv.NewTxIndex(store) + txIndexer = kv.NewTxIndex(store, strings.Split(config.TxIndex.IndexTags, ",")) default: txIndexer = &null.TxIndex{} } @@ -299,7 +299,7 @@ func NewNode(config *cfg.Config, for event := range ch { // XXX: may be not perfomant to write one event at a time txResult := event.(types.TMEventData).Unwrap().(types.EventDataTx).TxResult - txIndexer.Index(&txResult, strings.Split(config.TxIndex.IndexTags, ",")) + txIndexer.Index(&txResult) } }() diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 3609c05d..20fc2c96 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -88,6 +88,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { var proof types.TxProof if prove { + // TODO: handle overflow block := blockStore.LoadBlock(int(height)) proof = block.Data.Txs.Proof(int(index)) } @@ -109,21 +110,24 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { q, err := tmquery.New(query) if err != nil { - return []*ctypes.ResultTx{}, err + return nil, err } results, err := txIndexer.Search(q) if err != nil { - return []*ctypes.ResultTx{}, err + return nil, err } + // TODO: we may want to consider putting a maximum on this length and somehow + // informing the user that things were truncated. apiResults := make([]*ctypes.ResultTx, len(results)) + var proof types.TxProof for i, r := range results { height := r.Height index := r.Index - var proof types.TxProof if prove { + // TODO: handle overflow block := blockStore.LoadBlock(int(height)) proof = block.Data.Txs.Proof(int(index)) } diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index 07a544bd..bd51fbb2 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -11,10 +11,10 @@ import ( type TxIndexer interface { // AddBatch analyzes, indexes and stores a batch of transactions. - AddBatch(b *Batch, allowedTags []string) error + AddBatch(b *Batch) error // Index analyzes, indexes and stores a single transaction. - Index(result *types.TxResult, allowedTags []string) error + Index(result *types.TxResult) error // Get returns the transaction specified by hash or nil if the transaction is not indexed // or stored. diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index ee81674b..d77711ed 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -14,21 +14,26 @@ import ( wire "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" db "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/pubsub/query" ) +const ( + tagKeySeparator = "/" +) + var _ txindex.TxIndexer = (*TxIndex)(nil) -// TxIndex is the simplest possible indexer, backed by Key-Value storage (levelDB). -// It can only index transaction by its identifier. +// TxIndex is the simplest possible indexer, backed by key-value storage (levelDB). type TxIndex struct { - store db.DB + store db.DB + tagsToIndex []string } -// NewTxIndex returns new instance of TxIndex. -func NewTxIndex(store db.DB) *TxIndex { - return &TxIndex{store: store} +// NewTxIndex creates new KV indexer. +func NewTxIndex(store db.DB, tagsToIndex []string) *TxIndex { + return &TxIndex{store: store, tagsToIndex: tagsToIndex} } // Get gets transaction from the TxIndex storage and returns it or nil if the @@ -55,7 +60,7 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { } // AddBatch indexes a batch of transactions using the given list of tags. -func (txi *TxIndex) AddBatch(b *txindex.Batch, allowedTags []string) error { +func (txi *TxIndex) AddBatch(b *txindex.Batch) error { storeBatch := txi.store.NewBatch() for _, result := range b.Ops { @@ -63,7 +68,7 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch, allowedTags []string) error { // index tx by tags for _, tag := range result.Result.Tags { - if stringInSlice(tag.Key, allowedTags) { + if stringInSlice(tag.Key, txi.tagsToIndex) { storeBatch.Set(keyForTag(tag, result), hash) } } @@ -78,14 +83,21 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch, allowedTags []string) error { } // Index indexes a single transaction using the given list of tags. -func (txi *TxIndex) Index(result *types.TxResult, allowedTags []string) error { +func (txi *TxIndex) Index(result *types.TxResult) error { batch := txindex.NewBatch(1) batch.Add(result) - return txi.AddBatch(batch, allowedTags) + return txi.AddBatch(batch) } +// Search performs a search using the given query. It breaks the query into +// conditions (like "tx.height > 5"). For each condition, it queries the DB +// index. One special use cases here: (1) if "tx.hash" is found, it returns tx +// result for it (2) for range queries it is better for the client to provide +// both lower and upper bounds, so we are not performing a full scan. Results +// from querying indexes are then intersected and returned to the caller. func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { - hashes := make(map[string][]byte) // key - (base 16, upper-case hash) + var hashes [][]byte + var hashesInitialized bool // get a list of conditions (like "tx.height > 5") conditions := q.Conditions() @@ -93,13 +105,13 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { // if there is a hash condition, return the result immediately hash, err, ok := lookForHash(conditions) if err != nil { - return []*types.TxResult{}, errors.Wrap(err, "error during searching for a hash in the query") + return nil, errors.Wrap(err, "error during searching for a hash in the query") } else if ok { res, err := txi.Get(hash) return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } - // conditions to skip + // conditions to skip because they're handled before "everything else" skipIndexes := make([]int, 0) // if there is a height condition ("tx.height=3"), extract it for faster lookups @@ -108,36 +120,19 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { skipIndexes = append(skipIndexes, heightIndex) } - var hashes2 [][]byte - // extract ranges // if both upper and lower bounds exist, it's better to get them in order not // no iterate over kvs that are not within range. ranges, rangeIndexes := lookForRanges(conditions) if len(ranges) > 0 { skipIndexes = append(skipIndexes, rangeIndexes...) - } - for _, r := range ranges { - hashes2 = txi.matchRange(r, startKeyForRange(r, height, heightIndex > 0)) - // initialize hashes if we're running the first time - if len(hashes) == 0 { - for _, h := range hashes2 { - hashes[hashKey(h)] = h - } - continue - } - - // no matches - if len(hashes2) == 0 { - hashes = make(map[string][]byte) - } else { - // perform intersection as we go - for _, h := range hashes2 { - k := hashKey(h) - if _, ok := hashes[k]; !ok { - delete(hashes, k) - } + for _, r := range ranges { + if !hashesInitialized { + hashes = txi.matchRange(r, startKeyForRange(r, height)) + hashesInitialized = true + } else { + hashes = intersect(hashes, txi.matchRange(r, startKeyForRange(r, height))) } } } @@ -148,27 +143,11 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { continue } - hashes2 = txi.match(c, startKey(c, height, heightIndex > 0)) - - // initialize hashes if we're running the first time - if len(hashes) == 0 { - for _, h := range hashes2 { - hashes[hashKey(h)] = h - } - continue - } - - // no matches - if len(hashes2) == 0 { - hashes = make(map[string][]byte) + if !hashesInitialized { + hashes = txi.match(c, startKey(c, height)) + hashesInitialized = true } else { - // perform intersection as we go - for _, h := range hashes2 { - k := hashKey(h) - if _, ok := hashes[k]; !ok { - delete(hashes, k) - } - } + hashes = intersect(hashes, txi.match(c, startKey(c, height))) } } @@ -177,7 +156,7 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { for _, h := range hashes { results[i], err = txi.Get(h) if err != nil { - return []*types.TxResult{}, errors.Wrapf(err, "failed to get Tx{%X}", h) + return nil, errors.Wrapf(err, "failed to get Tx{%X}", h) } i++ } @@ -253,15 +232,16 @@ func isRangeOperation(op query.Operator) bool { func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) { if c.Op == query.OpEqual { it := txi.store.IteratorPrefix(startKey) + defer it.Release() for it.Next() { hashes = append(hashes, it.Value()) } } else if c.Op == query.OpContains { - // XXX: full scan + // XXX: doing full scan because startKey does not apply here it := txi.store.Iterator() + defer it.Release() for it.Next() { - // if it is a hash key, continue - if !strings.Contains(string(it.Key()), "/") { + if !isTagKey(it.Key()) { continue } if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { @@ -274,9 +254,42 @@ func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) return } -func startKey(c query.Condition, height uint64, heightSpecified bool) []byte { +func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { + it := txi.store.IteratorPrefix(startKey) + defer it.Release() +LOOP: + for it.Next() { + if !isTagKey(it.Key()) { + continue + } + // no other way to stop iterator other than checking for upperBound + switch (r.upperBound).(type) { + case int64: + v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + if err == nil && v == r.upperBound { + if r.includeUpperBound { + hashes = append(hashes, it.Value()) + } + break LOOP + } + // XXX: passing time in a ABCI Tags is not yet implemented + // case time.Time: + // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + // if v == r.upperBound { + // break + // } + } + hashes = append(hashes, it.Value()) + } + return +} + +/////////////////////////////////////////////////////////////////////////////// +// Keys + +func startKey(c query.Condition, height uint64) []byte { var key string - if heightSpecified { + if height > 0 { key = fmt.Sprintf("%s/%v/%d", c.Tag, c.Operand, height) } else { key = fmt.Sprintf("%s/%v", c.Tag, c.Operand) @@ -284,7 +297,7 @@ func startKey(c query.Condition, height uint64, heightSpecified bool) []byte { return []byte(key) } -func startKeyForRange(r queryRange, height uint64, heightSpecified bool) []byte { +func startKeyForRange(r queryRange, height uint64) []byte { var lowerBound interface{} if r.includeLowerBound { lowerBound = r.lowerBound @@ -299,7 +312,7 @@ func startKeyForRange(r queryRange, height uint64, heightSpecified bool) []byte } } var key string - if heightSpecified { + if height > 0 { key = fmt.Sprintf("%s/%v/%d", r.key, lowerBound, height) } else { key = fmt.Sprintf("%s/%v", r.key, lowerBound) @@ -307,35 +320,12 @@ func startKeyForRange(r queryRange, height uint64, heightSpecified bool) []byte return []byte(key) } -func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { - it := txi.store.IteratorPrefix(startKey) - defer it.Release() - for it.Next() { - // no other way to stop iterator other than checking for upperBound - switch (r.upperBound).(type) { - case int64: - v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - if err == nil && v == r.upperBound { - if r.includeUpperBound { - hashes = append(hashes, it.Value()) - } - break - } - // XXX: passing time in a ABCI Tags is not yet implemented - // case time.Time: - // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - // if v == r.upperBound { - // break - // } - } - hashes = append(hashes, it.Value()) - } - return +func isTagKey(key []byte) bool { + return strings.Count(string(key), tagKeySeparator) == 3 } func extractValueFromKey(key []byte) string { - s := string(key) - parts := strings.SplitN(s, "/", 3) + parts := strings.SplitN(string(key), tagKeySeparator, 3) return parts[1] } @@ -356,6 +346,9 @@ func hashKey(hash []byte) string { return fmt.Sprintf("%X", hash) } +/////////////////////////////////////////////////////////////////////////////// +// Utils + func stringInSlice(a string, list []string) bool { for _, b := range list { if b == a { @@ -373,3 +366,15 @@ func intInSlice(a int, list []int) bool { } return false } + +func intersect(as, bs [][]byte) [][]byte { + i := make([][]byte, 0, cmn.MinInt(len(as), len(bs))) + for _, a := range as { + for _, b := range bs { + if bytes.Equal(a, b) { + i = append(i, a) + } + } + } + return i +} diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index b1f9840e..a51bb4bf 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -16,7 +16,7 @@ import ( ) func TestTxIndex(t *testing.T) { - indexer := &TxIndex{store: db.NewMemDB()} + indexer := NewTxIndex(db.NewMemDB(), []string{}) tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} @@ -26,7 +26,7 @@ func TestTxIndex(t *testing.T) { if err := batch.Add(txResult); err != nil { t.Error(err) } - err := indexer.AddBatch(batch, []string{}) + err := indexer.AddBatch(batch) require.NoError(t, err) loadedTxResult, err := indexer.Get(hash) @@ -37,7 +37,7 @@ func TestTxIndex(t *testing.T) { txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} hash2 := tx2.Hash() - err = indexer.Index(txResult2, []string{}) + err = indexer.Index(txResult2) require.NoError(t, err) loadedTxResult2, err := indexer.Get(hash2) @@ -46,61 +46,55 @@ func TestTxIndex(t *testing.T) { } func TestTxSearch(t *testing.T) { - indexer := &TxIndex{store: db.NewMemDB()} + tagsToIndex := []string{"account.number", "account.owner", "account.date"} + indexer := NewTxIndex(db.NewMemDB(), tagsToIndex) tx := types.Tx("HELLO WORLD") tags := []*abci.KVPair{ - &abci.KVPair{Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 1}, - &abci.KVPair{Key: "account.owner", ValueType: abci.KVPair_STRING, ValueString: "Ivan"}, - &abci.KVPair{Key: "not_allowed", ValueType: abci.KVPair_STRING, ValueString: "Vlad"}, + {Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 1}, + {Key: "account.owner", ValueType: abci.KVPair_STRING, ValueString: "Ivan"}, + {Key: "not_allowed", ValueType: abci.KVPair_STRING, ValueString: "Vlad"}, } txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: tags}} hash := tx.Hash() - allowedTags := []string{"account.number", "account.owner", "account.date"} - err := indexer.Index(txResult, allowedTags) + err := indexer.Index(txResult) require.NoError(t, err) testCases := []struct { q string - expectError bool resultsLength int - results []*types.TxResult }{ // search by hash - {fmt.Sprintf("tx.hash = '%X'", hash), false, 1, []*types.TxResult{txResult}}, + {fmt.Sprintf("tx.hash = '%X'", hash), 1}, // search by exact match (one tag) - {"account.number = 1", false, 1, []*types.TxResult{txResult}}, + {"account.number = 1", 1}, // search by exact match (two tags) - {"account.number = 1 AND account.owner = 'Ivan'", false, 1, []*types.TxResult{txResult}}, + {"account.number = 1 AND account.owner = 'Ivan'", 1}, // search by exact match (two tags) - {"account.number = 1 AND account.owner = 'Vlad'", false, 0, []*types.TxResult{}}, + {"account.number = 1 AND account.owner = 'Vlad'", 0}, // search by range - {"account.number >= 1 AND account.number <= 5", false, 1, []*types.TxResult{txResult}}, + {"account.number >= 1 AND account.number <= 5", 1}, // search using not allowed tag - {"not_allowed = 'boom'", false, 0, []*types.TxResult{}}, + {"not_allowed = 'boom'", 0}, // search for not existing tx result - {"account.number >= 2 AND account.number <= 5", false, 0, []*types.TxResult{}}, + {"account.number >= 2 AND account.number <= 5", 0}, // search using not existing tag - {"account.date >= TIME 2013-05-03T14:45:00Z", false, 0, []*types.TxResult{}}, + {"account.date >= TIME 2013-05-03T14:45:00Z", 0}, // search using CONTAINS - {"account.owner CONTAINS 'an'", false, 1, []*types.TxResult{txResult}}, + {"account.owner CONTAINS 'an'", 1}, // search using CONTAINS - {"account.owner CONTAINS 'Vlad'", false, 0, []*types.TxResult{}}, + {"account.owner CONTAINS 'Vlad'", 0}, } for _, tc := range testCases { t.Run(tc.q, func(t *testing.T) { results, err := indexer.Search(query.MustParse(tc.q)) - if tc.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } + assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) if tc.resultsLength > 0 { - assert.Equal(t, tc.results, results) + assert.Equal(t, []*types.TxResult{txResult}, results) } }) } @@ -117,7 +111,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { defer os.RemoveAll(dir) // nolint: errcheck store := db.NewDB("tx_index", "leveldb", dir) - indexer := &TxIndex{store: store} + indexer := NewTxIndex(store, []string{}) batch := txindex.NewBatch(txsCount) for i := 0; i < txsCount; i++ { @@ -130,7 +124,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { - err = indexer.AddBatch(batch, []string{}) + err = indexer.AddBatch(batch) } if err != nil { b.Fatal(err) diff --git a/state/txindex/null/null.go b/state/txindex/null/null.go index 12f5eb91..0764faa9 100644 --- a/state/txindex/null/null.go +++ b/state/txindex/null/null.go @@ -19,12 +19,12 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { } // AddBatch is a noop and always returns nil. -func (txi *TxIndex) AddBatch(batch *txindex.Batch, allowedTags []string) error { +func (txi *TxIndex) AddBatch(batch *txindex.Batch) error { return nil } // Index is a noop and always returns nil. -func (txi *TxIndex) Index(result *types.TxResult, allowedTags []string) error { +func (txi *TxIndex) Index(result *types.TxResult) error { return nil } diff --git a/types/event_bus.go b/types/event_bus.go index 1a89ef29..6cee1d82 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -106,26 +106,15 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { } // add predefined tags - if tag, ok := tags[EventTypeKey]; ok { - b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) - } + logIfTagExists(EventTypeKey, tags, b.Logger) tags[EventTypeKey] = EventTx - if tag, ok := tags[TxHashKey]; ok { - b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) - } + logIfTagExists(TxHashKey, tags, b.Logger) tags[TxHashKey] = fmt.Sprintf("%X", event.Tx.Hash()) - if tag, ok := tags[TxHeightKey]; ok { - b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) - } + logIfTagExists(TxHeightKey, tags, b.Logger) tags[TxHeightKey] = event.Height - if tag, ok := tags[TxIndexKey]; ok { - b.Logger.Error("Found predefined tag (value will be overwritten)", "tag", tag) - } - tags[TxIndexKey] = event.Index - b.pubsub.PublishWithTags(ctx, TMEventData{event}, tags) return nil } @@ -171,3 +160,9 @@ func (b *EventBus) PublishEventRelock(event EventDataRoundState) error { func (b *EventBus) PublishEventLock(event EventDataRoundState) error { return b.Publish(EventLock, TMEventData{event}) } + +func logIfTagExists(tag string, tags map[string]interface{}, logger log.Logger) { + if value, ok := tags[tag]; ok { + logger.Error("Found predefined tag (value will be overwritten)", "tag", tag, "value", value) + } +} diff --git a/types/events.go b/types/events.go index 10df2643..9bf7a5a4 100644 --- a/types/events.go +++ b/types/events.go @@ -143,9 +143,6 @@ const ( // TxHeightKey is a reserved key, used to specify transaction block's height. // see EventBus#PublishEventTx TxHeightKey = "tx.height" - // TxIndexKey is a reserved key, used to specify transaction's index within the block. - // see EventBus#PublishEventTx - TxIndexKey = "tx.index" ) var ( From 686e0eea9fbb1e1e8add7369fc79be9ab9862f36 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 28 Nov 2017 21:24:37 -0600 Subject: [PATCH 13/27] extract indexing goroutine to a separate indexer service --- node/node.go | 21 +++++++------- state/txindex/indexer_service.go | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 state/txindex/indexer_service.go diff --git a/node/node.go b/node/node.go index 57fbfbf2..ea668f8f 100644 --- a/node/node.go +++ b/node/node.go @@ -111,6 +111,7 @@ type Node struct { proxyApp proxy.AppConns // connection to the application rpcListeners []net.Listener // rpc servers txIndexer txindex.TxIndexer + indexerService *txindex.IndexerService } // NewNode returns a new, ready to go, Tendermint Node. @@ -292,16 +293,7 @@ func NewNode(config *cfg.Config, txIndexer = &null.TxIndex{} } - // subscribe for all transactions and index them by tags - ch := make(chan interface{}) - eventBus.Subscribe(context.Background(), "tx_index", types.EventQueryTx, ch) - go func() { - for event := range ch { - // XXX: may be not perfomant to write one event at a time - txResult := event.(types.TMEventData).Unwrap().(types.EventDataTx).TxResult - txIndexer.Index(&txResult) - } - }() + indexerService := txindex.NewIndexerService(txIndexer, eventBus) // run the profile server profileHost := config.ProfListenAddress @@ -328,6 +320,7 @@ func NewNode(config *cfg.Config, consensusReactor: consensusReactor, proxyApp: proxyApp, txIndexer: txIndexer, + indexerService: indexerService, eventBus: eventBus, } node.BaseService = *cmn.NewBaseService(logger, "Node", node) @@ -373,6 +366,12 @@ func (n *Node) OnStart() error { } } + // start tx indexer + _, err = n.indexerService.Start() + if err != nil { + return err + } + return nil } @@ -392,6 +391,8 @@ func (n *Node) OnStop() { } n.eventBus.Stop() + + n.indexerService.Stop() } // RunForever waits for an interrupt signal and stops the node. diff --git a/state/txindex/indexer_service.go b/state/txindex/indexer_service.go new file mode 100644 index 00000000..80f12fd3 --- /dev/null +++ b/state/txindex/indexer_service.go @@ -0,0 +1,48 @@ +package txindex + +import ( + "context" + + "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" +) + +const ( + subscriber = "IndexerService" +) + +type IndexerService struct { + cmn.BaseService + + idr TxIndexer + eventBus *types.EventBus +} + +func NewIndexerService(idr TxIndexer, eventBus *types.EventBus) *IndexerService { + is := &IndexerService{idr: idr, eventBus: eventBus} + is.BaseService = *cmn.NewBaseService(nil, "IndexerService", is) + return is +} + +// OnStart implements cmn.Service by subscribing for all transactions +// and indexing them by tags. +func (is *IndexerService) OnStart() error { + ch := make(chan interface{}) + if err := is.eventBus.Subscribe(context.Background(), subscriber, types.EventQueryTx, ch); err != nil { + return err + } + go func() { + for event := range ch { + // TODO: may be not perfomant to write one event at a time + txResult := event.(types.TMEventData).Unwrap().(types.EventDataTx).TxResult + is.idr.Index(&txResult) + } + }() + return nil +} + +func (is *IndexerService) OnStop() { + if is.eventBus.IsRunning() { + _ = is.eventBus.UnsubscribeAll(context.Background(), subscriber) + } +} From 2a5e8c4a4749f3dbd4d5c00799a4c3e8fb82557c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 28 Nov 2017 21:32:40 -0600 Subject: [PATCH 14/27] add minimal documentation for tx_search RPC method [ci skip] --- rpc/core/tx.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 20fc2c96..4e4285a2 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -102,6 +102,39 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { }, nil } +// TxSearch allows you to query for multiple transactions results. +// +// ```shell +// curl "localhost:46657/tx_search?query='account.owner=\'Ivan\''&prove=true" +// ``` +// +// ```go +// client := client.NewHTTP("tcp://0.0.0.0:46657", "/websocket") +// q, err := tmquery.New("account.owner='Ivan'") +// tx, err := client.TxSearch(q, true) +// ``` +// +// > The above command returns JSON structured like this: +// +// ```json +// ``` +// +// Returns transactions matching the given query. +// +// ### Query Parameters +// +// | Parameter | Type | Default | Required | Description | +// |-----------+--------+---------+----------+-----------------------------------------------------------| +// | query | string | "" | true | Query | +// | prove | bool | false | false | Include proofs of the transactions inclusion in the block | +// +// ### Returns +// +// - `proof`: the `types.TxProof` object +// - `tx`: `[]byte` - the transaction +// - `tx_result`: the `abci.Result` object +// - `index`: `int` - index of the transaction +// - `height`: `int` - height of the block where this transaction was in func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { From 09941b9aa9e4bc72fc2148a1340588234464776b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 28 Nov 2017 21:38:16 -0600 Subject: [PATCH 15/27] fix metalinter warnings --- state/txindex/kv/kv.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index d77711ed..ae320cc1 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -85,7 +85,10 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // Index indexes a single transaction using the given list of tags. func (txi *TxIndex) Index(result *types.TxResult) error { batch := txindex.NewBatch(1) - batch.Add(result) + err := batch.Add(result) + if err != nil { + return errors.Wrap(err, "failed to add tx result to batch") + } return txi.AddBatch(batch) } @@ -342,10 +345,6 @@ func keyForTag(tag *abci.KVPair, result *types.TxResult) []byte { } } -func hashKey(hash []byte) string { - return fmt.Sprintf("%X", hash) -} - /////////////////////////////////////////////////////////////////////////////// // Utils From 1e198605852fac70b65a74ed9e6f4f64c4ff2c29 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 11:22:52 -0600 Subject: [PATCH 16/27] fixes from my own review --- consensus/replay.go | 4 ++-- consensus/replay_test.go | 4 ++-- glide.lock | 8 +++++--- glide.yaml | 2 +- mempool/mempool_test.go | 3 ++- proxy/app_conn.go | 14 +++++++------- rpc/core/mempool.go | 6 +----- state/execution.go | 4 ++-- state/txindex/indexer_service.go | 1 + state/txindex/kv/kv.go | 4 ++++ 10 files changed, 27 insertions(+), 23 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 853d3a8d..da68df51 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -236,7 +236,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p // If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain if appBlockHeight == 0 { validators := types.TM2PB.Validators(h.state.Validators) - if err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil { + if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil { return nil, err } } @@ -391,7 +391,7 @@ func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx { return *r } -func (mock *mockProxyApp) EndBlock(height uint64) abci.ResponseEndBlock { +func (mock *mockProxyApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock { mock.txCount = 0 return *mock.abciResponses.EndBlock } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 381c9021..25fdf4db 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -411,7 +411,7 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, } validators := types.TM2PB.Validators(state.Validators) - if err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil { + if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil { panic(err) } @@ -447,7 +447,7 @@ func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.B defer proxyApp.Stop() validators := types.TM2PB.Validators(state.Validators) - if err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil { + if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}); err != nil { panic(err) } diff --git a/glide.lock b/glide.lock index 31f1aaa9..18a5d6a7 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: e279cca35a5cc9a68bb266015dc6a57da749b28dabca3994b2c5dbe02309f470 -updated: 2017-11-28T00:53:04.816567531Z +hash: ffe610ffb74c1ea5cbe8da5d0d3ae30d2640c7426fe9a889a60218ea36daaf53 +updated: 2017-11-29T17:21:18.25916493Z imports: - name: github.com/btcsuite/btcd version: 8cea3866d0f7fb12d567a20744942c0d078c7d15 @@ -98,7 +98,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 2cfad8523a54d64271d7cbc69a39433eab918aa0 + version: 5c29adc081795b04f9d046fb51d76903c22cfa6d subpackages: - client - example/counter @@ -160,6 +160,8 @@ imports: - trace - name: golang.org/x/sys version: b98136db334ff9cb24f28a68e3be3cb6608f7630 + subpackages: + - unix - name: golang.org/x/text version: 88f656faf3f37f690df1a32515b479415e1a6769 subpackages: diff --git a/glide.yaml b/glide.yaml index a20e76db..62c06fc9 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: 2cfad8523a54d64271d7cbc69a39433eab918aa0 + version: 5c29adc081795b04f9d046fb51d76903c22cfa6d subpackages: - client - example/dummy diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index aa19e380..e26ef966 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/abci/example/counter" "github.com/tendermint/abci/example/dummy" + abci "github.com/tendermint/abci/types" "github.com/tendermint/tmlibs/log" cfg "github.com/tendermint/tendermint/config" @@ -115,7 +116,7 @@ func TestTxsAvailable(t *testing.T) { func TestSerialReap(t *testing.T) { app := counter.NewCounterApplication(true) - app.SetOption("serial", "on") + app.SetOption(abci.RequestSetOption{"serial", "on"}) cc := proxy.NewLocalClientCreator(app) mempool := newMempoolWithApp(cc) diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 49c88a37..2319fed8 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -12,11 +12,11 @@ type AppConnConsensus interface { SetResponseCallback(abcicli.Callback) Error() error - InitChainSync(types.RequestInitChain) error + InitChainSync(types.RequestInitChain) (*types.ResponseInitChain, error) - BeginBlockSync(types.RequestBeginBlock) error + BeginBlockSync(types.RequestBeginBlock) (*types.ResponseBeginBlock, error) DeliverTxAsync(tx []byte) *abcicli.ReqRes - EndBlockSync(height uint64) (*types.ResponseEndBlock, error) + EndBlockSync(types.RequestEndBlock) (*types.ResponseEndBlock, error) CommitSync() (*types.ResponseCommit, error) } @@ -61,11 +61,11 @@ func (app *appConnConsensus) Error() error { return app.appConn.Error() } -func (app *appConnConsensus) InitChainSync(req types.RequestInitChain) error { +func (app *appConnConsensus) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) { return app.appConn.InitChainSync(req) } -func (app *appConnConsensus) BeginBlockSync(req types.RequestBeginBlock) error { +func (app *appConnConsensus) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { return app.appConn.BeginBlockSync(req) } @@ -73,8 +73,8 @@ func (app *appConnConsensus) DeliverTxAsync(tx []byte) *abcicli.ReqRes { return app.appConn.DeliverTxAsync(tx) } -func (app *appConnConsensus) EndBlockSync(height uint64) (*types.ResponseEndBlock, error) { - return app.appConn.EndBlockSync(height) +func (app *appConnConsensus) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) { + return app.appConn.EndBlockSync(req) } func (app *appConnConsensus) CommitSync() (*types.ResponseCommit, error) { diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 857ea75b..c2e5d2f9 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -191,11 +191,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { case deliverTxResMsg := <-deliverTxResCh: deliverTxRes := deliverTxResMsg.(types.TMEventData).Unwrap().(types.EventDataTx) // The tx was included in a block. - deliverTxR := abci.ResponseDeliverTx{ - Code: deliverTxRes.Result.Code, - Data: deliverTxRes.Result.Data, - Log: deliverTxRes.Result.Log, - } + deliverTxR := deliverTxRes.Result logger.Info("DeliverTx passed ", "tx", data.Bytes(tx), "response", deliverTxR) return &ctypes.ResultBroadcastTxCommit{ CheckTx: *checkTxR, diff --git a/state/execution.go b/state/execution.go index 5b324eff..3622a663 100644 --- a/state/execution.go +++ b/state/execution.go @@ -77,7 +77,7 @@ func execBlockOnProxyApp(txEventPublisher types.TxEventPublisher, proxyAppConn p proxyAppConn.SetResponseCallback(proxyCb) // Begin block - err := proxyAppConn.BeginBlockSync(abci.RequestBeginBlock{ + _, err := proxyAppConn.BeginBlockSync(abci.RequestBeginBlock{ block.Hash(), types.TM2PB.Header(block.Header), }) @@ -95,7 +95,7 @@ func execBlockOnProxyApp(txEventPublisher types.TxEventPublisher, proxyAppConn p } // End block - abciResponses.EndBlock, err = proxyAppConn.EndBlockSync(uint64(block.Height)) + abciResponses.EndBlock, err = proxyAppConn.EndBlockSync(abci.RequestEndBlock{uint64(block.Height)}) if err != nil { logger.Error("Error in proxyAppConn.EndBlock", "err", err) return nil, err diff --git a/state/txindex/indexer_service.go b/state/txindex/indexer_service.go index 80f12fd3..3e5fab12 100644 --- a/state/txindex/indexer_service.go +++ b/state/txindex/indexer_service.go @@ -41,6 +41,7 @@ func (is *IndexerService) OnStart() error { return nil } +// OnStop implements cmn.Service by unsubscribing from all transactions. func (is *IndexerService) OnStop() { if is.eventBus.IsRunning() { _ = is.eventBus.UnsubscribeAll(context.Background(), subscriber) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index ae320cc1..53d07325 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -186,6 +186,8 @@ func lookForHeight(conditions []query.Condition) (height uint64, index int) { return 0, -1 } +// special map to hold range conditions +// Example: account.number => queryRange{lowerBound: 1, upperBound: 5} type queryRanges map[string]queryRange type queryRange struct { @@ -241,6 +243,8 @@ func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) } } else if c.Op == query.OpContains { // XXX: doing full scan because startKey does not apply here + // For example, if startKey = "account.owner=an" and search query = "accoutn.owner CONSISTS an" + // we can't iterate with prefix "account.owner=an" because we might miss keys like "account.owner=Ulan" it := txi.store.Iterator() defer it.Release() for it.Next() { From acbc0717d4206514e4094468d710ed2fe2358aaa Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 13:42:11 -0600 Subject: [PATCH 17/27] add client methods --- rpc/client/httpclient.go | 17 ++++++++++++++-- rpc/client/interface.go | 1 + rpc/client/localclient.go | 4 ++++ rpc/client/rpc_test.go | 41 +++++++++++++++++++++++++++++++++++++++ rpc/core/routes.go | 2 +- state/txindex/kv/kv.go | 6 +++++- 6 files changed, 67 insertions(+), 4 deletions(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 47c99fd3..5ceace97 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -163,17 +163,30 @@ func (c *HTTP) Commit(height *int) (*ctypes.ResultCommit, error) { func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { result := new(ctypes.ResultTx) - query := map[string]interface{}{ + params := map[string]interface{}{ "hash": hash, "prove": prove, } - _, err := c.rpc.Call("tx", query, result) + _, err := c.rpc.Call("tx", params, result) if err != nil { return nil, errors.Wrap(err, "Tx") } return result, nil } +func (c *HTTP) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { + results := new([]*ctypes.ResultTx) + params := map[string]interface{}{ + "query": query, + "prove": prove, + } + _, err := c.rpc.Call("tx_search", params, results) + if err != nil { + return nil, errors.Wrap(err, "TxSearch") + } + return *results, nil +} + func (c *HTTP) Validators(height *int) (*ctypes.ResultValidators, error) { result := new(ctypes.ResultValidators) _, err := c.rpc.Call("validators", map[string]interface{}{"height": height}, result) diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 443ea89d..c0d7e052 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -50,6 +50,7 @@ type SignClient interface { Commit(height *int) (*ctypes.ResultCommit, error) Validators(height *int) (*ctypes.ResultValidators, error) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) + TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) } // HistoryClient shows us data from genesis to now in large chunks. diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 55a0e0fb..d5444007 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -124,6 +124,10 @@ func (Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return core.Tx(hash, prove) } +func (Local) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { + return core.TxSearch(query, prove) +} + func (c *Local) Subscribe(ctx context.Context, query string, out chan<- interface{}) error { q, err := tmquery.New(query) if err != nil { diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index b6b3d9e2..6eab5b85 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -1,6 +1,7 @@ package client_test import ( + "fmt" "strings" "testing" @@ -294,3 +295,43 @@ func TestTx(t *testing.T) { } } } + +func TestTxSearch(t *testing.T) { + // first we broadcast a tx + c := getHTTPClient() + _, _, tx := MakeTxKV() + bres, err := c.BroadcastTxCommit(tx) + require.Nil(t, err, "%+v", err) + + txHeight := bres.Height + txHash := bres.Hash + + anotherTxHash := types.Tx("a different tx").Hash() + + for i, c := range GetClients() { + t.Logf("client %d", i) + + // now we query for the tx. + // since there's only one tx, we know index=0. + results, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true) + require.Nil(t, err, "%+v", err) + require.Len(t, results, 1) + + ptx := results[0] + assert.EqualValues(t, txHeight, ptx.Height) + assert.EqualValues(t, tx, ptx.Tx) + assert.Zero(t, ptx.Index) + assert.True(t, ptx.TxResult.Code.IsOK()) + + // time to verify the proof + proof := ptx.Proof + if assert.EqualValues(t, tx, proof.Data) { + assert.True(t, proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) + } + + // we query for non existing tx + results, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false) + require.Nil(t, err, "%+v", err) + require.Len(t, results, 0) + } +} diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 2ae352c1..111c010a 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -19,7 +19,7 @@ var Routes = map[string]*rpc.RPCFunc{ "block": rpc.NewRPCFunc(Block, "height"), "commit": rpc.NewRPCFunc(Commit, "height"), "tx": rpc.NewRPCFunc(Tx, "hash,prove"), - "tx_search": rpc.NewRPCFunc(Tx, "query,prove"), + "tx_search": rpc.NewRPCFunc(TxSearch, "query,prove"), "validators": rpc.NewRPCFunc(Validators, "height"), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxs, ""), diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 53d07325..e5ae048c 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -111,7 +111,11 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { return nil, errors.Wrap(err, "error during searching for a hash in the query") } else if ok { res, err := txi.Get(hash) - return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") + if res == nil { + return []*types.TxResult{}, nil + } else { + return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") + } } // conditions to skip because they're handled before "everything else" From 10d893ee9b50d9ee376fc3a0df2b239e56ba537e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 13:51:28 -0600 Subject: [PATCH 18/27] update deps --- glide.lock | 6 +++--- glide.yaml | 4 ++-- node/node.go | 2 +- state/txindex/kv/kv.go | 22 ++-------------------- 4 files changed, 8 insertions(+), 26 deletions(-) diff --git a/glide.lock b/glide.lock index 18a5d6a7..2e49ff5a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: ffe610ffb74c1ea5cbe8da5d0d3ae30d2640c7426fe9a889a60218ea36daaf53 -updated: 2017-11-29T17:21:18.25916493Z +hash: b4e6f2f40e2738e45cec07ed91a5733d94d29cdfa0c7eb686a4d0a34512e2097 +updated: 2017-11-29T18:57:12.922510534Z imports: - name: github.com/btcsuite/btcd version: 8cea3866d0f7fb12d567a20744942c0d078c7d15 @@ -123,7 +123,7 @@ imports: subpackages: - iavl - name: github.com/tendermint/tmlibs - version: 1e12754b3a3b5f1c23bf44c2d882faae688fb2e8 + version: 21fb7819891997c96838308b4eba5a50b07ff03f subpackages: - autofile - cli diff --git a/glide.yaml b/glide.yaml index 62c06fc9..9d37891d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: 5c29adc081795b04f9d046fb51d76903c22cfa6d + version: develop subpackages: - client - example/dummy @@ -34,7 +34,7 @@ import: subpackages: - iavl - package: github.com/tendermint/tmlibs - version: 1e12754b3a3b5f1c23bf44c2d882faae688fb2e8 + version: develop subpackages: - autofile - cli diff --git a/node/node.go b/node/node.go index ea668f8f..865b8741 100644 --- a/node/node.go +++ b/node/node.go @@ -367,7 +367,7 @@ func (n *Node) OnStart() error { } // start tx indexer - _, err = n.indexerService.Start() + err = n.indexerService.Start() if err != nil { return err } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index e5ae048c..5228a3c4 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -68,7 +68,7 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // index tx by tags for _, tag := range result.Result.Tags { - if stringInSlice(tag.Key, txi.tagsToIndex) { + if cmn.StringInSlice(tag.Key, txi.tagsToIndex) { storeBatch.Set(keyForTag(tag, result), hash) } } @@ -146,7 +146,7 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { // for all other conditions for i, c := range conditions { - if intInSlice(i, skipIndexes) { + if cmn.IntInSlice(i, skipIndexes) { continue } @@ -356,24 +356,6 @@ func keyForTag(tag *abci.KVPair, result *types.TxResult) []byte { /////////////////////////////////////////////////////////////////////////////// // Utils -func stringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -func intInSlice(a int, list []int) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - func intersect(as, bs [][]byte) [][]byte { i := make([][]byte, 0, cmn.MinInt(len(as), len(bs))) for _, a := range as { From a762253e24c8c0bc446c4e94ee0f310198927567 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 15:25:12 -0600 Subject: [PATCH 19/27] do not use AddBatch, prefer copying for now --- state/txindex/kv/kv.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 5228a3c4..ad108069 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -84,12 +84,23 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // Index indexes a single transaction using the given list of tags. func (txi *TxIndex) Index(result *types.TxResult) error { - batch := txindex.NewBatch(1) - err := batch.Add(result) - if err != nil { - return errors.Wrap(err, "failed to add tx result to batch") + b := txi.store.NewBatch() + + hash := result.Tx.Hash() + + // index tx by tags + for _, tag := range result.Result.Tags { + if cmn.StringInSlice(tag.Key, txi.tagsToIndex) { + b.Set(keyForTag(tag, result), hash) + } } - return txi.AddBatch(batch) + + // index tx by hash + rawBytes := wire.BinaryBytes(result) + b.Set(hash, rawBytes) + + b.Write() + return nil } // Search performs a search using the given query. It breaks the query into From 58789c52cd29a16ab6a152f6e8fff89d103ea200 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 15:30:12 -0600 Subject: [PATCH 20/27] add example for tx_search endpoint --- rpc/core/tx.go | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 4e4285a2..b6973591 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -105,7 +105,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // TxSearch allows you to query for multiple transactions results. // // ```shell -// curl "localhost:46657/tx_search?query='account.owner=\'Ivan\''&prove=true" +// curl "localhost:46657/tx_search?query=\"account.owner='Ivan'\"&prove=true" // ``` // // ```go @@ -117,6 +117,33 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // > The above command returns JSON structured like this: // // ```json +// { +// "result": [ +// { +// "proof": { +// "Proof": { +// "aunts": [ +// "J3LHbizt806uKnABNLwG4l7gXCA=", +// "iblMO/M1TnNtlAefJyNCeVhjAb0=", +// "iVk3ryurVaEEhdeS0ohAJZ3wtB8=", +// "5hqMkTeGqpct51ohX0lZLIdsn7Q=", +// "afhsNxFnLlZgFDoyPpdQSe0bR8g=" +// ] +// }, +// "Data": "mvZHHa7HhZ4aRT0xMDA=", +// "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776", +// "Total": 32, +// "Index": 31 +// }, +// "tx": "mvZHHa7HhZ4aRT0xMDA=", +// "tx_result": {}, +// "index": 31, +// "height": 12 +// } +// ], +// "id": "", +// "jsonrpc": "2.0" +// } // ``` // // Returns transactions matching the given query. From 864ad8546e56fa2778b85b499d06dbd1f6b4f629 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 20:04:00 -0600 Subject: [PATCH 21/27] more test cases --- state/txindex/kv/kv.go | 34 ++++++++++++++++++++-------------- state/txindex/kv/kv_test.go | 4 ++++ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index ad108069..413569b1 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -284,22 +284,24 @@ LOOP: if !isTagKey(it.Key()) { continue } - // no other way to stop iterator other than checking for upperBound - switch (r.upperBound).(type) { - case int64: - v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - if err == nil && v == r.upperBound { - if r.includeUpperBound { - hashes = append(hashes, it.Value()) + if r.upperBound != nil { + // no other way to stop iterator other than checking for upperBound + switch (r.upperBound).(type) { + case int64: + v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + if err == nil && v == r.upperBound { + if r.includeUpperBound { + hashes = append(hashes, it.Value()) + } + break LOOP } - break LOOP + // XXX: passing time in a ABCI Tags is not yet implemented + // case time.Time: + // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + // if v == r.upperBound { + // break + // } } - // XXX: passing time in a ABCI Tags is not yet implemented - // case time.Time: - // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - // if v == r.upperBound { - // break - // } } hashes = append(hashes, it.Value()) } @@ -320,6 +322,10 @@ func startKey(c query.Condition, height uint64) []byte { } func startKeyForRange(r queryRange, height uint64) []byte { + if r.lowerBound == nil { + return []byte(fmt.Sprintf("%s", r.key)) + } + var lowerBound interface{} if r.includeLowerBound { lowerBound = r.lowerBound diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index a51bb4bf..a5c46d6b 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -75,6 +75,10 @@ func TestTxSearch(t *testing.T) { {"account.number = 1 AND account.owner = 'Vlad'", 0}, // search by range {"account.number >= 1 AND account.number <= 5", 1}, + // search by range (lower bound) + {"account.number >= 1", 1}, + // search by range (upper bound) + {"account.number <= 5", 1}, // search using not allowed tag {"not_allowed = 'boom'", 0}, // search for not existing tx result From 66ad366a4fd19c13ce6cfb28d6f45e64810f2271 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 20:04:26 -0600 Subject: [PATCH 22/27] test searching for tx with multiple same tags --- state/txindex/kv/kv_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index a5c46d6b..3da91a5d 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -104,6 +104,27 @@ func TestTxSearch(t *testing.T) { } } +func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { + tagsToIndex := []string{"account.number"} + indexer := NewTxIndex(db.NewMemDB(), tagsToIndex) + + tx := types.Tx("SAME MULTIPLE TAGS WITH DIFFERENT VALUES") + tags := []*abci.KVPair{ + {Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 1}, + {Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 2}, + } + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: tags}} + + err := indexer.Index(txResult) + require.NoError(t, err) + + results, err := indexer.Search(query.MustParse("account.number >= 1")) + assert.NoError(t, err) + + assert.Len(t, results, 1) + assert.Equal(t, []*types.TxResult{txResult}, results) +} + func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} From cb9743e5671776d683cdb9a51de7d12854a2acae Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 29 Nov 2017 20:30:37 -0600 Subject: [PATCH 23/27] dummy app now returns one DeliverTx tag --- glide.lock | 6 +++--- glide.yaml | 2 +- rpc/client/rpc_test.go | 7 +++++++ rpc/test/helpers.go | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index 2e49ff5a..47ce1697 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: b4e6f2f40e2738e45cec07ed91a5733d94d29cdfa0c7eb686a4d0a34512e2097 -updated: 2017-11-29T18:57:12.922510534Z +hash: dba99959eb071d0e99be1a11c608ddafe5349866c8141000efbd57f4c5f8353e +updated: 2017-11-30T02:23:12.150634867Z imports: - name: github.com/btcsuite/btcd version: 8cea3866d0f7fb12d567a20744942c0d078c7d15 @@ -98,7 +98,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 5c29adc081795b04f9d046fb51d76903c22cfa6d + version: 72c3ea3872424fba6b564de9d722acd74e6ecedc subpackages: - client - example/counter diff --git a/glide.yaml b/glide.yaml index 9d37891d..4c7d69bd 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: develop + version: 72c3ea3872424fba6b564de9d722acd74e6ecedc subpackages: - client - example/dummy diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 6eab5b85..2f449cf9 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -333,5 +333,12 @@ func TestTxSearch(t *testing.T) { results, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false) require.Nil(t, err, "%+v", err) require.Len(t, results, 0) + + // we query using a tag (see dummy application) + results, err = c.TxSearch("app.creator='jae'", false) + require.Nil(t, err, "%+v", err) + if len(results) == 0 { + t.Fatal("expected a lot of transactions") + } } } diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index f6526011..73da30ad 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -80,6 +80,7 @@ func GetConfig() *cfg.Config { globalConfig.P2P.ListenAddress = tm globalConfig.RPC.ListenAddress = rpc globalConfig.RPC.GRPCListenAddress = grpc + globalConfig.TxIndex.IndexTags = "app.creator" // see dummy application } return globalConfig } From 03222d834b8dccde63ed10840a294164caa17f04 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 30 Nov 2017 11:46:28 -0600 Subject: [PATCH 24/27] update abci dependency --- glide.lock | 6 +++--- glide.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index 47ce1697..79455dd7 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: dba99959eb071d0e99be1a11c608ddafe5349866c8141000efbd57f4c5f8353e -updated: 2017-11-30T02:23:12.150634867Z +hash: b4e6f2f40e2738e45cec07ed91a5733d94d29cdfa0c7eb686a4d0a34512e2097 +updated: 2017-11-30T17:46:14.710809367Z imports: - name: github.com/btcsuite/btcd version: 8cea3866d0f7fb12d567a20744942c0d078c7d15 @@ -98,7 +98,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 72c3ea3872424fba6b564de9d722acd74e6ecedc + version: 5c29adc081795b04f9d046fb51d76903c22cfa6d subpackages: - client - example/counter diff --git a/glide.yaml b/glide.yaml index 4c7d69bd..9d37891d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: 72c3ea3872424fba6b564de9d722acd74e6ecedc + version: develop subpackages: - client - example/dummy From e538e0e0772163437c08c30c5c4eee0424c6feac Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 30 Nov 2017 20:02:39 -0600 Subject: [PATCH 25/27] config variable to index all tags --- config/config.go | 10 ++++++++-- node/node.go | 8 +++++++- state/txindex/kv/kv.go | 31 +++++++++++++++++++++++++------ state/txindex/kv/kv_test.go | 12 ++++++------ 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index fc3671d8..ea3fa13e 100644 --- a/config/config.go +++ b/config/config.go @@ -430,13 +430,19 @@ type TxIndexConfig struct { // bloat. This is, of course, depends on the indexer's DB and the volume of // transactions. IndexTags string `mapstructure:"index_tags"` + + // When set to true, tells indexer to index all tags. Note this may be not + // desirable (see the comment above). IndexTags has a precedence over + // IndexAllTags (i.e. when given both, IndexTags will be indexed). + IndexAllTags bool `mapstructure:"index_all_tags"` } // DefaultTxIndexConfig returns a default configuration for the transaction indexer. func DefaultTxIndexConfig() *TxIndexConfig { return &TxIndexConfig{ - Indexer: "kv", - IndexTags: "", + Indexer: "kv", + IndexTags: "", + IndexAllTags: false, } } diff --git a/node/node.go b/node/node.go index 865b8741..7841a103 100644 --- a/node/node.go +++ b/node/node.go @@ -288,7 +288,13 @@ func NewNode(config *cfg.Config, if err != nil { return nil, err } - txIndexer = kv.NewTxIndex(store, strings.Split(config.TxIndex.IndexTags, ",")) + if config.TxIndex.IndexTags != "" { + txIndexer = kv.NewTxIndex(store, kv.IndexTags(strings.Split(config.TxIndex.IndexTags, ","))) + } else if config.TxIndex.IndexAllTags { + txIndexer = kv.NewTxIndex(store, kv.IndexAllTags()) + } else { + txIndexer = kv.NewTxIndex(store) + } default: txIndexer = &null.TxIndex{} } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 413569b1..5ca4d062 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -27,13 +27,32 @@ var _ txindex.TxIndexer = (*TxIndex)(nil) // TxIndex is the simplest possible indexer, backed by key-value storage (levelDB). type TxIndex struct { - store db.DB - tagsToIndex []string + store db.DB + tagsToIndex []string + indexAllTags bool } // NewTxIndex creates new KV indexer. -func NewTxIndex(store db.DB, tagsToIndex []string) *TxIndex { - return &TxIndex{store: store, tagsToIndex: tagsToIndex} +func NewTxIndex(store db.DB, options ...func(*TxIndex)) *TxIndex { + txi := &TxIndex{store: store, tagsToIndex: make([]string, 0), indexAllTags: false} + for _, o := range options { + o(txi) + } + return txi +} + +// IndexTags is an option for setting which tags to index. +func IndexTags(tags []string) func(*TxIndex) { + return func(txi *TxIndex) { + txi.tagsToIndex = tags + } +} + +// IndexAllTags is an option for indexing all tags. +func IndexAllTags() func(*TxIndex) { + return func(txi *TxIndex) { + txi.indexAllTags = true + } } // Get gets transaction from the TxIndex storage and returns it or nil if the @@ -68,7 +87,7 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // index tx by tags for _, tag := range result.Result.Tags { - if cmn.StringInSlice(tag.Key, txi.tagsToIndex) { + if txi.indexAllTags || cmn.StringInSlice(tag.Key, txi.tagsToIndex) { storeBatch.Set(keyForTag(tag, result), hash) } } @@ -90,7 +109,7 @@ func (txi *TxIndex) Index(result *types.TxResult) error { // index tx by tags for _, tag := range result.Result.Tags { - if cmn.StringInSlice(tag.Key, txi.tagsToIndex) { + if txi.indexAllTags || cmn.StringInSlice(tag.Key, txi.tagsToIndex) { b.Set(keyForTag(tag, result), hash) } } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 3da91a5d..e55f4887 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -16,7 +16,7 @@ import ( ) func TestTxIndex(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB(), []string{}) + indexer := NewTxIndex(db.NewMemDB()) tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}} @@ -46,8 +46,8 @@ func TestTxIndex(t *testing.T) { } func TestTxSearch(t *testing.T) { - tagsToIndex := []string{"account.number", "account.owner", "account.date"} - indexer := NewTxIndex(db.NewMemDB(), tagsToIndex) + tags := []string{"account.number", "account.owner", "account.date"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(tags)) tx := types.Tx("HELLO WORLD") tags := []*abci.KVPair{ @@ -105,8 +105,8 @@ func TestTxSearch(t *testing.T) { } func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { - tagsToIndex := []string{"account.number"} - indexer := NewTxIndex(db.NewMemDB(), tagsToIndex) + tags := []string{"account.number"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(tags)) tx := types.Tx("SAME MULTIPLE TAGS WITH DIFFERENT VALUES") tags := []*abci.KVPair{ @@ -136,7 +136,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { defer os.RemoveAll(dir) // nolint: errcheck store := db.NewDB("tx_index", "leveldb", dir) - indexer := NewTxIndex(store, []string{}) + indexer := NewTxIndex(store) batch := txindex.NewBatch(txsCount) for i := 0; i < txsCount; i++ { From c5b62ce1eeca02d13aafe4dbdc72815a0fe15318 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 30 Nov 2017 20:15:03 -0600 Subject: [PATCH 26/27] correct abci version --- glide.lock | 8 ++++---- glide.yaml | 2 +- state/txindex/kv/kv_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/glide.lock b/glide.lock index 79455dd7..d69aacb0 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: b4e6f2f40e2738e45cec07ed91a5733d94d29cdfa0c7eb686a4d0a34512e2097 -updated: 2017-11-30T17:46:14.710809367Z +hash: 8c38726da2666831affa40474117d3cef5dad083176e81fb013d7e8493b83e6f +updated: 2017-12-01T02:14:22.08770964Z imports: - name: github.com/btcsuite/btcd version: 8cea3866d0f7fb12d567a20744942c0d078c7d15 @@ -98,7 +98,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 5c29adc081795b04f9d046fb51d76903c22cfa6d + version: 22b491bb1952125dd2fb0730d6ca8e59e310547c subpackages: - client - example/counter @@ -113,7 +113,7 @@ imports: - name: github.com/tendermint/go-crypto version: dd20358a264c772b4a83e477b0cfce4c88a7001d - name: github.com/tendermint/go-wire - version: 7d50b38b3815efe313728de77e2995c8813ce13f + version: 5ab49b4c6ad674da6b81442911cf713ef0afb544 subpackages: - data - data/base58 diff --git a/glide.yaml b/glide.yaml index 9d37891d..18f0dae8 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,7 +18,7 @@ import: - package: github.com/spf13/viper version: v1.0.0 - package: github.com/tendermint/abci - version: develop + version: 22b491bb1952125dd2fb0730d6ca8e59e310547c subpackages: - client - example/dummy diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index e55f4887..ce63df9e 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -46,8 +46,8 @@ func TestTxIndex(t *testing.T) { } func TestTxSearch(t *testing.T) { - tags := []string{"account.number", "account.owner", "account.date"} - indexer := NewTxIndex(db.NewMemDB(), IndexTags(tags)) + allowedTags := []string{"account.number", "account.owner", "account.date"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) tx := types.Tx("HELLO WORLD") tags := []*abci.KVPair{ @@ -105,8 +105,8 @@ func TestTxSearch(t *testing.T) { } func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { - tags := []string{"account.number"} - indexer := NewTxIndex(db.NewMemDB(), IndexTags(tags)) + allowedTags := []string{"account.number"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) tx := types.Tx("SAME MULTIPLE TAGS WITH DIFFERENT VALUES") tags := []*abci.KVPair{ From 64233069804d6631792701479dbed8d3eb0a9ab2 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 30 Nov 2017 23:02:40 -0600 Subject: [PATCH 27/27] TestIndexAllTags (unit) --- state/txindex/kv/kv_test.go | 41 +++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index ce63df9e..efe17a18 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -49,14 +49,12 @@ func TestTxSearch(t *testing.T) { allowedTags := []string{"account.number", "account.owner", "account.date"} indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) - tx := types.Tx("HELLO WORLD") - tags := []*abci.KVPair{ + txResult := txResultWithTags([]*abci.KVPair{ {Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 1}, {Key: "account.owner", ValueType: abci.KVPair_STRING, ValueString: "Ivan"}, {Key: "not_allowed", ValueType: abci.KVPair_STRING, ValueString: "Vlad"}, - } - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: tags}} - hash := tx.Hash() + }) + hash := txResult.Tx.Hash() err := indexer.Index(txResult) require.NoError(t, err) @@ -108,12 +106,10 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { allowedTags := []string{"account.number"} indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) - tx := types.Tx("SAME MULTIPLE TAGS WITH DIFFERENT VALUES") - tags := []*abci.KVPair{ + txResult := txResultWithTags([]*abci.KVPair{ {Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 1}, {Key: "account.number", ValueType: abci.KVPair_INT, ValueInt: 2}, - } - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: tags}} + }) err := indexer.Index(txResult) require.NoError(t, err) @@ -125,6 +121,33 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { assert.Equal(t, []*types.TxResult{txResult}, results) } +func TestIndexAllTags(t *testing.T) { + indexer := NewTxIndex(db.NewMemDB(), IndexAllTags()) + + txResult := txResultWithTags([]*abci.KVPair{ + abci.KVPairString("account.owner", "Ivan"), + abci.KVPairInt("account.number", 1), + }) + + err := indexer.Index(txResult) + require.NoError(t, err) + + results, err := indexer.Search(query.MustParse("account.number >= 1")) + assert.NoError(t, err) + assert.Len(t, results, 1) + assert.Equal(t, []*types.TxResult{txResult}, results) + + results, err = indexer.Search(query.MustParse("account.owner = 'Ivan'")) + assert.NoError(t, err) + assert.Len(t, results, 1) + assert.Equal(t, []*types.TxResult{txResult}, results) +} + +func txResultWithTags(tags []*abci.KVPair) *types.TxResult { + tx := types.Tx("HELLO WORLD") + return &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: tags}} +} + func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: "", Tags: []*abci.KVPair{}}}