From a6b74b82d19efd0951b794e3df54a0988e5f640a Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 14 May 2018 16:01:49 +0400 Subject: [PATCH] limit /tx_search output Refs #909 --- rpc/client/httpclient.go | 14 ++++--- rpc/client/interface.go | 2 +- rpc/client/localclient.go | 4 +- rpc/client/rpc_test.go | 14 +++---- rpc/core/README.md | 6 +++ rpc/core/pipe.go | 20 ++++++++++ rpc/core/routes.go | 2 +- rpc/core/tx.go | 77 +++++++++++++++++++++---------------- rpc/core/types/responses.go | 6 +++ 9 files changed, 94 insertions(+), 51 deletions(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 89a1293a..ed1a5b32 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -204,17 +204,19 @@ func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return result, nil } -func (c *HTTP) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { - results := new([]*ctypes.ResultTx) +func (c *HTTP) TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) { + result := new(ctypes.ResultTxSearch) params := map[string]interface{}{ - "query": query, - "prove": prove, + "query": query, + "prove": prove, + "page": page, + "per_page": perPage, } - _, err := c.rpc.Call("tx_search", params, results) + _, err := c.rpc.Call("tx_search", params, result) if err != nil { return nil, errors.Wrap(err, "TxSearch") } - return *results, nil + return result, nil } func (c *HTTP) Validators(height *int64) (*ctypes.ResultValidators, error) { diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 0cc9333e..afe2d8fa 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -50,7 +50,7 @@ type SignClient interface { Commit(height *int64) (*ctypes.ResultCommit, error) Validators(height *int64) (*ctypes.ResultValidators, error) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) - TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) + TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) } // HistoryClient shows us data from genesis to now in large chunks. diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 0c47de83..c9bdddf1 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -128,8 +128,8 @@ func (Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return core.Tx(hash, prove) } -func (Local) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { - return core.TxSearch(query, prove) +func (Local) TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) { + return core.TxSearch(query, prove, page, perPage) } func (c *Local) Subscribe(ctx context.Context, subscriber string, query tmpubsub.Query, out chan<- interface{}) error { diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index eb25b94e..13109f78 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -334,11 +334,11 @@ func TestTxSearch(t *testing.T) { // now we query for the tx. // since there's only one tx, we know index=0. - results, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true) + result, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true, 1, 30) require.Nil(t, err, "%+v", err) - require.Len(t, results, 1) + require.Len(t, result.Txs, 1) - ptx := results[0] + ptx := result.Txs[0] assert.EqualValues(t, txHeight, ptx.Height) assert.EqualValues(t, tx, ptx.Tx) assert.Zero(t, ptx.Index) @@ -352,14 +352,14 @@ func TestTxSearch(t *testing.T) { } // we query for non existing tx - results, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false) + result, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false, 1, 30) require.Nil(t, err, "%+v", err) - require.Len(t, results, 0) + require.Len(t, result.Txs, 0) // we query using a tag (see kvstore application) - results, err = c.TxSearch("app.creator='jae'", false) + result, err = c.TxSearch("app.creator='jae'", false, 1, 30) require.Nil(t, err, "%+v", err) - if len(results) == 0 { + if len(result.Txs) == 0 { t.Fatal("expected a lot of transactions") } } diff --git a/rpc/core/README.md b/rpc/core/README.md index 1ed2f849..df84d6e6 100644 --- a/rpc/core/README.md +++ b/rpc/core/README.md @@ -13,3 +13,9 @@ go get github.com/melekes/godoc2md godoc2md -template rpc/core/doc_template.txt github.com/tendermint/tendermint/rpc/core | grep -v -e "pipe.go" -e "routes.go" -e "dev.go" | sed 's$/src/target$https://github.com/tendermint/tendermint/tree/master/rpc/core$' ``` + +## Pagination + +Requests that return multiple items will be paginated to 30 items by default. +You can specify further pages with the ?page parameter. You can also set a +custom page size up to 100 with the ?per_page parameter. diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index e93ba2f8..f97d1817 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -14,6 +14,12 @@ import ( "github.com/tendermint/tmlibs/log" ) +const ( + // see README + defaultPerPage = 30 + maxPerPage = 100 +) + var subscribeTimeout = 5 * time.Second //---------------------------------------------- @@ -117,3 +123,17 @@ func SetLogger(l log.Logger) { func SetEventBus(b *types.EventBus) { eventBus = b } + +func validatePage(page int) int { + if page < 1 { + return 1 + } + return page +} + +func validatePerPage(perPage int) int { + if perPage < 1 || perPage > maxPerPage { + return defaultPerPage + } + return perPage +} diff --git a/rpc/core/routes.go b/rpc/core/routes.go index bf90d6fb..ed6c6981 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -22,7 +22,7 @@ var Routes = map[string]*rpc.RPCFunc{ "block_results": rpc.NewRPCFunc(BlockResults, "height"), "commit": rpc.NewRPCFunc(Commit, "height"), "tx": rpc.NewRPCFunc(Tx, "hash,prove"), - "tx_search": rpc.NewRPCFunc(TxSearch, "query,prove"), + "tx_search": rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page"), "validators": rpc.NewRPCFunc(Validators, "height"), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""), "consensus_state": rpc.NewRPCFunc(ConsensusState, ""), diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 7ddc7080..ba68331e 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -6,6 +6,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" tmquery "github.com/tendermint/tmlibs/pubsub/query" ) @@ -104,7 +105,8 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { }, nil } -// TxSearch allows you to query for multiple transactions results. +// TxSearch allows you to query for multiple transactions results. It returns a +// list of transactions (maximum ?per_page entries) and the total count. // // ```shell // curl "localhost:46657/tx_search?query=\"account.owner='Ivan'\"&prove=true" @@ -120,43 +122,46 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // // ```json // { -// "result": [ -// { -// "proof": { -// "Proof": { -// "aunts": [ -// "J3LHbizt806uKnABNLwG4l7gXCA=", -// "iblMO/M1TnNtlAefJyNCeVhjAb0=", -// "iVk3ryurVaEEhdeS0ohAJZ3wtB8=", -// "5hqMkTeGqpct51ohX0lZLIdsn7Q=", -// "afhsNxFnLlZgFDoyPpdQSe0bR8g=" -// ] -// }, -// "Data": "mvZHHa7HhZ4aRT0xMDA=", -// "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776", -// "Total": 32, -// "Index": 31 -// }, -// "tx": "mvZHHa7HhZ4aRT0xMDA=", -// "tx_result": {}, -// "index": 31, -// "height": 12, -// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" -// } -// ], +// "jsonrpc": "2.0", // "id": "", -// "jsonrpc": "2.0" +// "result": { +// "txs": [ +// { +// "proof": { +// "Proof": { +// "aunts": [ +// "J3LHbizt806uKnABNLwG4l7gXCA=", +// "iblMO/M1TnNtlAefJyNCeVhjAb0=", +// "iVk3ryurVaEEhdeS0ohAJZ3wtB8=", +// "5hqMkTeGqpct51ohX0lZLIdsn7Q=", +// "afhsNxFnLlZgFDoyPpdQSe0bR8g=" +// ] +// }, +// "Data": "mvZHHa7HhZ4aRT0xMDA=", +// "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776", +// "Total": 32, +// "Index": 31 +// }, +// "tx": "mvZHHa7HhZ4aRT0xMDA=", +// "tx_result": {}, +// "index": 31, +// "height": 12, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" +// } +// ], +// "total_count": 1 +// } // } // ``` // -// Returns transactions matching the given query. -// // ### Query Parameters // // | Parameter | Type | Default | Required | Description | // |-----------+--------+---------+----------+-----------------------------------------------------------| // | query | string | "" | true | Query | // | prove | bool | false | false | Include proofs of the transactions inclusion in the block | +// | page | int | 1 | false | Page number (1-based) | +// | per_page | int | 30 | false | Number of entries per page (max: 100) | // // ### Returns // @@ -166,7 +171,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in // - `hash`: `[]byte` - hash of the transaction -func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { +func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { return nil, fmt.Errorf("Transaction indexing is disabled") @@ -182,11 +187,15 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { return nil, err } - // TODO: we may want to consider putting a maximum on this length and somehow - // informing the user that things were truncated. - apiResults := make([]*ctypes.ResultTx, len(results)) + totalCount := len(results) + page = validatePage(page) + perPage = validatePerPage(perPage) + skipCount := (page - 1) * perPage + + apiResults := make([]*ctypes.ResultTx, cmn.MinInt(perPage, totalCount-skipCount)) var proof types.TxProof - for i, r := range results { + for i := 0; i < len(apiResults); i++ { + r := results[skipCount+i] height := r.Height index := r.Index @@ -205,5 +214,5 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { } } - return apiResults, nil + return &ctypes.ResultTxSearch{Txs: apiResults, TotalCount: totalCount}, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 18c54545..5b001d7d 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -172,6 +172,12 @@ type ResultTx struct { Proof types.TxProof `json:"proof,omitempty"` } +// Result of searching for txs +type ResultTxSearch struct { + Txs []*ResultTx `json:"txs"` + TotalCount int `json:"total_count"` +} + // List of mempool txs type ResultUnconfirmedTxs struct { N int `json:"n_txs"`