Merge PR #2730: add tx search pagination related CLI/REST API parameter

PENDING.md

@@ -43,6 +43,7 @@ FEATURES
 
 * Gaia CLI  (`gaiacli`)
   * \#2399 Implement `params` command to query slashing parameters.
+  * [\#2730](https://github.com/cosmos/cosmos-sdk/issues/2730) Add tx search pagination parameter
   * [\#3027](https://github.com/cosmos/cosmos-sdk/issues/3027) Implement
   `query gov proposer [proposal-id]` to query for a proposal's proposer.
 

client/tx/search.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/cosmos/cosmos-sdk/client"
@@ -17,11 +18,16 @@ import (
 	"github.com/spf13/viper"
 
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
 )
 
 const (
 	flagTags     = "tags"
 	flagAny      = "any"
+	flagPage     = "page"
+	flagLimit    = "limit"
+	defaultPage  = 1
+	defaultLimit = 30 // should be consistent with tendermint/tendermint/rpc/core/pipe.go:19
 )
 
 // default client command to search through tagged transactions
@@ -32,7 +38,7 @@ func SearchTxCmd(cdc *codec.Codec) *cobra.Command {
 		Long: strings.TrimSpace(`
 Search for transactions that match exactly the given tags. For example:
 
-$ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>'
+$ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>' --page 1 --limit 30
 `),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			tagsStr := viper.GetString(flagTags)
@@ -53,12 +59,18 @@ $ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>'
 				}
 
 				keyValue := strings.Split(tag, ":")
+				if keyValue[0] == types.TxHeightKey {
+					tag = fmt.Sprintf("%s=%s", keyValue[0], keyValue[1])
+				} else {
 					tag = fmt.Sprintf("%s='%s'", keyValue[0], keyValue[1])
+				}
 				tmTags = append(tmTags, tag)
 			}
+			page := viper.GetInt(flagPage)
+			limit := viper.GetInt(flagLimit)
 
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			txs, err := SearchTxs(cliCtx, cdc, tmTags)
+			txs, err := SearchTxs(cliCtx, cdc, tmTags, page, limit)
 			if err != nil {
 				return err
 			}
@@ -86,17 +98,27 @@ $ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>'
 	cmd.Flags().Bool(client.FlagTrustNode, false, "Trust connected full node (don't verify proofs for responses)")
 	viper.BindPFlag(client.FlagTrustNode, cmd.Flags().Lookup(client.FlagTrustNode))
 	cmd.Flags().String(flagTags, "", "tag:value list of tags that must match")
+	cmd.Flags().Int32(flagPage, defaultPage, "Query a specific page of paginated results")
+	cmd.Flags().Int32(flagLimit, defaultLimit, "Query number of transactions results per page returned")
 	return cmd
 }
 
 // SearchTxs performs a search for transactions for a given set of tags via
 // Tendermint RPC. It returns a slice of Info object containing txs and metadata.
 // An error is returned if the query fails.
-func SearchTxs(cliCtx context.CLIContext, cdc *codec.Codec, tags []string) ([]Info, error) {
+func SearchTxs(cliCtx context.CLIContext, cdc *codec.Codec, tags []string, page, limit int) ([]Info, error) {
 	if len(tags) == 0 {
 		return nil, errors.New("must declare at least one tag to search")
 	}
 
+	if page <= 0 {
+		return nil, errors.New("page must greater than 0")
+	}
+
+	if limit <= 0 {
+		return nil, errors.New("limit must greater than 0")
+	}
+
 	// XXX: implement ANY
 	query := strings.Join(tags, " AND ")
 
@@ -108,10 +130,7 @@ func SearchTxs(cliCtx context.CLIContext, cdc *codec.Codec, tags []string) ([]In
 
 	prove := !cliCtx.TrustNode
 
-	// TODO: take these as args
+	res, err := node.TxSearch(query, prove, page, limit)
-	page := 0
-	perPage := 100
-	res, err := node.TxSearch(query, prove, page, perPage)
 	if err != nil {
 		return nil, err
 	}
@@ -153,6 +172,7 @@ func FormatTxResults(cdc *codec.Codec, res []*ctypes.ResultTx) ([]Info, error) {
 func SearchTxRequestHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var tags []string
+		var page, limit int
 		var txs []Info
 		err := r.ParseForm()
 		if err != nil {
@@ -164,18 +184,14 @@ func SearchTxRequestHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec) http.
 			return
 		}
 
-		for key, values := range r.Form {
+		tags, page, limit, err = parseHTTPArgs(r)
-			value, err := url.QueryUnescape(values[0])
+
 		if err != nil {
-				utils.WriteErrorResponse(w, http.StatusBadRequest, sdk.AppendMsgToErr("could not decode query value", err.Error()))
+			utils.WriteErrorResponse(w, http.StatusBadRequest, err.Error())
 			return
 		}
 
-			tag := fmt.Sprintf("%s='%s'", key, value)
+		txs, err = SearchTxs(cliCtx, cdc, tags, page, limit)
-			tags = append(tags, tag)
-		}
-
-		txs, err = SearchTxs(cliCtx, cdc, tags)
 		if err != nil {
 			utils.WriteErrorResponse(w, http.StatusInternalServerError, err.Error())
 			return
@@ -184,3 +200,51 @@ func SearchTxRequestHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec) http.
 		utils.PostProcessResponse(w, cdc, txs, cliCtx.Indent)
 	}
 }
+
+func parseHTTPArgs(r *http.Request) (tags []string, page, limit int, err error) {
+	tags = make([]string, 0, len(r.Form))
+	for key, values := range r.Form {
+		if key == "page" || key == "limit" {
+			continue
+		}
+		var value string
+		value, err = url.QueryUnescape(values[0])
+		if err != nil {
+			return tags, page, limit, err
+		}
+
+		var tag string
+		if key == types.TxHeightKey {
+			tag = fmt.Sprintf("%s=%s", key, value)
+		} else {
+			tag = fmt.Sprintf("%s='%s'", key, value)
+		}
+		tags = append(tags, tag)
+	}
+
+	pageStr := r.FormValue("page")
+	if pageStr == "" {
+		page = defaultPage
+	} else {
+		page, err = strconv.Atoi(pageStr)
+		if err != nil {
+			return tags, page, limit, err
+		} else if page <= 0 {
+			return tags, page, limit, errors.New("page must greater than 0")
+		}
+	}
+
+	limitStr := r.FormValue("limit")
+	if limitStr == "" {
+		limit = defaultLimit
+	} else {
+		limit, err = strconv.Atoi(limitStr)
+		if err != nil {
+			return tags, page, limit, err
+		} else if limit <= 0 {
+			return tags, page, limit, errors.New("limit must greater than 0")
+		}
+	}
+
+	return tags, page, limit, nil
+}

cmd/gaia/cli_test/cli_test.go

@@ -1,6 +1,7 @@
 package clitest
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -354,7 +355,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	tests.WaitForNextNBlocksTM(1, f.Port)
 
 	// Ensure transaction tags can be queried
-	txs := f.QueryTxs("action:submit_proposal", fmt.Sprintf("proposer:%s", fooAddr))
+	txs := f.QueryTxs(1, 50, "action:submit_proposal", fmt.Sprintf("proposer:%s", fooAddr))
 	require.Len(t, txs, 1)
 
 	// Ensure deposit was deducted
@@ -397,7 +398,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	require.Equal(t, int64(15), deposit.Amount.AmountOf(denom).Int64())
 
 	// Ensure tags are set on the transaction
-	txs = f.QueryTxs("action:deposit", fmt.Sprintf("depositor:%s", fooAddr))
+	txs = f.QueryTxs(1, 50, "action:deposit", fmt.Sprintf("depositor:%s", fooAddr))
 	require.Len(t, txs, 1)
 
 	// Ensure account has expected amount of funds
@@ -434,7 +435,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	require.Equal(t, gov.OptionYes, votes[0].Option)
 
 	// Ensure tags are applied to voting transaction properly
-	txs = f.QueryTxs("action:vote", fmt.Sprintf("voter:%s", fooAddr))
+	txs = f.QueryTxs(1, 50, "action:vote", fmt.Sprintf("voter:%s", fooAddr))
 	require.Len(t, txs, 1)
 
 	// Ensure no proposals in deposit period
@@ -456,6 +457,54 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	f.Cleanup()
 }
 
+func TestGaiaCLIQueryTxPagination(t *testing.T) {
+	t.Parallel()
+	f := InitFixtures(t)
+
+	// start gaiad server
+	proc := f.GDStart()
+	defer proc.Stop(false)
+
+	fooAddr := f.KeyAddress(keyFoo)
+	barAddr := f.KeyAddress(keyBar)
+
+	for i := 1; i <= 30; i++ {
+		success := executeWrite(t, fmt.Sprintf(
+			"gaiacli tx send %s --amount=%dfootoken --to=%s --from=foo",
+			f.Flags(), i, barAddr), app.DefaultKeyPass)
+		require.True(t, success)
+		tests.WaitForNextNBlocksTM(1, f.Port)
+	}
+
+	// perPage = 15, 2 pages
+	txsPage1 := f.QueryTxs(1, 15, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage1, 15)
+	txsPage2 := f.QueryTxs(2, 15, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage2, 15)
+	require.NotEqual(t, txsPage1, txsPage2)
+	txsPage3 := f.QueryTxs(3, 15, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage3, 15)
+	require.Equal(t, txsPage2, txsPage3)
+
+	// perPage = 16, 2 pages
+	txsPage1 = f.QueryTxs(1, 16, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage1, 16)
+	txsPage2 = f.QueryTxs(2, 16, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage2, 14)
+	require.NotEqual(t, txsPage1, txsPage2)
+
+	// perPage = 50
+	txsPageFull := f.QueryTxs(1, 50, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPageFull, 30)
+	require.Equal(t, txsPageFull, append(txsPage1, txsPage2...))
+
+	// perPage = 0
+	f.QueryTxsInvalid(errors.New("ERROR: page must greater than 0"), 0, 50, fmt.Sprintf("sender:%s", fooAddr))
+
+	// limit = 0
+	f.QueryTxsInvalid(errors.New("ERROR: limit must greater than 0"), 1, 0, fmt.Sprintf("sender:%s", fooAddr))
+}
+
 func TestGaiaCLIValidateSignatures(t *testing.T) {
 	t.Parallel()
 	f := InitFixtures(t)

cmd/gaia/cli_test/test_helpers.go

@@ -294,8 +294,8 @@ func (f *Fixtures) QueryAccount(address sdk.AccAddress, flags ...string) auth.Ba
 // gaiacli query txs
 
 // QueryTxs is gaiacli query txs
-func (f *Fixtures) QueryTxs(tags ...string) []tx.Info {
+func (f *Fixtures) QueryTxs(page, limit int, tags ...string) []tx.Info {
-	cmd := fmt.Sprintf("gaiacli query txs --tags='%s' %v", queryTags(tags), f.Flags())
+	cmd := fmt.Sprintf("gaiacli query txs --page=%d --limit=%d --tags='%s' %v", page, limit, queryTags(tags), f.Flags())
 	out, _ := tests.ExecuteT(f.T, cmd, "")
 	var txs []tx.Info
 	cdc := app.MakeCodec()
@@ -304,6 +304,13 @@ func (f *Fixtures) QueryTxs(tags ...string) []tx.Info {
 	return txs
 }
 
+// QueryTxsInvalid query txs with wrong parameters and compare expected error
+func (f *Fixtures) QueryTxsInvalid(expectedErr error, page, limit int, tags ...string) {
+	cmd := fmt.Sprintf("gaiacli query txs --page=%d --limit=%d --tags='%s' %v", page, limit, queryTags(tags), f.Flags())
+	_, err := tests.ExecuteT(f.T, cmd, "")
+	require.EqualError(f.T, expectedErr, err)
+}
+
 //___________________________________________________________________________________
 // gaiacli query staking
 

docs/gaia/gaiacli.md

@@ -214,6 +214,11 @@ And for using multiple `tags`:
 gaiacli query txs --tags='<tag1>:<value1>&<tag2>:<value2>'
 ```
 
+The pagination is supported as well via `page` and `limit`:
+```bash
+gaiacli query txs --tags='<tag>:<value>' --page=1 --limit=20
+```
+
 ::: tip Note
 
 The action tag always equals the message type returned by the `Type()` function of the relevant message.

x/gov/client/utils/query.go

@@ -2,7 +2,6 @@ package utils
 
 import (
 	"fmt"
-
 	"github.com/cosmos/cosmos-sdk/client/context"
 	"github.com/cosmos/cosmos-sdk/client/tx"
 	"github.com/cosmos/cosmos-sdk/codec"
@@ -10,6 +9,11 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/gov/tags"
 )
 
+const (
+	defaultPage  = 1
+	defaultLimit = 30 // should be consistent with tendermint/tendermint/rpc/core/pipe.go:19
+)
+
 // Proposer contains metadata of a governance proposal used for querying a
 // proposer.
 type Proposer struct {
@@ -32,7 +36,9 @@ func QueryDepositsByTxQuery(
 		fmt.Sprintf("%s='%s'", tags.ProposalID, []byte(fmt.Sprintf("%d", params.ProposalID))),
 	}
 
-	infos, err := tx.SearchTxs(cliCtx, cdc, tags)
+	// NOTE: SearchTxs is used to facilitate the txs query which does not currently
+	// support configurable pagination.
+	infos, err := tx.SearchTxs(cliCtx, cdc, tags, defaultPage, defaultLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -75,7 +81,9 @@ func QueryVotesByTxQuery(
 		fmt.Sprintf("%s='%s'", tags.ProposalID, []byte(fmt.Sprintf("%d", params.ProposalID))),
 	}
 
-	infos, err := tx.SearchTxs(cliCtx, cdc, tags)
+	// NOTE: SearchTxs is used to facilitate the txs query which does not currently
+	// support configurable pagination.
+	infos, err := tx.SearchTxs(cliCtx, cdc, tags, defaultPage, defaultLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +122,9 @@ func QueryVoteByTxQuery(
 		fmt.Sprintf("%s='%s'", tags.Voter, []byte(params.Voter.String())),
 	}
 
-	infos, err := tx.SearchTxs(cliCtx, cdc, tags)
+	// NOTE: SearchTxs is used to facilitate the txs query which does not currently
+	// support configurable pagination.
+	infos, err := tx.SearchTxs(cliCtx, cdc, tags, defaultPage, defaultLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -155,7 +165,9 @@ func QueryDepositByTxQuery(
 		fmt.Sprintf("%s='%s'", tags.Depositor, []byte(params.Depositor.String())),
 	}
 
-	infos, err := tx.SearchTxs(cliCtx, cdc, tags)
+	// NOTE: SearchTxs is used to facilitate the txs query which does not currently
+	// support configurable pagination.
+	infos, err := tx.SearchTxs(cliCtx, cdc, tags, defaultPage, defaultLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -195,7 +207,9 @@ func QueryProposerByTxQuery(
 		fmt.Sprintf("%s='%s'", tags.ProposalID, []byte(fmt.Sprintf("%d", proposalID))),
 	}
 
-	infos, err := tx.SearchTxs(cliCtx, cdc, tags)
+	// NOTE: SearchTxs is used to facilitate the txs query which does not currently
+	// support configurable pagination.
+	infos, err := tx.SearchTxs(cliCtx, cdc, tags, defaultPage, defaultLimit)
 	if err != nil {
 		return nil, err
 	}