Extend abci-cli to allow integration tests

This commit adds the basic test command 'abci-cli test' that will
allow developers of server for their own language to continuously test
their implementation.

README.md

@@ -192,3 +192,47 @@ Here, we describe the requests and responses as function arguments and return va
 #### Flush
   * __Usage__:<br/>
     * Signals that messages queued on the client should be flushed to the server. It is called periodically by the client implementation to ensure asynchronous requests are actually sent, and is called immediately to make a synchronous request, which returns when the Flush response comes back.
+
+# Implementation
+
+We provide three implementations of the ABCI in Go:
+
+1. ABCI-socket
+2. GRPC
+3. Golang in-process
+
+## Socket
+
+ABCI is best implemented as a streaming protocol.
+The socket implementation provides for asynchronous, ordered message passing over unix or tcp.
+Messages are serialized using Protobuf3 and length-prefixed.
+Protobuf3 doesn't have an official length-prefix standard, so we use our own.  The first byte represents the length of the big-endian encoded length.
+
+For example, if the Protobuf3 encoded ABCI message is `0xDEADBEEF` (4 bytes), the length-prefixed message is `0x0104DEADBEEF`.  If the Protobuf3 encoded ABCI message is 65535 bytes long, the length-prefixed message would be like `0x02FFFF...`.
+
+## GRPC
+
+GRPC is an rpc framework native to Protocol Buffers with support in many languages.
+Implementing the ABCI using GRPC can allow for faster prototyping, but is expected to be much slower than
+the ordered, asynchronous socket protocol.
+
+Note the length-prefixing used in the socket implementation does not apply for GRPC.
+
+## In Process
+
+The simplest implementation just uses function calls within Go.
+This means ABCI applications written in Golang can be compiled with TendermintCore and run as a single binary.
+
+
+# Tools and Apps
+
+The `abci-cli` tool wraps any ABCI client and can be used for probing/testing an ABCI application.
+See the [guide](https://tendermint.readthedocs.io/en/master/abci-cli.html) for more details.
+
+Multiple example apps are included:
+- the `counter` application, which illustrates nonce checking in txs
+- the `dummy` application, which illustrates a simple key-value merkle tree
+- the `dummy --persistent` application, which augments the dummy with persistence and validator set changes
+
+When you are developing an implementation of the ABCI server in your favourite language, we provide the `abci-cli test`
+command that allows you to run our tests against your own ABCI server. Please make sure that your server supports at least the TSP protocol, and optionally gRPC.

client/grpc_client.go

@@ -48,7 +48,6 @@ func (cli *grpcClient) OnStart() error {
 		return err
 	}
 RETRY_LOOP:
-
 	for {
 		conn, err := grpc.Dial(cli.addr, grpc.WithInsecure(), grpc.WithDialer(dialerFunc))
 		if err != nil {
@@ -83,8 +82,8 @@ func (cli *grpcClient) OnStop() {
 	cli.mtx.Lock()
 	defer cli.mtx.Unlock()
 	// TODO: how to close conn? its not a net.Conn and grpc doesn't expose a Close()
-	/*if cli.conn != nil {
-		cli.conn.Close()
+	/*if cli.client.conn != nil {
+		cli.client.conn.Close()
 	}*/
 }
 

cmd/abci-cli/abci-cli.go

@@ -19,6 +19,7 @@ import (
 	"github.com/tendermint/abci/example/counter"
 	"github.com/tendermint/abci/example/dummy"
 	"github.com/tendermint/abci/server"
+	servertest "github.com/tendermint/abci/tests/server"
 	"github.com/tendermint/abci/types"
 	"github.com/tendermint/abci/version"
 )
@@ -141,6 +142,7 @@ func addCommands() {
 	RootCmd.AddCommand(checkTxCmd)
 	RootCmd.AddCommand(commitCmd)
 	RootCmd.AddCommand(versionCmd)
+	RootCmd.AddCommand(testCmd)
 	addQueryFlags()
 	RootCmd.AddCommand(queryCmd)
 
@@ -271,6 +273,16 @@ var dummyCmd = &cobra.Command{
 	},
 }
 
+var testCmd = &cobra.Command{
+	Use: "test",
+	Short: "Run integration tests",
+	Long: "",
+	Args: cobra.ExactArgs(0),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return cmdTest(cmd, args)
+	},
+}
+
 // Generates new Args array based off of previous call args to maintain flag persistence
 func persistentArgs(line []byte) []string {
 
@@ -287,6 +299,45 @@ func persistentArgs(line []byte) []string {
 
 //--------------------------------------------------------------------------------
 
+func cmdTest(cmd *cobra.Command, args []string) error {
+	fmt.Println("Running tests")
+
+	var err error
+
+	err = servertest.InitChain(client)
+	fmt.Println("")
+	err = servertest.SetOption(client, "serial", "on")
+	fmt.Println("")
+	err = servertest.Commit(client, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte("abc"), types.CodeType_BadNonce, nil)
+	fmt.Println("")
+	err = servertest.Commit(client, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x00}, types.CodeType_OK, nil)
+	fmt.Println("")
+	err = servertest.Commit(client, []byte{0, 0, 0, 0, 0, 0, 0, 1})
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x00}, types.CodeType_BadNonce, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x01}, types.CodeType_OK, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x00, 0x02}, types.CodeType_OK, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x00, 0x03}, types.CodeType_OK, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x00, 0x00, 0x04}, types.CodeType_OK, nil)
+	fmt.Println("")
+	err = servertest.DeliverTx(client, []byte{0x00, 0x00, 0x06}, types.CodeType_BadNonce, nil)
+	fmt.Println("")
+	err = servertest.Commit(client, []byte{0, 0, 0, 0, 0, 0, 0, 5})
+
+	if err != nil {
+		return errors.New("Some checks didn't pass, please use the cli to see the exact failures.")
+	}
+	return nil
+}
+
 func cmdBatch(cmd *cobra.Command, args []string) error {
 	bufReader := bufio.NewReader(os.Stdin)
 	for {

tests/server/client.go

@@ -0,0 +1,103 @@
+package testsuite
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	abcicli "github.com/tendermint/abci/client"
+	"github.com/tendermint/abci/types"
+	crypto "github.com/tendermint/go-crypto"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+func InitChain(client abcicli.Client) error {
+	total := 10
+	vals := make([]*types.Validator, total)
+	for i := 0; i < total; i++ {
+		pubkey := crypto.GenPrivKeyEd25519FromSecret([]byte(cmn.Fmt("test%d", i))).PubKey().Bytes()
+		power := cmn.RandInt()
+		vals[i] = &types.Validator{pubkey, uint64(power)}
+	}
+	err := client.InitChainSync(vals)
+	if err != nil {
+		fmt.Println("Failed test: InitChain - %v", err)
+		return err
+	}
+	fmt.Println("Passed test: InitChain")
+	return nil
+}
+
+func SetOption(client abcicli.Client, key, value string) error {
+	res := client.SetOptionSync(key, value)
+	_, _, log := res.Code, res.Data, res.Log
+	if res.IsErr() {
+		fmt.Println("Failed test: SetOption")
+		fmt.Printf("setting %v=%v: \nlog: %v", key, value, log)
+		fmt.Println("Failed test: SetOption")
+		return errors.New(res.Error())
+	}
+	fmt.Println("Passed test: SetOption")
+	return nil
+}
+
+func Commit(client abcicli.Client, hashExp []byte) error {
+	res := client.CommitSync()
+	_, data, log := res.Code, res.Data, res.Log
+	if res.IsErr() {
+		fmt.Println("Failed test: Commit")
+		fmt.Printf("committing %v\nlog: %v", log)
+		return errors.New(res.Error())
+	}
+	if !bytes.Equal(res.Data, hashExp) {
+		fmt.Println("Failed test: Commit")
+		fmt.Printf("Commit hash was unexpected. Got %X expected %X",
+			data, hashExp)
+		return errors.New("CommitTx failed")
+	}
+	fmt.Println("Passed test: Commit")
+	return nil
+}
+
+func DeliverTx(client abcicli.Client, txBytes []byte, codeExp types.CodeType, dataExp []byte) error {
+	res := client.DeliverTxSync(txBytes)
+	code, data, log := res.Code, res.Data, res.Log
+	if code != codeExp {
+		fmt.Println("Failed test: DeliverTx")
+		fmt.Printf("DeliverTx response code was unexpected. Got %v expected %v. Log: %v",
+			code, codeExp, log)
+		return errors.New("DeliverTx error")
+	}
+	if !bytes.Equal(data, dataExp) {
+		fmt.Println("Failed test: DeliverTx")
+		fmt.Printf("DeliverTx response data was unexpected. Got %X expected %X",
+			data, dataExp)
+		return errors.New("DeliverTx error")
+	}
+	fmt.Println("Passed test: DeliverTx")
+	return nil
+}
+
+func CheckTx(client abcicli.Client, txBytes []byte, codeExp types.CodeType, dataExp []byte) error {
+	res := client.CheckTxSync(txBytes)
+	code, data, log := res.Code, res.Data, res.Log
+	if res.IsErr() {
+		fmt.Println("Failed test: CheckTx")
+		fmt.Printf("checking tx %X: %v\nlog: %v", txBytes, log)
+		return errors.New(res.Error())
+	}
+	if code != codeExp {
+		fmt.Println("Failed test: CheckTx")
+		fmt.Printf("CheckTx response code was unexpected. Got %v expected %v. Log: %v",
+			code, codeExp, log)
+		return errors.New("CheckTx")
+	}
+	if !bytes.Equal(data, dataExp) {
+		fmt.Println("Failed test: CheckTx")
+		fmt.Printf("CheckTx response data was unexpected. Got %X expected %X",
+			data, dataExp)
+		return errors.New("CheckTx")
+	}
+	fmt.Println("Passed test: CheckTx")
+	return nil
+}

tests/tests.go

@@ -0,0 +1 @@
+package tests