Merge pull request #183 from tendermint/feature/121-add-benchmarks

Add benchmark for abci app

benchmarks/app_test.go

@@ -0,0 +1,199 @@
+package app
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	wire "github.com/tendermint/go-wire"
+	eyesApp "github.com/tendermint/merkleeyes/app"
+	eyes "github.com/tendermint/merkleeyes/client"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+
+	"github.com/tendermint/basecoin"
+	"github.com/tendermint/basecoin/app"
+	"github.com/tendermint/basecoin/modules/auth"
+	"github.com/tendermint/basecoin/modules/base"
+	"github.com/tendermint/basecoin/modules/coin"
+	"github.com/tendermint/basecoin/modules/fee"
+	"github.com/tendermint/basecoin/modules/nonce"
+	"github.com/tendermint/basecoin/modules/roles"
+	"github.com/tendermint/basecoin/stack"
+)
+
+type BenchApp struct {
+	App      *app.Basecoin
+	Accounts []*coin.AccountWithKey
+	ChainID  string
+}
+
+// DefaultHandler - placeholder to just handle sendtx
+func DefaultHandler(feeDenom string) basecoin.Handler {
+	// use the default stack
+	c := coin.NewHandler()
+	r := roles.NewHandler()
+	d := stack.NewDispatcher(
+		stack.WrapHandler(c),
+		stack.WrapHandler(r),
+	)
+	return stack.New(
+		base.Logger{},
+		stack.Recovery{},
+		auth.Signatures{},
+		base.Chain{},
+		nonce.ReplayCheck{},
+		roles.NewMiddleware(),
+		fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank),
+	).Use(d)
+}
+
+func NewBenchApp(h basecoin.Handler, chainID string, n int,
+	persist bool) BenchApp {
+
+	logger := log.NewNopLogger()
+	// logger := log.NewFilter(log.NewTMLogger(os.Stdout), log.AllowError())
+	// logger = log.NewTracingLogger(logger)
+
+	// TODO: disk writing
+	var eyesCli *eyes.Client
+	if persist {
+		tmpDir, _ := ioutil.TempDir("", "bc-app-benchmark")
+		eyesCli = eyes.NewLocalClient(tmpDir, 500)
+	} else {
+		eyesCli = eyes.NewLocalClient("", 0)
+	}
+	eyesApp.SetLogger(logger.With("module", "merkle"))
+
+	app := app.NewBasecoin(
+		h,
+		eyesCli,
+		logger.With("module", "app"),
+	)
+	res := app.SetOption("base/chain_id", chainID)
+	if res != "Success" {
+		panic("cannot set chain")
+	}
+
+	// make keys
+	money := coin.Coins{{"mycoin", 1234567890}}
+	accts := make([]*coin.AccountWithKey, n)
+	for i := 0; i < n; i++ {
+		accts[i] = coin.NewAccountWithKey(money)
+		res := app.SetOption("coin/account", accts[i].MakeOption())
+		if res != "Success" {
+			panic("can't set account")
+		}
+	}
+
+	return BenchApp{
+		App:      app,
+		Accounts: accts,
+		ChainID:  chainID,
+	}
+}
+
+// make a random tx...
+func (b BenchApp) makeTx(useFee bool) []byte {
+	n := len(b.Accounts)
+	sender := b.Accounts[cmn.RandInt()%n]
+	recipient := b.Accounts[cmn.RandInt()%n]
+	amount := coin.Coins{{"mycoin", 123}}
+	tx := coin.NewSendOneTx(sender.Actor(), recipient.Actor(), amount)
+	if useFee {
+		toll := coin.Coin{"mycoin", 2}
+		tx = fee.NewFee(tx, toll, sender.Actor())
+	}
+	sequence := sender.NextSequence()
+	tx = nonce.NewTx(sequence, []basecoin.Actor{sender.Actor()}, tx)
+	tx = base.NewChainTx(b.ChainID, 0, tx)
+	stx := auth.NewMulti(tx)
+	auth.Sign(stx, sender.Key)
+	res := wire.BinaryBytes(stx.Wrap())
+	return res
+}
+
+func BenchmarkMakeTx(b *testing.B) {
+	h := DefaultHandler("mycoin")
+	app := NewBenchApp(h, "bench-chain", 10, false)
+	b.ResetTimer()
+	for i := 1; i <= b.N; i++ {
+		txBytes := app.makeTx(true)
+		if len(txBytes) < 2 {
+			panic("cannot commit")
+		}
+	}
+}
+
+func benchmarkTransfers(b *testing.B, app BenchApp, blockSize int, useFee bool) {
+	// prepare txs
+	txs := make([][]byte, b.N)
+	for i := 1; i <= b.N; i++ {
+		txBytes := app.makeTx(useFee)
+		if len(txBytes) < 2 {
+			panic("cannot make bytes")
+		}
+		txs[i-1] = txBytes
+	}
+
+	b.ResetTimer()
+
+	for i := 1; i <= b.N; i++ {
+		res := app.App.DeliverTx(txs[i-1])
+		if res.IsErr() {
+			panic(res.Error())
+		}
+		if i%blockSize == 0 {
+			res := app.App.Commit()
+			if res.IsErr() {
+				panic("cannot commit")
+			}
+		}
+	}
+}
+
+func BenchmarkSimpleTransfer(b *testing.B) {
+	benchmarks := []struct {
+		accounts  int
+		blockSize int
+		useFee    bool
+		toDisk    bool
+	}{
+		{100, 10, false, false},
+		{100, 10, true, false},
+		{100, 200, false, false},
+		{100, 200, true, false},
+		{10000, 10, false, false},
+		{10000, 10, true, false},
+		{10000, 200, false, false},
+		{10000, 200, true, false},
+		{100, 10, false, true},
+		{100, 10, true, true},
+		{100, 200, false, true},
+		{100, 200, true, true},
+		{10000, 10, false, true},
+		{10000, 10, true, true},
+		{10000, 200, false, true},
+		{10000, 200, true, true},
+	}
+
+	for _, bb := range benchmarks {
+		prefix := fmt.Sprintf("%d-%d", bb.accounts, bb.blockSize)
+		if bb.useFee {
+			prefix += "-fee"
+		} else {
+			prefix += "-nofee"
+		}
+		if bb.toDisk {
+			prefix += "-persist"
+		} else {
+			prefix += "-memdb"
+		}
+
+		h := DefaultHandler("mycoin")
+		app := NewBenchApp(h, "bench-chain", bb.accounts, bb.toDisk)
+		b.Run(prefix, func(sub *testing.B) {
+			benchmarkTransfers(sub, app, bb.blockSize, bb.useFee)
+		})
+	}
+}

modules/coin/helper.go

@@ -11,7 +11,8 @@ import (
 // AccountWithKey is a helper for tests, that includes and account
 // along with the private key to access it.
 type AccountWithKey struct {
-	Key crypto.PrivKey
+	Key      crypto.PrivKey
+	Sequence uint32
 	Account
 }
 
@@ -34,6 +35,12 @@ func (a *AccountWithKey) Actor() basecoin.Actor {
 	return auth.SigPerm(a.Key.PubKey().Address())
 }
 
+// NextSequence returns the next sequence to sign with
+func (a *AccountWithKey) NextSequence() uint32 {
+	a.Sequence++
+	return a.Sequence
+}
+
 // MakeOption returns a string to use with SetOption to initialize this account
 //
 // This is intended for use in test cases

tests/tmsp/tmsp_test.go

@@ -1,159 +0,0 @@
-package tmsp_test
-
-// TODO: replace with benchmarker
-
-// import (
-// 	"encoding/json"
-// 	"testing"
-
-// 	"github.com/stretchr/testify/assert"
-// 	"github.com/stretchr/testify/require"
-// 	"github.com/tendermint/basecoin/app"
-// 	"github.com/tendermint/basecoin/types"
-// 	wire "github.com/tendermint/go-wire"
-// 	eyescli "github.com/tendermint/merkleeyes/client"
-// 	cmn "github.com/tendermint/tmlibs/common"
-// 	"github.com/tendermint/tmlibs/log"
-// )
-
-// func TestSendTx(t *testing.T) {
-// 	eyesCli := eyescli.NewLocalClient("", 0)
-// 	chainID := "test_chain_id"
-// 	bcApp := app.NewBasecoin(eyesCli, log.TestingLogger().With("module", "app"))
-// 	bcApp.SetOption("base/chain_id", chainID)
-// 	// t.Log(bcApp.Info())
-
-// 	test1PrivAcc := types.PrivAccountFromSecret("test1")
-// 	test2PrivAcc := types.PrivAccountFromSecret("test2")
-
-// 	// Seed Basecoin with account
-// 	test1Acc := test1PrivAcc.Account
-// 	test1Acc.Balance = coin.Coins{{"", 1000}}
-// 	accOpt, err := json.Marshal(test1Acc)
-// 	require.Nil(t, err)
-// 	bcApp.SetOption("base/account", string(accOpt))
-
-// 	// Construct a SendTx signature
-// 	tx := &types.SendTx{
-// 		Gas: 0,
-// 		Fee: coin.Coin{"", 0},
-// 		Inputs: []types.TxInput{
-// 			types.NewTxInput(test1PrivAcc.Account.PubKey, coin.Coins{{"", 1}}, 1),
-// 		},
-// 		Outputs: []types.TxOutput{
-// 			types.TxOutput{
-// 				Address: test2PrivAcc.Account.PubKey.Address(),
-// 				Coins:   coin.Coins{{"", 1}},
-// 			},
-// 		},
-// 	}
-
-// 	// Sign request
-// 	signBytes := tx.SignBytes(chainID)
-// 	// t.Log("Sign bytes: %X\n", signBytes)
-// 	sig := test1PrivAcc.Sign(signBytes)
-// 	tx.Inputs[0].Signature = sig
-// 	// t.Log("Signed TX bytes: %X\n", wire.BinaryBytes(types.TxS{tx}))
-
-// 	// Write request
-// 	txBytes := wire.BinaryBytes(types.TxS{tx})
-// 	res := bcApp.DeliverTx(txBytes)
-// 	// t.Log(res)
-// 	assert.True(t, res.IsOK(), "Failed: %v", res.Error())
-// }
-
-// func TestSequence(t *testing.T) {
-// 	eyesCli := eyescli.NewLocalClient("", 0)
-// 	chainID := "test_chain_id"
-// 	bcApp := app.NewBasecoin(eyesCli, log.TestingLogger().With("module", "app"))
-// 	bcApp.SetOption("base/chain_id", chainID)
-// 	// t.Log(bcApp.Info())
-
-// 	// Get the test account
-// 	test1PrivAcc := types.PrivAccountFromSecret("test1")
-// 	test1Acc := test1PrivAcc.Account
-// 	test1Acc.Balance = coin.Coins{{"", 1 << 53}}
-// 	accOpt, err := json.Marshal(test1Acc)
-// 	require.Nil(t, err)
-// 	bcApp.SetOption("base/account", string(accOpt))
-
-// 	sequence := int(1)
-// 	// Make a bunch of PrivAccounts
-// 	privAccounts := types.RandAccounts(1000, 1000000, 0)
-// 	privAccountSequences := make(map[string]int)
-// 	// Send coins to each account
-
-// 	for i := 0; i < len(privAccounts); i++ {
-// 		privAccount := privAccounts[i]
-
-// 		tx := &types.SendTx{
-// 			Gas: 2,
-// 			Fee: coin.Coin{"", 2},
-// 			Inputs: []types.TxInput{
-// 				types.NewTxInput(test1Acc.PubKey, coin.Coins{{"", 1000002}}, sequence),
-// 			},
-// 			Outputs: []types.TxOutput{
-// 				types.TxOutput{
-// 					Address: privAccount.Account.PubKey.Address(),
-// 					Coins:   coin.Coins{{"", 1000000}},
-// 				},
-// 			},
-// 		}
-// 		sequence += 1
-
-// 		// Sign request
-// 		signBytes := tx.SignBytes(chainID)
-// 		sig := test1PrivAcc.Sign(signBytes)
-// 		tx.Inputs[0].Signature = sig
-// 		// t.Log("ADDR: %X -> %X\n", tx.Inputs[0].Address, tx.Outputs[0].Address)
-
-// 		// Write request
-// 		txBytes := wire.BinaryBytes(struct{ types.Tx }{tx})
-// 		res := bcApp.DeliverTx(txBytes)
-// 		assert.True(t, res.IsOK(), "DeliverTx error: %v", res.Error())
-// 	}
-
-// 	res := bcApp.Commit()
-// 	assert.True(t, res.IsOK(), "Failed Commit: %v", res.Error())
-
-// 	t.Log("-------------------- RANDOM SENDS --------------------")
-
-// 	// Now send coins between these accounts
-// 	for i := 0; i < 10000; i++ {
-// 		randA := cmn.RandInt() % len(privAccounts)
-// 		randB := cmn.RandInt() % len(privAccounts)
-// 		if randA == randB {
-// 			continue
-// 		}
-
-// 		privAccountA := privAccounts[randA]
-// 		privAccountASequence := privAccountSequences[privAccountA.Account.PubKey.KeyString()]
-// 		privAccountSequences[privAccountA.Account.PubKey.KeyString()] = privAccountASequence + 1
-// 		privAccountB := privAccounts[randB]
-
-// 		tx := &types.SendTx{
-// 			Gas: 2,
-// 			Fee: coin.Coin{"", 2},
-// 			Inputs: []types.TxInput{
-// 				types.NewTxInput(privAccountA.PubKey, coin.Coins{{"", 3}}, privAccountASequence+1),
-// 			},
-// 			Outputs: []types.TxOutput{
-// 				types.TxOutput{
-// 					Address: privAccountB.PubKey.Address(),
-// 					Coins:   coin.Coins{{"", 1}},
-// 				},
-// 			},
-// 		}
-
-// 		// Sign request
-// 		signBytes := tx.SignBytes(chainID)
-// 		sig := privAccountA.Sign(signBytes)
-// 		tx.Inputs[0].Signature = sig
-// 		// t.Log("ADDR: %X -> %X\n", tx.Inputs[0].Address, tx.Outputs[0].Address)
-
-// 		// Write request
-// 		txBytes := wire.BinaryBytes(struct{ types.Tx }{tx})
-// 		res := bcApp.DeliverTx(txBytes)
-// 		assert.True(t, res.IsOK(), "DeliverTx error: %v", res.Error())
-// 	}
-// }