From a6b74b82d19efd0951b794e3df54a0988e5f640a Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 14 May 2018 16:01:49 +0400 Subject: [PATCH 01/34] limit /tx_search output Refs #909 --- rpc/client/httpclient.go | 14 ++++--- rpc/client/interface.go | 2 +- rpc/client/localclient.go | 4 +- rpc/client/rpc_test.go | 14 +++---- rpc/core/README.md | 6 +++ rpc/core/pipe.go | 20 ++++++++++ rpc/core/routes.go | 2 +- rpc/core/tx.go | 77 +++++++++++++++++++++---------------- rpc/core/types/responses.go | 6 +++ 9 files changed, 94 insertions(+), 51 deletions(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 89a1293a..ed1a5b32 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -204,17 +204,19 @@ func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return result, nil } -func (c *HTTP) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { - results := new([]*ctypes.ResultTx) +func (c *HTTP) TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) { + result := new(ctypes.ResultTxSearch) params := map[string]interface{}{ - "query": query, - "prove": prove, + "query": query, + "prove": prove, + "page": page, + "per_page": perPage, } - _, err := c.rpc.Call("tx_search", params, results) + _, err := c.rpc.Call("tx_search", params, result) if err != nil { return nil, errors.Wrap(err, "TxSearch") } - return *results, nil + return result, nil } func (c *HTTP) Validators(height *int64) (*ctypes.ResultValidators, error) { diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 0cc9333e..afe2d8fa 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -50,7 +50,7 @@ type SignClient interface { Commit(height *int64) (*ctypes.ResultCommit, error) Validators(height *int64) (*ctypes.ResultValidators, error) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) - TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) + TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) } // HistoryClient shows us data from genesis to now in large chunks. diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 0c47de83..c9bdddf1 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -128,8 +128,8 @@ func (Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return core.Tx(hash, prove) } -func (Local) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { - return core.TxSearch(query, prove) +func (Local) TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) { + return core.TxSearch(query, prove, page, perPage) } func (c *Local) Subscribe(ctx context.Context, subscriber string, query tmpubsub.Query, out chan<- interface{}) error { diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index eb25b94e..13109f78 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -334,11 +334,11 @@ func TestTxSearch(t *testing.T) { // now we query for the tx. // since there's only one tx, we know index=0. - results, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true) + result, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true, 1, 30) require.Nil(t, err, "%+v", err) - require.Len(t, results, 1) + require.Len(t, result.Txs, 1) - ptx := results[0] + ptx := result.Txs[0] assert.EqualValues(t, txHeight, ptx.Height) assert.EqualValues(t, tx, ptx.Tx) assert.Zero(t, ptx.Index) @@ -352,14 +352,14 @@ func TestTxSearch(t *testing.T) { } // we query for non existing tx - results, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false) + result, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false, 1, 30) require.Nil(t, err, "%+v", err) - require.Len(t, results, 0) + require.Len(t, result.Txs, 0) // we query using a tag (see kvstore application) - results, err = c.TxSearch("app.creator='jae'", false) + result, err = c.TxSearch("app.creator='jae'", false, 1, 30) require.Nil(t, err, "%+v", err) - if len(results) == 0 { + if len(result.Txs) == 0 { t.Fatal("expected a lot of transactions") } } diff --git a/rpc/core/README.md b/rpc/core/README.md index 1ed2f849..df84d6e6 100644 --- a/rpc/core/README.md +++ b/rpc/core/README.md @@ -13,3 +13,9 @@ go get github.com/melekes/godoc2md godoc2md -template rpc/core/doc_template.txt github.com/tendermint/tendermint/rpc/core | grep -v -e "pipe.go" -e "routes.go" -e "dev.go" | sed 's$/src/target$https://github.com/tendermint/tendermint/tree/master/rpc/core$' ``` + +## Pagination + +Requests that return multiple items will be paginated to 30 items by default. +You can specify further pages with the ?page parameter. You can also set a +custom page size up to 100 with the ?per_page parameter. diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index e93ba2f8..f97d1817 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -14,6 +14,12 @@ import ( "github.com/tendermint/tmlibs/log" ) +const ( + // see README + defaultPerPage = 30 + maxPerPage = 100 +) + var subscribeTimeout = 5 * time.Second //---------------------------------------------- @@ -117,3 +123,17 @@ func SetLogger(l log.Logger) { func SetEventBus(b *types.EventBus) { eventBus = b } + +func validatePage(page int) int { + if page < 1 { + return 1 + } + return page +} + +func validatePerPage(perPage int) int { + if perPage < 1 || perPage > maxPerPage { + return defaultPerPage + } + return perPage +} diff --git a/rpc/core/routes.go b/rpc/core/routes.go index bf90d6fb..ed6c6981 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -22,7 +22,7 @@ var Routes = map[string]*rpc.RPCFunc{ "block_results": rpc.NewRPCFunc(BlockResults, "height"), "commit": rpc.NewRPCFunc(Commit, "height"), "tx": rpc.NewRPCFunc(Tx, "hash,prove"), - "tx_search": rpc.NewRPCFunc(TxSearch, "query,prove"), + "tx_search": rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page"), "validators": rpc.NewRPCFunc(Validators, "height"), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""), "consensus_state": rpc.NewRPCFunc(ConsensusState, ""), diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 7ddc7080..ba68331e 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -6,6 +6,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" tmquery "github.com/tendermint/tmlibs/pubsub/query" ) @@ -104,7 +105,8 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { }, nil } -// TxSearch allows you to query for multiple transactions results. +// TxSearch allows you to query for multiple transactions results. It returns a +// list of transactions (maximum ?per_page entries) and the total count. // // ```shell // curl "localhost:46657/tx_search?query=\"account.owner='Ivan'\"&prove=true" @@ -120,43 +122,46 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // // ```json // { -// "result": [ -// { -// "proof": { -// "Proof": { -// "aunts": [ -// "J3LHbizt806uKnABNLwG4l7gXCA=", -// "iblMO/M1TnNtlAefJyNCeVhjAb0=", -// "iVk3ryurVaEEhdeS0ohAJZ3wtB8=", -// "5hqMkTeGqpct51ohX0lZLIdsn7Q=", -// "afhsNxFnLlZgFDoyPpdQSe0bR8g=" -// ] -// }, -// "Data": "mvZHHa7HhZ4aRT0xMDA=", -// "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776", -// "Total": 32, -// "Index": 31 -// }, -// "tx": "mvZHHa7HhZ4aRT0xMDA=", -// "tx_result": {}, -// "index": 31, -// "height": 12, -// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" -// } -// ], +// "jsonrpc": "2.0", // "id": "", -// "jsonrpc": "2.0" +// "result": { +// "txs": [ +// { +// "proof": { +// "Proof": { +// "aunts": [ +// "J3LHbizt806uKnABNLwG4l7gXCA=", +// "iblMO/M1TnNtlAefJyNCeVhjAb0=", +// "iVk3ryurVaEEhdeS0ohAJZ3wtB8=", +// "5hqMkTeGqpct51ohX0lZLIdsn7Q=", +// "afhsNxFnLlZgFDoyPpdQSe0bR8g=" +// ] +// }, +// "Data": "mvZHHa7HhZ4aRT0xMDA=", +// "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776", +// "Total": 32, +// "Index": 31 +// }, +// "tx": "mvZHHa7HhZ4aRT0xMDA=", +// "tx_result": {}, +// "index": 31, +// "height": 12, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" +// } +// ], +// "total_count": 1 +// } // } // ``` // -// Returns transactions matching the given query. -// // ### Query Parameters // // | Parameter | Type | Default | Required | Description | // |-----------+--------+---------+----------+-----------------------------------------------------------| // | query | string | "" | true | Query | // | prove | bool | false | false | Include proofs of the transactions inclusion in the block | +// | page | int | 1 | false | Page number (1-based) | +// | per_page | int | 30 | false | Number of entries per page (max: 100) | // // ### Returns // @@ -166,7 +171,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in // - `hash`: `[]byte` - hash of the transaction -func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { +func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { return nil, fmt.Errorf("Transaction indexing is disabled") @@ -182,11 +187,15 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { return nil, err } - // TODO: we may want to consider putting a maximum on this length and somehow - // informing the user that things were truncated. - apiResults := make([]*ctypes.ResultTx, len(results)) + totalCount := len(results) + page = validatePage(page) + perPage = validatePerPage(perPage) + skipCount := (page - 1) * perPage + + apiResults := make([]*ctypes.ResultTx, cmn.MinInt(perPage, totalCount-skipCount)) var proof types.TxProof - for i, r := range results { + for i := 0; i < len(apiResults); i++ { + r := results[skipCount+i] height := r.Height index := r.Index @@ -205,5 +214,5 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { } } - return apiResults, nil + return &ctypes.ResultTxSearch{Txs: apiResults, TotalCount: totalCount}, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 18c54545..5b001d7d 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -172,6 +172,12 @@ type ResultTx struct { Proof types.TxProof `json:"proof,omitempty"` } +// Result of searching for txs +type ResultTxSearch struct { + Txs []*ResultTx `json:"txs"` + TotalCount int `json:"total_count"` +} + // List of mempool txs type ResultUnconfirmedTxs struct { N int `json:"n_txs"` From 5115618550dd24e14e9de65bfe9e65d1c465b435 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 15 May 2018 11:31:35 +0400 Subject: [PATCH 02/34] add limit param to /unconfirmed_txs --- rpc/core/mempool.go | 15 ++++++++++++--- rpc/core/routes.go | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 77c8c844..0e477243 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -209,7 +209,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { } } -// Get unconfirmed transactions including their number. +// Get unconfirmed transactions (maximum ?limit entries) including their number. // // ```shell // curl 'localhost:46657/unconfirmed_txs' @@ -232,9 +232,18 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // "id": "", // "jsonrpc": "2.0" // } +// +// ### Query Parameters +// +// | Parameter | Type | Default | Required | Description | +// |-----------+------+---------+----------+--------------------------------------| +// | limit | int | 30 | false | Maximum number of entries (max: 100) | // ``` -func UnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { - txs := mempool.Reap(-1) +func UnconfirmedTxs(limit int) (*ctypes.ResultUnconfirmedTxs, error) { + // reuse per_page validator + limit = validatePerPage(limit) + + txs := mempool.Reap(limit) return &ctypes.ResultUnconfirmedTxs{len(txs), txs}, nil } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index ed6c6981..f26fadb6 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -26,7 +26,7 @@ var Routes = map[string]*rpc.RPCFunc{ "validators": rpc.NewRPCFunc(Validators, "height"), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""), "consensus_state": rpc.NewRPCFunc(ConsensusState, ""), - "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxs, ""), + "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxs, "limit"), "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxs, ""), // broadcast API From 8d60a5a7bd43bb97b78d059e9a5e65dbd7aac6b4 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 15 May 2018 11:37:48 +0400 Subject: [PATCH 03/34] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c1b47b0..bc9a6344 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.19.4 (TBD) +BREAKING: +- [rpc] `/tx_search` now outputs maximum `?per_page` txs (you can provide custom `?per_page` up to 100; defaults to 30). You can set the `?page` (starts at 1). +- [rpc] `/unconfirmed_txs` now outputs maximum `?limit` txs (you can provide custom `?limit` up to 100; defaults to 30) + ## 0.19.3 (May 14th, 2018) FEATURES From e0dbc3673cb4c607ecf9000c1ccb20384495b6c0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 17 May 2018 22:28:53 -0400 Subject: [PATCH 04/34] version bump --- CHANGELOG.md | 2 ++ version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd319323..258cc35b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 0.19.5 (TBD) + ## 0.19.4 (May 17th, 2018) IMPROVEMENTS diff --git a/version/version.go b/version/version.go index 6bf9bdd4..f0cbb46d 100644 --- a/version/version.go +++ b/version/version.go @@ -4,13 +4,13 @@ package version const ( Maj = "0" Min = "19" - Fix = "4" + Fix = "5" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.19.4" + Version = "0.19.5-dev" // GitCommit is the current HEAD set using ldflags. GitCommit string From 0908e668bd4ab90af0f9f45e00e20d512393a71c Mon Sep 17 00:00:00 2001 From: Zach Date: Fri, 18 May 2018 01:52:31 -0400 Subject: [PATCH 05/34] document testnet command, closes #1535 (#1588) --- docs/deploy-testnets.rst | 28 ++++++++++++---------------- docs/terraform-and-ansible.rst | 2 +- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/deploy-testnets.rst b/docs/deploy-testnets.rst index d7ea97b3..c5df973a 100644 --- a/docs/deploy-testnets.rst +++ b/docs/deploy-testnets.rst @@ -29,23 +29,28 @@ Here are the steps to setting up a testnet manually: ``46656``. Thus, if the IP addresses of your nodes were ``192.168.0.1, 192.168.0.2, 192.168.0.3, 192.168.0.4``, the command would look like: - ``tendermint node --proxy_app=kvstore --p2p.persistent_peers=96663a3dd0d7b9d17d4c8211b191af259621c693@192.168.0.1:46656, 429fcf25974313b95673f58d77eacdd434402665@192.168.0.2:46656, 0491d373a8e0fcf1023aaf18c51d6a1d0d4f31bd@192.168.0.3:46656, f9baeaa15fedf5e1ef7448dd60f46c01f1a9e9c4@192.168.0.4:46656``. + +:: + + tendermint node --proxy_app=kvstore --p2p.persistent_peers=96663a3dd0d7b9d17d4c8211b191af259621c693@192.168.0.1:46656, 429fcf25974313b95673f58d77eacdd434402665@192.168.0.2:46656, 0491d373a8e0fcf1023aaf18c51d6a1d0d4f31bd@192.168.0.3:46656, f9baeaa15fedf5e1ef7448dd60f46c01f1a9e9c4@192.168.0.4:46656 After a few seconds, all the nodes should connect to each other and start making blocks! For more information, see the Tendermint Networks section of `the guide to using Tendermint `__. -While the manual deployment is easy enough, an automated deployment is -usually quicker. The below examples show different tools that can be used -for automated deployments. +But wait! Steps 3 and 4 are quite manual. Instead, use `this script `__, which does the heavy lifting for you. And it gets better. + +Instead of the previously linked script to initialize the files required for a testnet, we have the ``tendermint testnet`` command. By default, running ``tendermint testnet`` will create all the required files, just like the script. Of course, you'll still need to manually edit some fields in the ``config.toml``. Alternatively, see the available flags to auto-populate the ``config.toml`` with the fields that would otherwise be passed in via flags when running ``tendermint node``. As you might imagine, this command is useful for manual or automated deployments. Automated Deployments --------------------- +The easiest and fastest way to get a testnet up in less than 5 minutes. + Local ^^^^^ -With ``docker`` installed, run the command: +With ``docker`` and ``docker-compose`` installed, run the command: :: @@ -53,16 +58,7 @@ With ``docker`` installed, run the command: from the root of the tendermint repository. This will spin up a 4-node local testnet. -Cloud Deployment using Kubernetes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The `mintnet-kubernetes tool `__ -allows automating the deployment of a Tendermint network on an already -provisioned Kubernetes cluster. For simple provisioning of a Kubernetes -cluster, check out the `Google Cloud Platform `__. - -Cloud Deployment using Terraform and Ansible -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Cloud +^^^^^ See the `next section <./terraform-and-ansible.html>`__ for details. - diff --git a/docs/terraform-and-ansible.rst b/docs/terraform-and-ansible.rst index f11b67d5..c47de9d4 100644 --- a/docs/terraform-and-ansible.rst +++ b/docs/terraform-and-ansible.rst @@ -7,7 +7,7 @@ Automated deployments are done using `Terraform `__ t Install ------- -NOTE: see the `integration bash script `__ that can be run on a fresh DO droplet and will automatically spin up a 4 node testnet. The script more or less does everything described below. +NOTE: see the `integration bash script `__ that can be run on a fresh DO droplet and will automatically spin up a 4 node testnet. The script more or less does everything described below. - Install `Terraform `__ and `Ansible `__ on a Linux machine. - Create a `DigitalOcean API token `__ with read and write capability. From ae572b9038bf1086ad4257c9127afa48d49e113e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 17 May 2018 11:52:07 +0400 Subject: [PATCH 06/34] remove extra call to Exists Refs #345 https://github.com/tendermint/tendermint/issues/1539#issuecomment-387240606 --- mempool/mempool.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/mempool/mempool.go b/mempool/mempool.go index aa2aa4f4..70ef7a38 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -203,10 +203,9 @@ func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) { defer mem.proxyMtx.Unlock() // CACHE - if mem.cache.Exists(tx) { + if !mem.cache.Push(tx) { return ErrTxInCache } - mem.cache.Push(tx) // END CACHE // WAL @@ -463,14 +462,6 @@ func (cache *txCache) Reset() { cache.mtx.Unlock() } -// Exists returns true if the given tx is cached. -func (cache *txCache) Exists(tx types.Tx) bool { - cache.mtx.Lock() - _, exists := cache.map_[string(tx)] - cache.mtx.Unlock() - return exists -} - // Push adds the given tx to the txCache. It returns false if tx is already in the cache. func (cache *txCache) Push(tx types.Tx) bool { cache.mtx.Lock() From 90446261f3e38df6c85c86f0b4e00f4017b48820 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 17 May 2018 12:20:29 +0400 Subject: [PATCH 07/34] [docs] document transactional semantics Refs #1494, #345 --- docs/transactional-semantics.rst | 27 +++++++++++++++++++++++++++ docs/using-tendermint.rst | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 docs/transactional-semantics.rst diff --git a/docs/transactional-semantics.rst b/docs/transactional-semantics.rst new file mode 100644 index 00000000..268247a2 --- /dev/null +++ b/docs/transactional-semantics.rst @@ -0,0 +1,27 @@ +Transactional Semantics +======================= + +In `"Using +Tendermint"<./specification/using-tendermint.html#broadcast-api>`__ we +discussed different API endpoints for sending transactions and +differences between them. + +What we have not yet covered is transactional semantics. + +When you send a transaction using one of the available methods, it +first goes to the mempool. Currently, it does not provide strong +guarantees like "if the transaction were accepted, it would be +eventually included in a block (given CheckTx passes)." + +For instance a tx could enter the mempool, but before it can be sent +to peers the node crashes. + +We are planning to provide such guarantees by using a WAL and +replaying transactions (See +`GH#248`__), but +it's non-trivial to do this all efficiently. + +The temporary solution is for clients to monitor the node and resubmit +transaction(s) or/and send them to more nodes at once, so the +probability of all of them crashing at the same time and losing the +msg decreases substantially. diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index 394a7f3e..f572277c 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -214,7 +214,7 @@ Broadcast API Earlier, we used the ``broadcast_tx_commit`` endpoint to send a transaction. When a transaction is sent to a Tendermint node, it will run via ``CheckTx`` against the application. If it passes ``CheckTx``, -it will be included in the mempool, broadcast to other peers, and +it will be included in the mempool, broadcasted to other peers, and eventually included in a block. Since there are multiple phases to processing a transaction, we offer From c9001d5a11dba8de01883128c81f0702a72361fd Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 17 May 2018 13:38:40 +0400 Subject: [PATCH 08/34] bound the mempool Refs #345 --- config/config.go | 2 ++ config/toml.go | 6 ++++++ docs/specification/configuration.rst | 6 ++++++ mempool/mempool.go | 13 +++++++++++-- 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index b76f5ed1..47df4626 100644 --- a/config/config.go +++ b/config/config.go @@ -335,6 +335,7 @@ type MempoolConfig struct { RecheckEmpty bool `mapstructure:"recheck_empty"` Broadcast bool `mapstructure:"broadcast"` WalPath string `mapstructure:"wal_dir"` + Size int `mapstructure:"size"` CacheSize int `mapstructure:"cache_size"` } @@ -345,6 +346,7 @@ func DefaultMempoolConfig() *MempoolConfig { RecheckEmpty: true, Broadcast: true, WalPath: filepath.Join(defaultDataDir, "mempool.wal"), + Size: 100000, CacheSize: 100000, } } diff --git a/config/toml.go b/config/toml.go index a19fb315..3f4c7dda 100644 --- a/config/toml.go +++ b/config/toml.go @@ -179,6 +179,12 @@ recheck_empty = {{ .Mempool.RecheckEmpty }} broadcast = {{ .Mempool.Broadcast }} wal_dir = "{{ .Mempool.WalPath }}" +# size of the mempool +size = {{ .Mempool.Size }} + +# size of the cache (used to filter transactions we saw earlier) +cache_size = {{ .Mempool.CacheSize }} + ##### consensus configuration options ##### [consensus] diff --git a/docs/specification/configuration.rst b/docs/specification/configuration.rst index 6b52dbd1..2282095b 100644 --- a/docs/specification/configuration.rst +++ b/docs/specification/configuration.rst @@ -136,6 +136,12 @@ like the file below, however, double check by inspecting the broadcast = true wal_dir = "data/mempool.wal" + # size of the mempool + size = 100000 + + # size of the cache (used to filter transactions we saw earlier) + cache_size = 100000 + ##### consensus configuration options ##### [consensus] diff --git a/mempool/mempool.go b/mempool/mempool.go index 70ef7a38..f4ecf00f 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -49,7 +49,13 @@ TODO: Better handle abci client errors. (make it automatically handle connection */ -var ErrTxInCache = errors.New("Tx already exists in cache") +var ( + // ErrTxInCache is returned to the client if we saw tx earlier + ErrTxInCache = errors.New("Tx already exists in cache") + + // ErrMempoolIsFull means Tendermint & an application can't handle that much load + ErrMempoolIsFull = errors.New("Mempool is full") +) // Mempool is an ordered in-memory pool for transactions before they are proposed in a consensus // round. Transaction validity is checked using the CheckTx abci message before the transaction is @@ -80,7 +86,6 @@ type Mempool struct { } // NewMempool returns a new Mempool with the given configuration and connection to an application. -// TODO: Extract logger into arguments. func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool, height int64) *Mempool { mempool := &Mempool{ config: config, @@ -202,6 +207,10 @@ func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) { mem.proxyMtx.Lock() defer mem.proxyMtx.Unlock() + if mem.Size() >= mem.config.Size { + return ErrMempoolIsFull + } + // CACHE if !mem.cache.Push(tx) { return ErrTxInCache From 2987158a653bfb0ce72c78621a2e42373cd2372d Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 18 May 2018 10:43:46 +0400 Subject: [PATCH 09/34] [docs] add a note about replay protection Refs #345, https://github.com/tendermint/tendermint/issues/458#issuecomment-297077148 --- docs/app-development.rst | 46 ++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/docs/app-development.rst b/docs/app-development.rst index 42083f74..9aefa99c 100644 --- a/docs/app-development.rst +++ b/docs/app-development.rst @@ -178,21 +178,22 @@ connection, to query the local state of the app. Mempool Connection ~~~~~~~~~~~~~~~~~~ -The mempool connection is used *only* for CheckTx requests. Transactions -are run using CheckTx in the same order they were received by the -validator. If the CheckTx returns ``OK``, the transaction is kept in -memory and relayed to other peers in the same order it was received. -Otherwise, it is discarded. +The mempool connection is used *only* for CheckTx requests. +Transactions are run using CheckTx in the same order they were +received by the validator. If the CheckTx returns ``OK``, the +transaction is kept in memory and relayed to other peers in the same +order it was received. Otherwise, it is discarded. -CheckTx requests run concurrently with block processing; so they should -run against a copy of the main application state which is reset after -every block. This copy is necessary to track transitions made by a -sequence of CheckTx requests before they are included in a block. When a -block is committed, the application must ensure to reset the mempool -state to the latest committed state. Tendermint Core will then filter -through all transactions in the mempool, removing any that were included -in the block, and re-run the rest using CheckTx against the post-Commit -mempool state. +CheckTx requests run concurrently with block processing; so they +should run against a copy of the main application state which is reset +after every block. This copy is necessary to track transitions made by +a sequence of CheckTx requests before they are included in a block. +When a block is committed, the application must ensure to reset the +mempool state to the latest committed state. Tendermint Core will then +filter through all transactions in the mempool, removing any that were +included in the block, and re-run the rest using CheckTx against the +post-Commit mempool state (this behaviour can be turned off with +``[mempool] recheck = false``). .. container:: toggle @@ -226,6 +227,23 @@ mempool state. } } +Replay Protection +^^^^^^^^^^^^^^^^^ +To prevent old transactions from being replayed, CheckTx must +implement replay protection. + +Tendermint provides the first defence layer by keeping a lightweight +in-memory cache of 100k (``[mempool] cache_size``) last transactions in +the mempool. If Tendermint is just started or the clients sent more +than 100k transactions, old transactions may be sent to the +application. So it is important CheckTx implements some logic to +handle them. + +There are cases where a transaction will (or may) become valid in some +future state, in which case you probably want to disable Tendermint's +cache. You can do that by setting ``[mempool] cache_size = 0`` in the +config. + Consensus Connection ~~~~~~~~~~~~~~~~~~~~ From 202a43a5af4675019adbfd601bfa14c883ff5e07 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 18 May 2018 10:48:13 +0400 Subject: [PATCH 10/34] remove TODO no longer relevant, I guess. since ABCI only defines 0 (success) code. --- mempool/mempool.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/mempool/mempool.go b/mempool/mempool.go index f4ecf00f..938fb2a7 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -272,8 +272,6 @@ func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) { // remove from cache (it might be good later) mem.cache.Remove(tx) - - // TODO: handle other retcodes } default: // ignore other messages From 5a041baa36848fa89cc37d836965a3f987f562eb Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 18 May 2018 12:45:09 +0400 Subject: [PATCH 11/34] nice output for msgBytes Closes #1227 --- p2p/conn/connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 6684941a..6e08c67f 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -523,7 +523,7 @@ FOR_LOOP: break FOR_LOOP } if msgBytes != nil { - c.Logger.Debug("Received bytes", "chID", pkt.ChannelID, "msgBytes", msgBytes) + c.Logger.Debug("Received bytes", "chID", pkt.ChannelID, "msgBytes", fmt.Sprintf("%X", msgBytes)) // NOTE: This means the reactor.Receive runs in the same thread as the p2p recv routine c.onReceive(pkt.ChannelID, msgBytes) } From 35428ceb539af5deacd0ddaa7395f07c518a0e22 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Fri, 18 May 2018 10:01:56 -0400 Subject: [PATCH 12/34] docs: lil fixes --- docs/conf.py | 8 +------- docs/index.rst | 2 +- docs/transactional-semantics.rst | 6 +++--- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8fefec4d..08617378 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -186,14 +186,8 @@ if os.path.isdir(assets_dir) != True: urllib.urlretrieve(tools_repo+tools_branch+'/docker/README.rst', filename=tools_dir+'/docker.rst') -urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/README.rst', filename=tools_dir+'/mintnet-kubernetes.rst') -urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/gce1.png', filename=assets_dir+'/gce1.png') -urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/gce2.png', filename=assets_dir+'/gce2.png') -urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/statefulset.png', filename=assets_dir+'/statefulset.png') -urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/t_plus_k.png', filename=assets_dir+'/t_plus_k.png') - urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking.rst') -urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/monitoring.rst') +urllib.urlretrieve(tools_repo+tools_branch+'/tm-monitor/README.rst', filename='tools/monitoring.rst') #### abci spec ################################# diff --git a/docs/index.rst b/docs/index.rst index 99ca11e7..2bed0744 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,7 +42,6 @@ Tendermint Tools deploy-testnets.rst terraform-and-ansible.rst tools/docker.rst - tools/mintnet-kubernetes.rst tools/benchmarking.rst tools/monitoring.rst @@ -66,6 +65,7 @@ Tendermint 201 specification.rst determinism.rst + transactional-semantics.rst * For a deeper dive, see `this thesis `__. * There is also the `original whitepaper `__, though it is now quite outdated. diff --git a/docs/transactional-semantics.rst b/docs/transactional-semantics.rst index 268247a2..988ac682 100644 --- a/docs/transactional-semantics.rst +++ b/docs/transactional-semantics.rst @@ -1,8 +1,8 @@ Transactional Semantics ======================= -In `"Using -Tendermint"<./specification/using-tendermint.html#broadcast-api>`__ we +In `Using +Tendermint <./using-tendermint.html#broadcast-api>`__ we discussed different API endpoints for sending transactions and differences between them. @@ -18,7 +18,7 @@ to peers the node crashes. We are planning to provide such guarantees by using a WAL and replaying transactions (See -`GH#248`__), but +`GH#248 `__), but it's non-trivial to do this all efficiently. The temporary solution is for clients to monitor the node and resubmit From 0fb33ca91d13e448e7b0af36df0f2dd48c432b29 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Fri, 18 May 2018 10:12:52 -0400 Subject: [PATCH 13/34] docs: update install instructions, closes #1580 --- docs/install.rst | 65 +++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index b2aae676..3fc392a3 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -4,53 +4,48 @@ Install Tendermint From Binary ----------- -To download pre-built binaries, see the `Download page `__. +To download pre-built binaries, see the `releases page `__. From Source ----------- -You'll need ``go``, maybe `dep `__, and the Tendermint source code. - -Install Go -^^^^^^^^^^ - -Make sure you have `installed Go `__ and -set the ``GOPATH``. You should also put ``GOPATH/bin`` on your ``PATH``. +You'll need ``go`` `installed `__ and the required +`environment variables set `__ Get Source Code ^^^^^^^^^^^^^^^ -You should be able to install the latest with a simple +:: + + mkdir -p $GOPATH/src/github.com/tendermint + cd $GOPATH/src/github.com/tendermint + git clone https://github.com/tendermint/tendermint.git + cd tendermint + +Get Tools & Dependencies +^^^^^^^^^^^^^^^^^^^^^^^^ :: - go get github.com/tendermint/tendermint/cmd/tendermint - -Run ``tendermint --help`` and ``tendermint version`` to ensure your -installation worked. - -If the installation failed, a dependency may have been updated and become -incompatible with the latest Tendermint master branch. We solve this -using the ``dep`` tool for dependency management. - -First, install ``dep``: - -:: - - cd $GOPATH/src/github.com/tendermint/tendermint make get_tools + make get_vendor_deps -Now we can fetch the correct versions of each dependency by running: +Compile +^^^^^^^ :: - make get_vendor_deps make install -Note that even though ``go get`` originally failed, the repository was -still cloned to the correct location in the ``$GOPATH``. +to put the binary in ``$GOPATH/bin`` or use: -The latest Tendermint Core version is now installed. +:: + + make build + +to put the binary in ``./build``. + +The latest ``tendermint version`` is now installed. Reinstall --------- @@ -86,20 +81,6 @@ do, use ``dep``, as above: Since the third option just uses ``dep`` right away, it should always work. -Troubleshooting ---------------- - -If ``go get`` failing bothers you, fetch the code using ``git``: - -:: - - mkdir -p $GOPATH/src/github.com/tendermint - git clone https://github.com/tendermint/tendermint $GOPATH/src/github.com/tendermint/tendermint - cd $GOPATH/src/github.com/tendermint/tendermint - make get_tools - make get_vendor_deps - make install - Run ^^^ From 595fc24c568d7acbc9ff73d4e9c94d866fd12282 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Fri, 18 May 2018 13:22:04 -0400 Subject: [PATCH 14/34] remove very old bash file --- Makefile | 2 +- test/docker/update.sh | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 test/docker/update.sh diff --git a/Makefile b/Makefile index 05d1889f..60a81fe3 100755 --- a/Makefile +++ b/Makefile @@ -212,7 +212,7 @@ sentry-start: cd networks/remote/terraform && terraform init && terraform apply -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" @if ! [ -f $(CURDIR)/build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 0 --n 4 --o . ; fi cd networks/remote/ansible && ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i inventory/digital_ocean.py -l sentrynet install.yml - @echo "Next step: Add your validator setup in the genesis.json and config.tml files and run \"make server-config\". (Public key of validator, chain ID, peer IP and node ID.)" + @echo "Next step: Add your validator setup in the genesis.json and config.tml files and run \"make sentry-config\". (Public key of validator, chain ID, peer IP and node ID.)" # Configuration management sentry-config: diff --git a/test/docker/update.sh b/test/docker/update.sh deleted file mode 100644 index b946bf39..00000000 --- a/test/docker/update.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -# update the `tester` image by copying in the latest tendermint binary - -docker run --name builder tester true -docker cp $GOPATH/bin/tendermint builder:/go/bin/tendermint -docker commit builder tester -docker rm -vf builder - From ca120798e448abf578123fe80411813193d0650a Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Fri, 18 May 2018 13:26:26 -0400 Subject: [PATCH 15/34] s/dummy/kvstore/g for compatibility (#1532) --- networks/local/localnode/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networks/local/localnode/Dockerfile b/networks/local/localnode/Dockerfile index c4556caa..3c9ddf54 100644 --- a/networks/local/localnode/Dockerfile +++ b/networks/local/localnode/Dockerfile @@ -9,7 +9,7 @@ VOLUME [ /tendermint ] WORKDIR /tendermint EXPOSE 46656 46657 ENTRYPOINT ["/usr/bin/wrapper.sh"] -CMD ["node", "--proxy_app", "dummy"] +CMD ["node", "--proxy_app", "kvstore"] STOPSIGNAL SIGTERM COPY wrapper.sh /usr/bin/wrapper.sh From 0b68ec4b8ec7586a1c79ce5265790f8bfc2ad672 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Fri, 18 May 2018 13:37:58 -0400 Subject: [PATCH 16/34] move a file to remove a directory --- .circleci/config.yml | 2 +- test/{circleci/p2p.sh => p2p/circleci.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename test/{circleci/p2p.sh => p2p/circleci.sh} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8dac95a8..177c1458 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -153,7 +153,7 @@ jobs: - checkout - run: mkdir -p $GOPATH/src/github.com/tendermint - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - - run: bash test/circleci/p2p.sh + - run: bash test/p2p/circleci.sh upload_coverage: <<: *defaults diff --git a/test/circleci/p2p.sh b/test/p2p/circleci.sh similarity index 100% rename from test/circleci/p2p.sh rename to test/p2p/circleci.sh From 0e1f730fbbeb706825763b7b55d73a3b15a1bc64 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Fri, 18 May 2018 13:59:30 -0400 Subject: [PATCH 17/34] rm unused file, fix docker-compose.yml --- docker-compose.yml | 16 ++++++++-------- test/p2p/clean.sh | 5 ----- 2 files changed, 8 insertions(+), 13 deletions(-) delete mode 100644 test/p2p/clean.sh diff --git a/docker-compose.yml b/docker-compose.yml index b9a3fc68..ebbd5062 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,9 @@ services: - "46656-46657:46656-46657" environment: - ID=0 - - LOG=${LOG:-tendermint.log} + - LOG=$${LOG:-tendermint.log} volumes: - - ${FOLDER:-./build}:/tendermint:Z + - ./build:/tendermint:Z networks: localnet: ipv4_address: 192.167.10.2 @@ -22,9 +22,9 @@ services: - "46659-46660:46656-46657" environment: - ID=1 - - LOG=${LOG:-tendermint.log} + - LOG=$${LOG:-tendermint.log} volumes: - - ${FOLDER:-./build}:/tendermint:Z + - ./build:/tendermint:Z networks: localnet: ipv4_address: 192.167.10.3 @@ -34,11 +34,11 @@ services: image: "tendermint/localnode" environment: - ID=2 - - LOG=${LOG:-tendermint.log} + - LOG=$${LOG:-tendermint.log} ports: - "46661-46662:46656-46657" volumes: - - ${FOLDER:-./build}:/tendermint:Z + - ./build:/tendermint:Z networks: localnet: ipv4_address: 192.167.10.4 @@ -48,11 +48,11 @@ services: image: "tendermint/localnode" environment: - ID=3 - - LOG=${LOG:-tendermint.log} + - LOG=$${LOG:-tendermint.log} ports: - "46663-46664:46656-46657" volumes: - - ${FOLDER:-./build}:/tendermint:Z + - ./build:/tendermint:Z networks: localnet: ipv4_address: 192.167.10.5 diff --git a/test/p2p/clean.sh b/test/p2p/clean.sh deleted file mode 100644 index 44a1276e..00000000 --- a/test/p2p/clean.sh +++ /dev/null @@ -1,5 +0,0 @@ -#! /bin/bash - -# clean everything -docker rm -vf $(docker ps -aq) -docker network rm local_testnet From 6c4a26f248b5a2f385128167483387fb3c8ecd97 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 May 2018 22:44:29 -0400 Subject: [PATCH 18/34] update readme --- README.md | 17 ++++++++--------- docs/specification/new-spec/README.md | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 89c28e9c..c230e6f9 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,7 @@ _NOTE: This is alpha software. Please contact us if you intend to run it in prod Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language - and securely replicates it on many machines. -For more information, from introduction to installation and application development, [Read The Docs](https://tendermint.readthedocs.io/en/master/). - -For protocol details, see [the specification](./docs/specification/new-spec). +For protocol details, see [the specification](/docs/spec). ## Minimum requirements @@ -36,19 +34,20 @@ Go version | Go1.9 or higher ## Install -To download pre-built binaries, see our [downloads page](https://tendermint.com/downloads). +See the [install instructions](/docs/install.rst) -To install from source, you should be able to: +## Documentation -`go get -u github.com/tendermint/tendermint/cmd/tendermint` - -For more details (or if it fails), [read the docs](https://tendermint.readthedocs.io/en/master/install.html). +- [Run a single Tendermint + node](/docs/using-tendermint.rst) +- [Run a local Tendermint cluster using docker-compose](/networks/local) +- [Run a remote Tendermint cluster using terraform and ansible](/networks/remote) ## Resources ### Tendermint Core -To use Tendermint, build apps on it, or develop it, [Read The Docs](https://tendermint.readthedocs.io/en/master/). +For more on Tendermint and how to build apps, [Read The Docs](https://tendermint.readthedocs.io/en/master/). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. ### Sub-projects diff --git a/docs/specification/new-spec/README.md b/docs/specification/new-spec/README.md index 20e8e89d..f5ebd271 100644 --- a/docs/specification/new-spec/README.md +++ b/docs/specification/new-spec/README.md @@ -1 +1 @@ -Spec moved to [docs/spec](./docs/spec). +Spec moved to [docs/spec](/docs/spec). From d7d12c8030b16418de1a2a4875412352c934d55e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 May 2018 23:21:19 -0400 Subject: [PATCH 19/34] update networks/local readme --- Makefile | 4 +++ docs/using-tendermint.rst | 7 ++-- networks/local/README.rst | 74 +++++++++++++++++++++++++++++++-------- 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 60a81fe3..ce3b9ed5 100755 --- a/Makefile +++ b/Makefile @@ -193,6 +193,10 @@ build-docker: build-linux: GOOS=linux GOARCH=amd64 $(MAKE) build +build-docker-localnode: + cd networks/local + make + # Run a 4-node testnet locally localnet-start: localnet-stop @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index f572277c..56790fea 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -28,8 +28,11 @@ genesis file (``genesis.json``) containing the associated public key, in ``$TMHOME/config``. This is all that's necessary to run a local testnet with one validator. -For more elaborate initialization, see our `testnet deployment -tool `__. +For more elaborate initialization, see the `tesnet` command: + +:: + + tendermint testnet --help Run --- diff --git a/networks/local/README.rst b/networks/local/README.rst index d22a24d9..601d238d 100644 --- a/networks/local/README.rst +++ b/networks/local/README.rst @@ -1,29 +1,79 @@ localnode ========= -It is assumed that you have already `setup docker `__. +Requirements +------------ -Description ------------ -Image for local testnets. +- `Install docker `__. +- `Install docker-compose `__. -Add the tendermint binary to the image by attaching it in a folder to the `/tendermint` mount point. +Build +----- -It assumes that the configuration was created by the `tendermint testnet` command and it is also attached to the `/tendermint` mount point. +Build the `tendermint` binary and the `tendermint/localnode` docker image: -Example: -This example builds a linux tendermint binary under the `build/` folder, creates tendermint configuration for a single-node validator and runs the node: ``` cd $GOPATH/src/github.com/tendermint/tendermint -#Build binary +# Install dependencies (skip if already done) +make get_tools +make get_vendor_deps + +# Build binary in ./build make build-linux -#Create configuration +# Build tendermint/localnode image +make build-docker-localnode + +``` + +Run a testnet +------------- + +To start a 4 node testnet run: + +``` +make localnet-start + +`` + +The nodes bind their RPC servers to ports 46657, 46660, 46662, and 46664 on the host. +This file creates a 4-node network using the localnode image. +The nodes of the network expose their P2P and RPC endpoints to the host machine on ports 46656-46657, 46659-46660, 46661-46662, and 46663-46664 respectively. + +To update the binary, just rebuild it and restart the nodes: + +``` +make build-linux +make localnet-stop +make localnet-start + +``` + +Configuration +----------- + +The `make localnet-start` creates files for a 4-node testnet in `./build` by calling the `tendermint testnet` command. + +The `./build` directory is mounted to the `/tendermint` mount point to attach the binary and config files to the container. + +For instance, to create a single node testnet: + +``` +cd $GOPATH/src/github.com/tendermint/tendermint + +# Clear the build folder +rm -rf ./build + +# Build binary +make build-linux + +# Create configuration docker run -e LOG="stdout" -v `pwd`/build:/tendermint tendermint/localnode testnet --o . --v 1 #Run the node docker run -v `pwd`/build:/tendermint tendermint/localnode + ``` Logging @@ -34,7 +84,3 @@ Special binaries ---------------- If you have multiple binaries with different names, you can specify which one to run with the BINARY environment variable. The path of the binary is relative to the attached volume. -docker-compose.yml -================== -This file creates a 4-node network using the localnode image. The nodes of the network are exposed to the host machine on ports 46656-46657, 46659-46660, 46661-46662, 46663-46664 respectively. - From 420f925a4d78b59bdeefa77766dfd807b4d800e7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 May 2018 23:21:42 -0400 Subject: [PATCH 20/34] link bug bounty in readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c230e6f9..c5375060 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,9 @@ and securely replicates it on many machines. For protocol details, see [the specification](/docs/spec). +To report a security vulnerability, see our [bug bounty +program](https://tendermint.com/security). + ## Minimum requirements Requirement|Notes From d76e2dc3ff0d27d851a8658decfff40999878f04 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 May 2018 23:26:02 -0400 Subject: [PATCH 21/34] readme.rst -> readme.md --- networks/local/{README.rst => README.md} | 32 +++++++++--------------- 1 file changed, 12 insertions(+), 20 deletions(-) rename networks/local/{README.rst => README.md} (85%) diff --git a/networks/local/README.rst b/networks/local/README.md similarity index 85% rename from networks/local/README.rst rename to networks/local/README.md index 601d238d..af3b6c87 100644 --- a/networks/local/README.rst +++ b/networks/local/README.md @@ -1,14 +1,11 @@ -localnode -========= +# Local Docker Testnet -Requirements ------------- +## Requirements -- `Install docker `__. -- `Install docker-compose `__. +- [Install docker](https://docs.docker.com/engine/installation/) +- [Install docker-compose](https://docs.docker.com/compose/install/) -Build ------ +## Build Build the `tendermint` binary and the `tendermint/localnode` docker image: @@ -24,18 +21,15 @@ make build-linux # Build tendermint/localnode image make build-docker-localnode - ``` -Run a testnet -------------- +## Run a testnet To start a 4 node testnet run: ``` make localnet-start - -`` +``` The nodes bind their RPC servers to ports 46657, 46660, 46662, and 46664 on the host. This file creates a 4-node network using the localnode image. @@ -47,11 +41,9 @@ To update the binary, just rebuild it and restart the nodes: make build-linux make localnet-stop make localnet-start - ``` -Configuration ------------ +## Configuration The `make localnet-start` creates files for a 4-node testnet in `./build` by calling the `tendermint testnet` command. @@ -76,11 +68,11 @@ docker run -v `pwd`/build:/tendermint tendermint/localnode ``` -Logging -------- +## Logging + Log is saved under the attached volume, in the `tendermint.log` file. If the `LOG` environment variable is set to `stdout` at start, the log is not saved, but printed on the screen. -Special binaries ----------------- +## Special binaries + If you have multiple binaries with different names, you can specify which one to run with the BINARY environment variable. The path of the binary is relative to the attached volume. From 26fdfe10fd64c8c0cc05b87203a17a86b22ff4d9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 May 2018 23:28:27 -0400 Subject: [PATCH 22/34] update readme --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c5375060..0c52f5f7 100644 --- a/README.md +++ b/README.md @@ -39,18 +39,17 @@ Go version | Go1.9 or higher See the [install instructions](/docs/install.rst) -## Documentation +## Quick Start -- [Run a single Tendermint - node](/docs/using-tendermint.rst) -- [Run a local Tendermint cluster using docker-compose](/networks/local) -- [Run a remote Tendermint cluster using terraform and ansible](/networks/remote) +- [Single node](/docs/using-tendermint.rst) +- [Local cluster using docker-compose](/networks/local) +- [Remote cluster using terraform and ansible](/networks/remote) ## Resources ### Tendermint Core -For more on Tendermint and how to build apps, [Read The Docs](https://tendermint.readthedocs.io/en/master/). +For more, [Read The Docs](https://tendermint.readthedocs.io/en/master/). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. ### Sub-projects From 773e3917ecc821f49adc6bf23525e13b0b4962e4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 May 2018 23:35:45 -0400 Subject: [PATCH 23/34] networks: update readmes --- networks/local/README.md | 15 ++++++++------- networks/remote/README.md | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 networks/remote/README.md diff --git a/networks/local/README.md b/networks/local/README.md index af3b6c87..528247f1 100644 --- a/networks/local/README.md +++ b/networks/local/README.md @@ -1,28 +1,29 @@ -# Local Docker Testnet +# Local Cluster with Docker Compose ## Requirements +- [Install tendermint](/docs/install.rst) - [Install docker](https://docs.docker.com/engine/installation/) - [Install docker-compose](https://docs.docker.com/compose/install/) ## Build -Build the `tendermint` binary and the `tendermint/localnode` docker image: +Build the `tendermint` binary and the `tendermint/localnode` docker image. + +Note the binary will be mounted into the container so it can be updated without +rebuilding the image. ``` cd $GOPATH/src/github.com/tendermint/tendermint -# Install dependencies (skip if already done) -make get_tools -make get_vendor_deps - -# Build binary in ./build +# Build the linux binary in ./build make build-linux # Build tendermint/localnode image make build-docker-localnode ``` + ## Run a testnet To start a 4 node testnet run: diff --git a/networks/remote/README.md b/networks/remote/README.md new file mode 100644 index 00000000..a5c0d110 --- /dev/null +++ b/networks/remote/README.md @@ -0,0 +1 @@ +# Remote Cluster with Terraform and Ansible From 1ef415728d583bc05c5f7f00469aa399c837f44a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 00:28:47 -0400 Subject: [PATCH 24/34] docs/spec: blockchain and consensus dirs --- docs/spec/README.md | 13 +++++++++---- docs/spec/{ => blockchain}/blockchain.md | 0 docs/spec/{ => blockchain}/encoding.md | 0 docs/spec/{ => blockchain}/light-client.md | 0 docs/spec/{ => blockchain}/pre-amino.md | 0 docs/spec/{ => blockchain}/state.md | 0 docs/spec/{ => consensus}/abci.md | 0 docs/spec/{ => consensus}/bft-time.md | 0 8 files changed, 9 insertions(+), 4 deletions(-) rename docs/spec/{ => blockchain}/blockchain.md (100%) rename docs/spec/{ => blockchain}/encoding.md (100%) rename docs/spec/{ => blockchain}/light-client.md (100%) rename docs/spec/{ => blockchain}/pre-amino.md (100%) rename docs/spec/{ => blockchain}/state.md (100%) rename docs/spec/{ => consensus}/abci.md (100%) rename docs/spec/{ => consensus}/bft-time.md (100%) diff --git a/docs/spec/README.md b/docs/spec/README.md index 12f5525c..e13e65c1 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -10,12 +10,17 @@ please submit them to our [bug bounty](https://tendermint.com/security)! ## Contents +- [Overview](#overview) + ### Data Structures -- [Overview](#overview) -- [Encoding and Digests](encoding.md) -- [Blockchain](blockchain.md) -- [State](state.md) +- [Encoding and Digests](./blockchain/encoding.md) +- [Blockchain](./blockchain/blockchain.md) +- [State](./blockchain/state.md) + +### Consensus Protocol + +- TODO ### P2P and Network Protocols diff --git a/docs/spec/blockchain.md b/docs/spec/blockchain/blockchain.md similarity index 100% rename from docs/spec/blockchain.md rename to docs/spec/blockchain/blockchain.md diff --git a/docs/spec/encoding.md b/docs/spec/blockchain/encoding.md similarity index 100% rename from docs/spec/encoding.md rename to docs/spec/blockchain/encoding.md diff --git a/docs/spec/light-client.md b/docs/spec/blockchain/light-client.md similarity index 100% rename from docs/spec/light-client.md rename to docs/spec/blockchain/light-client.md diff --git a/docs/spec/pre-amino.md b/docs/spec/blockchain/pre-amino.md similarity index 100% rename from docs/spec/pre-amino.md rename to docs/spec/blockchain/pre-amino.md diff --git a/docs/spec/state.md b/docs/spec/blockchain/state.md similarity index 100% rename from docs/spec/state.md rename to docs/spec/blockchain/state.md diff --git a/docs/spec/abci.md b/docs/spec/consensus/abci.md similarity index 100% rename from docs/spec/abci.md rename to docs/spec/consensus/abci.md diff --git a/docs/spec/bft-time.md b/docs/spec/consensus/bft-time.md similarity index 100% rename from docs/spec/bft-time.md rename to docs/spec/consensus/bft-time.md From 2df137193c65efc7459dfb1b1047f53e65a3b026 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 00:29:28 -0400 Subject: [PATCH 25/34] security.md --- README.md | 2 +- SECURITY.md | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 SECURITY.md diff --git a/README.md b/README.md index 0c52f5f7..f1aab737 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ and securely replicates it on many machines. For protocol details, see [the specification](/docs/spec). To report a security vulnerability, see our [bug bounty -program](https://tendermint.com/security). +program](SECURITY.md). ## Minimum requirements diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..8b979378 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,71 @@ +# Security + +As part of our [Coordinated Vulnerability Disclosure +Policy](https://tendermint.com/security), we operate a bug bounty. +See the policy for more details on submissions and rewards. + +Here is a list of examples of the kinds of bugs we're most interested in: + +## Specification + +- Conceptual flaws +- Ambiguities, inconsistencies, or incorrect statements +- Mis-match between specification and implementation of any component + +## Consensus + +Assuming less than 1/3 of the voting power is Byzantine (malicious): + +- Validation of blockchain data structures, including blocks, block parts, + votes, and so on +- Execution of blocks +- Validator set changes +- Proposer round robin +- Two nodes committing conflicting blocks for the same height (safety failure) +- A correct node signing conflicting votes +- A node halting (liveness failure) +- Syncing new and old nodes + +## Networking + +- Authenticated encryption (MITM, information leakage) +- Eclipse attacks +- Sybil attacks +- Long-range attacks +- Denial-of-Service + +## RPC + +- Write-access to anything besides sending transactions +- Denial-of-Service +- Leakage of secrets + +## Denial-of-Service + +Attacks may come through the P2P network or the RPC: + +- Amplification attacks +- Resource abuse +- Deadlocks and race conditions +- Panics and unhandled errors + +## Libraries + +- Serialization (Amino) +- Reading/Writing files and databases +- Logging and monitoring + +## Cryptography + +- Elliptic curves for validator signatures +- Hash algorithms and Merkle trees for block validation +- Authenticated encryption for P2P connections + +## Light Client + +- Validation of blockchain data structures +- Correctly validating an incorrect proof +- Incorrectly validating a correct proof +- Syncing validator set changes + + From 02615c86951fbb919e2f0e2fb4aba760eb23f6e8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 00:39:34 -0400 Subject: [PATCH 26/34] update readme --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1aab737..6f860606 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,12 @@ and securely replicates it on many machines. For protocol details, see [the specification](/docs/spec). +## Security + To report a security vulnerability, see our [bug bounty -program](SECURITY.md). +program](https://tendermint.com/security). + +For examples of the kinds of bugs we're looking for, see [SECURITY.md](SECURITY.md) ## Minimum requirements From 6f9867cba65f806eeb652606bd4010ed04e0e07c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 01:35:42 -0400 Subject: [PATCH 27/34] fix validate pagination params --- rpc/core/pipe.go | 8 ++++-- rpc/core/pipe_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++ rpc/core/tx.go | 2 +- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 rpc/core/pipe_test.go diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index f97d1817..e4bb5a29 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -124,10 +124,14 @@ func SetEventBus(b *types.EventBus) { eventBus = b } -func validatePage(page int) int { +func validatePage(page, perPage, totalCount int) int { + pages := ((totalCount - 1) / perPage) + 1 if page < 1 { - return 1 + page = 1 + } else if page > pages { + page = pages } + return page } diff --git a/rpc/core/pipe_test.go b/rpc/core/pipe_test.go new file mode 100644 index 00000000..a33e17cf --- /dev/null +++ b/rpc/core/pipe_test.go @@ -0,0 +1,67 @@ +package core + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPaginationPage(t *testing.T) { + + cases := []struct { + totalCount int + perPage int + page int + newPage int + }{ + {0, 10, 0, 1}, + {0, 10, 1, 1}, + {0, 10, 2, 1}, + + {5, 10, -1, 1}, + {5, 10, 0, 1}, + {5, 10, 1, 1}, + {5, 10, 2, 1}, + {5, 10, 2, 1}, + + {5, 5, 1, 1}, + {5, 5, 2, 1}, + {5, 5, 3, 1}, + + {5, 3, 2, 2}, + {5, 3, 3, 2}, + + {5, 2, 2, 2}, + {5, 2, 3, 3}, + {5, 2, 4, 3}, + } + + for _, c := range cases { + p := validatePage(c.page, c.perPage, c.totalCount) + assert.Equal(t, c.newPage, p, fmt.Sprintf("%v", c)) + } + +} + +func TestPaginationPerPage(t *testing.T) { + + cases := []struct { + totalCount int + perPage int + newPerPage int + }{ + {5, 0, defaultPerPage}, + {5, 1, 1}, + {5, 2, 2}, + {5, defaultPerPage, defaultPerPage}, + {5, maxPerPage - 1, maxPerPage - 1}, + {5, maxPerPage, maxPerPage}, + {5, maxPerPage + 1, defaultPerPage}, + } + + for _, c := range cases { + p := validatePerPage(c.perPage) + assert.Equal(t, c.newPerPage, p, fmt.Sprintf("%v", c)) + } +} diff --git a/rpc/core/tx.go b/rpc/core/tx.go index ba68331e..5fc01a86 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -188,7 +188,7 @@ func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSear } totalCount := len(results) - page = validatePage(page) + page = validatePage(page, perPage, totalCount) perPage = validatePerPage(perPage) skipCount := (page - 1) * perPage From 301aa92f9c7a9e50375aaf06b956205b8e4477f4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 09:53:38 -0400 Subject: [PATCH 28/34] phony --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ce3b9ed5..991bfb26 100755 --- a/Makefile +++ b/Makefile @@ -229,5 +229,5 @@ sentry-stop: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker sentry-start sentry-config sentry-stop +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop From 6701dba87609e1fe0483d2a179fc4e24c4e8c6ce Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 10:22:22 -0400 Subject: [PATCH 29/34] changelog --- CHANGELOG.md | 23 ++++++++++++++++++++++- README.md | 6 +++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8321523f..bf3e26b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,27 @@ # Changelog -## 0.19.5 (TBD) +## 0.19.5 + +May 20th, 2018 + +BREAKING CHANGES + +- [rpc/client] TxSearch and UnconfirmedTxs have new arguments (see below) +- [version] Breaking changes to Go APIs will not be reflected in breaking + version change, but will be included in changelog. + +FEATURES + +- [rpc] `/tx_search` takes `page` and `per_page` args to paginate results +- [rpc] `/unconfirmed_txs` takes `limit` arg to limit the output + +IMPROVEMENTS + +- [docs] Lots of updates + +BUG FIXES + +- [mempool] Enforce upper bound on number of transactions ## 0.19.4 (May 17th, 2018) diff --git a/README.md b/README.md index 6f860606..b0505b9c 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,11 @@ According to SemVer, anything in the public API can change at any time before ve To provide some stability to Tendermint users in these 0.X.X days, the MINOR version is used to signal breaking changes across a subset of the total public API. This subset includes all -interfaces exposed to other processes (cli, rpc, p2p, etc.), as well as parts of the following packages: +interfaces exposed to other processes (cli, rpc, p2p, etc.), but does not +include the in-process Go APIs. + +That said, breaking changes in the following packages will be documented in the +CHANGELOG even if they don't lead to MINOR version bumps: - types - rpc/client From 87cefb724dcffa1b7637ea616fba05f046f0f08f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 10:22:42 -0400 Subject: [PATCH 30/34] version --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index f0cbb46d..f5d77dc4 100644 --- a/version/version.go +++ b/version/version.go @@ -10,7 +10,7 @@ const ( var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.19.5-dev" + Version = "0.19.5" // GitCommit is the current HEAD set using ldflags. GitCommit string From 0a9dc9f875c5ed870473be8541ea25a4451a2081 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 10:25:21 -0400 Subject: [PATCH 31/34] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf3e26b3..24cb6704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.19.5 -May 20th, 2018 +*May 20th, 2018* BREAKING CHANGES @@ -14,6 +14,7 @@ FEATURES - [rpc] `/tx_search` takes `page` and `per_page` args to paginate results - [rpc] `/unconfirmed_txs` takes `limit` arg to limit the output +- [config] `mempool.size` and `mempool.cache_size` options IMPROVEMENTS From 2c40966e46f08e185c8dc9f210fe98a677be6732 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 14:08:35 -0400 Subject: [PATCH 32/34] fix changelog --- CHANGELOG.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24cb6704..507b8319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,14 @@ BREAKING CHANGES - [rpc/client] TxSearch and UnconfirmedTxs have new arguments (see below) +- [rpc/client] TxSearch returns ResultTxSearch - [version] Breaking changes to Go APIs will not be reflected in breaking version change, but will be included in changelog. FEATURES -- [rpc] `/tx_search` takes `page` and `per_page` args to paginate results -- [rpc] `/unconfirmed_txs` takes `limit` arg to limit the output +- [rpc] `/tx_search` takes `page` (starts at 1) and `per_page` (max 100, default 30) args to paginate results +- [rpc] `/unconfirmed_txs` takes `limit` (max 100, default 30) arg to limit the output - [config] `mempool.size` and `mempool.cache_size` options IMPROVEMENTS @@ -32,15 +33,10 @@ IMPROVEMENTS - [consensus, state] Improve logging (more consensus logs, fewer tx logs) - [spec] Moved to `docs/spec` (TODO cleanup the rest of the docs ...) - BUG FIXES - [consensus] Fix issue #1575 where a late proposer can get stuck -BREAKING: -- [rpc] `/tx_search` now outputs maximum `?per_page` txs (you can provide custom `?per_page` up to 100; defaults to 30). You can set the `?page` (starts at 1). -- [rpc] `/unconfirmed_txs` now outputs maximum `?limit` txs (you can provide custom `?limit` up to 100; defaults to 30) - ## 0.19.3 (May 14th, 2018) FEATURES From 082a02e6d1102d319418790284fe98ba4e021198 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 14:40:01 -0400 Subject: [PATCH 33/34] consensus: only fsync wal after internal msgs --- .github/ISSUE_TEMPLATE | 4 ---- CHANGELOG.md | 1 + consensus/replay_test.go | 12 ++++++++---- consensus/state.go | 10 +++++----- consensus/wal.go | 28 ++++++++++++++++++++++------ consensus/wal_generator.go | 8 ++++++-- 6 files changed, 42 insertions(+), 21 deletions(-) diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index e714a6ff..c9c1f6a0 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -19,10 +19,6 @@ in a case of bug. **ABCI app** (name for built-in, URL for self-written if it's publicly available): - -**Merkleeyes version** (use `git rev-parse --verify HEAD`, skip if you don't use it): - - **Environment**: - **OS** (e.g. from /etc/os-release): - **Install tools**: diff --git a/CHANGELOG.md b/CHANGELOG.md index 507b8319..4d2462c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ FEATURES IMPROVEMENTS - [docs] Lots of updates +- [consensus] Only Fsync() the WAL before executing msgs from ourselves BUG FIXES diff --git a/consensus/replay_test.go b/consensus/replay_test.go index ff0eee1c..84b1e118 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -218,15 +218,15 @@ func (e ReachedHeightToStopError) Error() string { return fmt.Sprintf("reached height to stop %d", e.height) } -// Save simulate WAL's crashing by sending an error to the panicCh and then +// Write simulate WAL's crashing by sending an error to the panicCh and then // exiting the cs.receiveRoutine. -func (w *crashingWAL) Save(m WALMessage) { +func (w *crashingWAL) Write(m WALMessage) { if endMsg, ok := m.(EndHeightMessage); ok { if endMsg.Height == w.heightToStop { w.panicCh <- ReachedHeightToStopError{endMsg.Height} runtime.Goexit() } else { - w.next.Save(m) + w.next.Write(m) } return } @@ -238,10 +238,14 @@ func (w *crashingWAL) Save(m WALMessage) { runtime.Goexit() } else { w.msgIndex++ - w.next.Save(m) + w.next.Write(m) } } +func (w *crashingWAL) WriteSync(m WALMessage) { + w.Write(m) +} + func (w *crashingWAL) Group() *auto.Group { return w.next.Group() } func (w *crashingWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) { return w.next.SearchForEndHeight(height, options) diff --git a/consensus/state.go b/consensus/state.go index e4477a9b..b5b94368 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -504,7 +504,7 @@ func (cs *ConsensusState) updateToState(state sm.State) { func (cs *ConsensusState) newStep() { rs := cs.RoundStateEvent() - cs.wal.Save(rs) + cs.wal.Write(rs) cs.nSteps++ // newStep is called by updateToStep in NewConsensusState before the eventBus is set! if cs.eventBus != nil { @@ -542,16 +542,16 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { case height := <-cs.mempool.TxsAvailable(): cs.handleTxsAvailable(height) case mi = <-cs.peerMsgQueue: - cs.wal.Save(mi) + cs.wal.Write(mi) // handles proposals, block parts, votes // may generate internal events (votes, complete proposals, 2/3 majorities) cs.handleMsg(mi) case mi = <-cs.internalMsgQueue: - cs.wal.Save(mi) + cs.wal.WriteSync(mi) // NOTE: fsync // handles proposals, block parts, votes cs.handleMsg(mi) case ti := <-cs.timeoutTicker.Chan(): // tockChan: - cs.wal.Save(ti) + cs.wal.Write(ti) // if the timeout is relevant to the rs // go to the next step cs.handleTimeout(ti, rs) @@ -1241,7 +1241,7 @@ func (cs *ConsensusState) finalizeCommit(height int64) { // Either way, the ConsensusState should not be resumed until we // successfully call ApplyBlock (ie. later here, or in Handshake after // restart). - cs.wal.Save(EndHeightMessage{height}) + cs.wal.WriteSync(EndHeightMessage{height}) // NOTE: fsync fail.Fail() // XXX diff --git a/consensus/wal.go b/consensus/wal.go index d22c3ea1..e9e05fc9 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -50,7 +50,8 @@ func RegisterWALMessages(cdc *amino.Codec) { // WAL is an interface for any write-ahead logger. type WAL interface { - Save(WALMessage) + Write(WALMessage) + WriteSync(WALMessage) Group() *auto.Group SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) @@ -98,7 +99,7 @@ func (wal *baseWAL) OnStart() error { if err != nil { return err } else if size == 0 { - wal.Save(EndHeightMessage{0}) + wal.WriteSync(EndHeightMessage{0}) } err = wal.group.Start() return err @@ -109,8 +110,22 @@ func (wal *baseWAL) OnStop() { wal.group.Stop() } -// called in newStep and for each pass in receiveRoutine -func (wal *baseWAL) Save(msg WALMessage) { +// called in newStep and for each receive on the +// peerMsgQueue and the timoutTicker +func (wal *baseWAL) Write(msg WALMessage) { + if wal == nil { + return + } + + // Write the wal message + if err := wal.enc.Encode(&TimedWALMessage{time.Now(), msg}); err != nil { + cmn.PanicQ(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg)) + } +} + +// called when we receive a msg from ourselves +// so that we write to disk before sending signed messages +func (wal *baseWAL) WriteSync(msg WALMessage) { if wal == nil { return } @@ -297,8 +312,9 @@ func (dec *WALDecoder) Decode() (*TimedWALMessage, error) { type nilWAL struct{} -func (nilWAL) Save(m WALMessage) {} -func (nilWAL) Group() *auto.Group { return nil } +func (nilWAL) Write(m WALMessage) {} +func (nilWAL) WriteSync(m WALMessage) {} +func (nilWAL) Group() *auto.Group { return nil } func (nilWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) { return nil, false, nil } diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 18ff614c..dc364df0 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -83,7 +83,7 @@ func WALWithNBlocks(numBlocks int) (data []byte, err error) { numBlocksWritten := make(chan struct{}) wal := newByteBufferWAL(logger, NewWALEncoder(wr), int64(numBlocks), numBlocksWritten) // see wal.go#103 - wal.Save(EndHeightMessage{0}) + wal.Write(EndHeightMessage{0}) consensusState.wal = wal if err := consensusState.Start(); err != nil { @@ -166,7 +166,7 @@ func newByteBufferWAL(logger log.Logger, enc *WALEncoder, nBlocks int64, signalS // Save writes message to the internal buffer except when heightToStop is // reached, in which case it will signal the caller via signalWhenStopsTo and // skip writing. -func (w *byteBufferWAL) Save(m WALMessage) { +func (w *byteBufferWAL) Write(m WALMessage) { if w.stopped { w.logger.Debug("WAL already stopped. Not writing message", "msg", m) return @@ -189,6 +189,10 @@ func (w *byteBufferWAL) Save(m WALMessage) { } } +func (w *byteBufferWAL) WriteSync(m WALMessage) { + w.Write(m) +} + func (w *byteBufferWAL) Group() *auto.Group { panic("not implemented") } From ee4eb59355718f5a0c9cf3eb91ef08ab1f98f1e1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 20 May 2018 16:44:08 -0400 Subject: [PATCH 34/34] update comments --- consensus/wal.go | 21 +++++++++------------ node/node_test.go | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index e9e05fc9..0db0dc50 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -110,8 +110,9 @@ func (wal *baseWAL) OnStop() { wal.group.Stop() } -// called in newStep and for each receive on the -// peerMsgQueue and the timoutTicker +// Write is called in newStep and for each receive on the +// peerMsgQueue and the timoutTicker. +// NOTE: does not call fsync() func (wal *baseWAL) Write(msg WALMessage) { if wal == nil { return @@ -119,25 +120,21 @@ func (wal *baseWAL) Write(msg WALMessage) { // Write the wal message if err := wal.enc.Encode(&TimedWALMessage{time.Now(), msg}); err != nil { - cmn.PanicQ(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg)) + panic(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg)) } } -// called when we receive a msg from ourselves -// so that we write to disk before sending signed messages +// WriteSync is called when we receive a msg from ourselves +// so that we write to disk before sending signed messages. +// NOTE: calls fsync() func (wal *baseWAL) WriteSync(msg WALMessage) { if wal == nil { return } - // Write the wal message - if err := wal.enc.Encode(&TimedWALMessage{time.Now(), msg}); err != nil { - cmn.PanicQ(cmn.Fmt("Error writing msg to consensus wal: %v \n\nMessage: %v", err, msg)) - } - - // TODO: only flush when necessary + wal.Write(msg) if err := wal.group.Flush(); err != nil { - cmn.PanicQ(cmn.Fmt("Error flushing consensus wal buf to file. Error: %v \n", err)) + panic(cmn.Fmt("Error flushing consensus wal buf to file. Error: %v \n", err)) } } diff --git a/node/node_test.go b/node/node_test.go index ca539382..cdabdbb3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -31,7 +31,7 @@ func TestNodeStartStop(t *testing.T) { assert.NoError(t, err) select { case <-blockCh: - case <-time.After(5 * time.Second): + case <-time.After(10 * time.Second): t.Fatal("timed out waiting for the node to produce a block") }