Merge pull request #1681 from tendermint/release/v0.19.8

Release/v0.19.8

.circleci/config.yml

@@ -133,18 +133,21 @@ jobs:
           key: v1-pkg-cache
       - restore_cache:
           key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run: mkdir -p /tmp/logs
       - run:
           name: Run tests
           command: |
             for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
               id=$(basename "$pkg")
 
-              GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg"
+              GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log"
             done
       - persist_to_workspace:
           root: /tmp/workspace
           paths:
             - "profiles/*"
+      - store_artifacts:
+          path: /tmp/logs
 
   test_persistence:
     <<: *defaults
@@ -196,9 +199,6 @@ workflows:
   test-suite:
     jobs:
       - setup_dependencies
-      - build_slate:
-          requires:
-            - setup_dependencies
       - setup_abci:
           requires:
             - setup_dependencies

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 0.19.8
+
+*June 4th, 2018*
+
+BREAKING:
+
+- [p2p] Remove `auth_enc` config option, peer connections are always auth
+  encrypted. Technically a breaking change but seems no one was using it and
+  arguably a bug fix :)
+
+BUG FIXES
+
+- [mempool] Fix deadlock under high load when `skip_timeout_commit=true` and
+  `create_empty_blocks=false`
+
 ## 0.19.7
 
 *May 31st, 2018*

config/config.go

@@ -287,9 +287,6 @@ type P2PConfig struct {
 	// Does not work if the peer-exchange reactor is disabled.
 	SeedMode bool `mapstructure:"seed_mode"`
 
-	// Authenticated encryption
-	AuthEnc bool `mapstructure:"auth_enc"`
-
 	// Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 	PrivatePeerIDs string `mapstructure:"private_peer_ids"`
 
@@ -310,7 +307,6 @@ func DefaultP2PConfig() *P2PConfig {
 		RecvRate:                512000, // 500 kB/s
 		PexReactor:              true,
 		SeedMode:                false,
-		AuthEnc:                 true,
 		AllowDuplicateIP:        true, // so non-breaking yet
 	}
 }

config/toml.go

@@ -165,9 +165,6 @@ pex = {{ .P2P.PexReactor }}
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = {{ .P2P.SeedMode }}
 
-# Authenticated encryption
-auth_enc = {{ .P2P.AuthEnc }}
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = "{{ .P2P.PrivatePeerIDs }}"
 

docs/examples/node0/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/examples/node1/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/examples/node2/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/examples/node3/config/config.toml

@@ -103,9 +103,6 @@ pex = true
 # Does not work if the peer-exchange reactor is disabled.
 seed_mode = false
 
-# Authenticated encryption
-auth_enc = true
-
 # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
 private_peer_ids = ""
 

docs/spec/blockchain/encoding.md

@@ -39,7 +39,7 @@ place of the public key. Here we list the concrete types, their names,
 and prefix bytes for public keys and signatures, as well as the address schemes
 for each PubKey. Note for brevity we don't
 include details of the private keys beyond their type and name, as they can be
-derrived the same way as the others using Amino.
+derived the same way as the others using Amino.
 
 All registered objects are encoded by Amino using a 4-byte PrefixBytes that
 uniquely identifies the object and includes information about its underlying
@@ -49,107 +49,35 @@ spec](https://github.com/tendermint/go-amino#computing-the-prefix-and-disambigua
 In what follows, we provide the type names and prefix bytes directly.
 Notice that when encoding byte-arrays, the length of the byte-array is appended
 to the PrefixBytes. Thus the encoding of a byte array becomes `<PrefixBytes>
-<Length> <ByteArray>`
+<Length> <ByteArray>`. In other words, to encode any type listed below you do not need to be 
+familiar with amino encoding. 
+You can simply use below table and concatenate Prefix || Length (of raw bytes) || raw bytes 
+( while || stands for byte concatenation here).
 
-NOTE: the remainder of this section on Public Key Cryptography can be generated
+| Type | Name | Prefix | Length |
-from [this script](https://github.com/tendermint/tendermint/blob/master/docs/spec/scripts/crypto.go)
+| ---- | ---- | ------ | ----- | 
+| PubKeyEd25519 | tendermint/PubKeyEd25519 | 0x1624DE62 | 0x20 | 
+| PubKeyLedgerEd25519 | tendermint/PubKeyLedgerEd25519 | 0x5C3453B2 | 0x20 |
+| PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE982 | 0x21 |
+| PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288912 | 0x40 |
+| PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79A | 0x20 |
+| PrivKeyLedgerSecp256k1 | tendermint/PrivKeyLedgerSecp256k1 | 0x10CAB393 | variable |
+| PrivKeyLedgerEd25519 | tendermint/PrivKeyLedgerEd25519 | 0x0CFEEF9B | variable |
+| SignatureEd25519 | tendermint/SignatureKeyEd25519 | 0x3DA1DB2A | 0x40 |
+| SignatureSecp256k1 | tendermint/SignatureKeySecp256k1 | 0x16E1FEEA | variable |
 
-### PubKeyEd25519
+### Examples
 
-```
+1. For example, the 33-byte (or 0x21-byte in hex) Secp256k1 pubkey
-// Name: tendermint/PubKeyEd25519
+`020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9`  
-// PrefixBytes: 0x1624DE62
-// Length: 0x20
-// Notes: raw 32-byte Ed25519 pubkey
-type PubKeyEd25519 [32]byte
-
-func (pubkey PubKeyEd25519) Address() []byte {
-      // NOTE: hash of the Amino encoded bytes!
-        return RIPEMD160(AminoEncode(pubkey))
-}
-```
-
-For example, the 32-byte Ed25519 pubkey
-`CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E` would be
-encoded as
-`1624DE6220CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E`.
-
-The address would then be
-`RIPEMD160(0x1624DE6220CCACD52F9B29D04393F01CD9AF6535455668115641F3D8BAEFD2295F24BAF60E)`
-or `430FF75BAF1EC4B0D51BB3EEC2955479D0071605`
-
-### SignatureEd25519
-
-```
-// Name: tendermint/SignatureKeyEd25519
-// PrefixBytes: 0x3DA1DB2A
-// Length: 0x40
-// Notes: raw 64-byte Ed25519 signature
-type SignatureEd25519 [64]byte
-```
-
-For example, the 64-byte Ed25519 signature
-`1B6034A8ED149D3C94FDA13EC03B26CC0FB264D9B0E47D3FA3DEF9FCDE658E49C80B35F9BE74949356401B15B18FB817D6E54495AD1C4A8401B248466CB0DB0B`
 would be encoded as
-`3DA1DB2A401B6034A8ED149D3C94FDA13EC03B26CC0FB264D9B0E47D3FA3DEF9FCDE658E49C80B35F9BE74949356401B15B18FB817D6E54495AD1C4A8401B248466CB0DB0B`
-
-### PrivKeyEd25519
-
-```
-// Name: tendermint/PrivKeyEd25519
-// Notes: raw 32-byte priv key concatenated to raw 32-byte pub key
-type PrivKeyEd25519 [64]byte
-```
-
-### PubKeySecp256k1
-
-```
-// Name: tendermint/PubKeySecp256k1
-// PrefixBytes: 0xEB5AE982
-// Length: 0x21
-// Notes: OpenSSL compressed pubkey prefixed with 0x02 or 0x03
-type PubKeySecp256k1 [33]byte
-
-func (pubkey PubKeySecp256k1) Address() []byte {
-      // NOTE: hash of the raw pubkey bytes (not Amino encoded!).
-        // Compatible with Bitcoin addresses.
-          return RIPEMD160(SHA256(pubkey[:]))
-}
-```
-
-For example, the 33-byte Secp256k1 pubkey
-`020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` would be
-encoded as
 `EB5AE98221020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9`
 
-The address would then be
+2. For example, the variable size Secp256k1 signature (in this particular example 70 or 0x46 bytes)
-`RIPEMD160(SHA256(0x020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9))`
-or `0AE5BEE929ABE51BAD345DB925EEA652680783FC`
-
-### SignatureSecp256k1
-
-```
-// Name: tendermint/SignatureKeySecp256k1
-// PrefixBytes: 0x16E1FEEA
-// Length: Variable
-// Encoding prefix: Variable
-// Notes: raw bytes of the Secp256k1 signature
-type SignatureSecp256k1 []byte
-```
-
-For example, the Secp256k1 signature
 `304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7`
 would be encoded as
 `16E1FEEA46304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7`
 
-### PrivKeySecp256k1
-
-```
-// Name: tendermint/PrivKeySecp256k1
-// Notes: raw 32-byte priv key
-type PrivKeySecp256k1 [32]byte
-```
-
 ## Other Common Types
 
 ### BitArray

docs/spec/p2p/peer.md

@@ -17,9 +17,6 @@ We will attempt to connect to the peer at IP:PORT, and verify,
 via authenticated encryption, that it is in possession of the private key
 corresponding to `<ID>`. This prevents man-in-the-middle attacks on the peer layer.
 
-If `auth_enc = false`, peers can use an arbitrary ID, but they must always use
-one. Authentication can then happen out-of-band of Tendermint, for instance via VPN.
-
 ## Connections
 
 All p2p connections use TCP.

docs/specification/configuration.rst

@@ -122,9 +122,6 @@ like the file below, however, double check by inspecting the
     # Does not work if the peer-exchange reactor is disabled.
     seed_mode = false
 
-    # Authenticated encryption
-    auth_enc = true
-
     # Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
     private_peer_ids = ""
 

docs/specification/secure-p2p.rst

@@ -65,9 +65,7 @@ are connected to at least one validator.
 Config
 ------
 
-Authenticated encryption is enabled by default. If you wish to use another
+Authenticated encryption is enabled by default.
-authentication scheme or your peers are connected via VPN, you can turn it off
-by setting ``auth_enc`` to ``false`` in the config file.
 
 Additional Reading
 ------------------

mempool/mempool.go

@@ -72,8 +72,8 @@ type Mempool struct {
 	rechecking           int32           // for re-checking filtered txs on Update()
 	recheckCursor        *clist.CElement // next expected response
 	recheckEnd           *clist.CElement // re-checking stops here
-	notifiedTxsAvailable bool            // true if fired on txsAvailable for this height
+	notifiedTxsAvailable bool
-	txsAvailable         chan int64      // fires the next height once for each height, when the mempool is not empty
+	txsAvailable         chan int64 // fires the next height once for each height, when the mempool is not empty
 
 	// Keep a cache of already-seen txs.
 	// This reduces the pressure on the proxyApp.
@@ -328,8 +328,12 @@ func (mem *Mempool) notifyTxsAvailable() {
 		panic("notified txs available but mempool is empty!")
 	}
 	if mem.txsAvailable != nil && !mem.notifiedTxsAvailable {
+		select {
+		case mem.txsAvailable <- mem.height + 1:
+		default:
+		}
+
 		mem.notifiedTxsAvailable = true
-		mem.txsAvailable <- mem.height + 1
 	}
 }
 
@@ -382,7 +386,7 @@ func (mem *Mempool) Update(height int64, txs types.Txs) error {
 	// Recheck mempool txs if any txs were committed in the block
 	// NOTE/XXX: in some apps a tx could be invalidated due to EndBlock,
 	//	so we really still do need to recheck, but this is for debugging
-	if mem.config.Recheck && (mem.config.RecheckEmpty || len(txs) > 0) {
+	if mem.config.Recheck && (mem.config.RecheckEmpty || len(goodTxs) > 0) {
 		mem.logger.Info("Recheck txs", "numtxs", len(goodTxs), "height", height)
 		mem.recheckTxs(goodTxs)
 		// At this point, mem.txs are being rechecked.

node/node.go

@@ -269,9 +269,6 @@ func NewNode(config *cfg.Config,
 	// but it would still be nice to have a clear list of the current "PersistentPeers"
 	// somewhere that we can return with net_info.
 	//
-	// Let's assume we always have IDs ... and we just dont authenticate them
-	// if auth_enc=false.
-	//
 	// If PEX is on, it should handle dialing the seeds. Otherwise the switch does it.
 	// Note we currently use the addrBook regardless at least for AddOurAddress
 	addrBook := pex.NewAddrBook(config.P2P.AddrBookFile(), config.P2P.AddrBookStrict)

p2p/peer.go

@@ -116,8 +116,6 @@ func newPeer(pc peerConn, nodeInfo NodeInfo,
 
 // PeerConfig is a Peer configuration.
 type PeerConfig struct {
-	AuthEnc bool `mapstructure:"auth_enc"` // authenticated encryption
-
 	// times are in seconds
 	HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
 	DialTimeout      time.Duration `mapstructure:"dial_timeout"`
@@ -132,7 +130,6 @@ type PeerConfig struct {
 // DefaultPeerConfig returns the default config.
 func DefaultPeerConfig() *PeerConfig {
 	return &PeerConfig{
-		AuthEnc:          true,
 		HandshakeTimeout: 20, // * time.Second,
 		DialTimeout:      3,  // * time.Second,
 		MConfig:          tmconn.DefaultMConnConfig(),
@@ -159,7 +156,7 @@ func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool,
 	}
 
 	// ensure dialed ID matches connection ID
-	if config.AuthEnc && addr.ID != pc.ID() {
+	if addr.ID != pc.ID() {
 		if err2 := conn.Close(); err2 != nil {
 			return pc, cmn.ErrorWrap(err, err2.Error())
 		}
@@ -187,17 +184,15 @@ func newPeerConn(rawConn net.Conn,
 		conn = FuzzConnAfterFromConfig(conn, 10*time.Second, config.FuzzConfig)
 	}
 
-	if config.AuthEnc {
+	// Set deadline for secret handshake
-		// Set deadline for secret handshake
+	if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil {
-		if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil {
+		return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection")
-			return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection")
+	}
-		}
 
-		// Encrypt connection
+	// Encrypt connection
-		conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey)
+	conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey)
-		if err != nil {
+	if err != nil {
-			return pc, cmn.ErrorWrap(err, "Error creating peer")
+		return pc, cmn.ErrorWrap(err, "Error creating peer")
-		}
 	}
 
 	// Only the information we already have

p2p/peer_test.go

@@ -41,32 +41,10 @@ func TestPeerBasic(t *testing.T) {
 	assert.Equal(rp.ID(), p.ID())
 }
 
-func TestPeerWithoutAuthEnc(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	config := DefaultPeerConfig()
-	config.AuthEnc = false
-
-	// simulate remote peer
-	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config}
-	rp.Start()
-	defer rp.Stop()
-
-	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), config)
-	require.Nil(err)
-
-	err = p.Start()
-	require.Nil(err)
-	defer p.Stop()
-
-	assert.True(p.IsRunning())
-}
-
 func TestPeerSend(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 
 	config := DefaultPeerConfig()
-	config.AuthEnc = false
 
 	// simulate remote peer
 	rp := &remotePeer{PrivKey: crypto.GenPrivKeyEd25519(), Config: config}

p2p/pex/pex_reactor_test.go

@@ -49,15 +49,12 @@ func TestPEXReactorAddRemovePeer(t *testing.T) {
 	assert.Equal(t, size+1, book.Size())
 
 	r.RemovePeer(peer, "peer not available")
-	assert.Equal(t, size+1, book.Size())
 
 	outboundPeer := p2p.CreateRandomPeer(true)
 
 	r.AddPeer(outboundPeer)
-	assert.Equal(t, size+1, book.Size(), "outbound peers should not be added to the address book")
 
 	r.RemovePeer(outboundPeer, "peer not available")
-	assert.Equal(t, size+1, book.Size())
 }
 
 // --- FAIL: TestPEXReactorRunning (11.10s)

p2p/switch.go

@@ -95,7 +95,6 @@ func NewSwitch(config *cfg.P2PConfig) *Switch {
 	sw.peerConfig.MConfig.SendRate = config.SendRate
 	sw.peerConfig.MConfig.RecvRate = config.RecvRate
 	sw.peerConfig.MConfig.MaxPacketMsgPayloadSize = config.MaxPacketMsgPayloadSize
-	sw.peerConfig.AuthEnc = config.AuthEnc
 
 	sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw)
 	return sw
@@ -534,10 +533,6 @@ func (sw *Switch) addPeer(pc peerConn) error {
 		return err
 	}
 
-	// NOTE: if AuthEnc==false, we don't have a peerID until after the handshake.
-	// If AuthEnc==true then we already know the ID and could do the checks first before the handshake,
-	// but it's simple to just deal with both cases the same after the handshake.
-
 	// Exchange NodeInfo on the conn
 	peerNodeInfo, err := pc.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.peerConfig.HandshakeTimeout*time.Second))
 	if err != nil {
@@ -547,13 +542,14 @@ func (sw *Switch) addPeer(pc peerConn) error {
 	peerID := peerNodeInfo.ID
 
 	// ensure connection key matches self reported key
-	if pc.config.AuthEnc {
+	connID := pc.ID()
-		connID := pc.ID()
 
-		if peerID != connID {
+	if peerID != connID {
-			return fmt.Errorf("nodeInfo.ID() (%v) doesn't match conn.ID() (%v)",
+		return fmt.Errorf(
-				peerID, connID)
+			"nodeInfo.ID() (%v) doesn't match conn.ID() (%v)",
-		}
+			peerID,
+			connID,
+		)
 	}
 
 	// Validate the peers nodeInfo

version/version.go

@@ -4,13 +4,13 @@ package version
 const (
 	Maj = "0"
 	Min = "19"
-	Fix = "7"
+	Fix = "8"
 )
 
 var (
 	// Version is the current version of Tendermint
 	// Must be a string because scripts like dist.sh read this file.
-	Version = "0.19.7"
+	Version = "0.19.8"
 
 	// GitCommit is the current HEAD set using ldflags.
 	GitCommit string