diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 48b7db0b..5728e558 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -10,6 +10,7 @@ BREAKING CHANGES: - [abci] Added address of the original proposer of the block to Header. - [abci] Change ABCI Header to match Tendermint exactly - [libs] Remove cmn.Fmt, in favor of fmt.Sprintf +- [config] Replace MaxNumPeers with MaxNumInboundPeers and MaxNumOutboundPeers FEATURES: diff --git a/config/config.go b/config/config.go index fb8e7908..51683ef0 100644 --- a/config/config.go +++ b/config/config.go @@ -239,6 +239,8 @@ type RPCConfig struct { // If you want to accept more significant number than the default, make sure // you increase your OS limits. // 0 - unlimited. + // Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} + // 1024 - 40 - 10 - 50 = 924 = ~900 MaxOpenConnections int `mapstructure:"max_open_connections"` } @@ -248,11 +250,9 @@ func DefaultRPCConfig() *RPCConfig { ListenAddress: "tcp://0.0.0.0:26657", GRPCListenAddress: "", - GRPCMaxOpenConnections: 900, // no ipv4 + GRPCMaxOpenConnections: 900, - Unsafe: false, - // should be < {ulimit -Sn} - {MaxNumPeers} - {N of wal, db and other open files} - // 1024 - 50 - 50 = 924 = ~900 + Unsafe: false, MaxOpenConnections: 900, } } @@ -295,8 +295,11 @@ type P2PConfig struct { // Set true for strict address routability rules AddrBookStrict bool `mapstructure:"addr_book_strict"` - // Maximum number of peers to connect to - MaxNumPeers int `mapstructure:"max_num_peers"` + // Maximum number of inbound peers + MaxNumInboundPeers int `mapstructure:"max_num_inbound_peers"` + + // Maximum number of outbound peers to connect to, excluding persistent peers + MaxNumOutboundPeers int `mapstructure:"max_num_outbound_peers"` // Time to wait before flushing messages out on the connection, in ms FlushThrottleTimeout int `mapstructure:"flush_throttle_timeout"` @@ -346,7 +349,8 @@ func DefaultP2PConfig() *P2PConfig { UPNP: false, AddrBook: defaultAddrBookPath, AddrBookStrict: true, - MaxNumPeers: 50, + MaxNumInboundPeers: 40, + MaxNumOutboundPeers: 10, FlushThrottleTimeout: 100, MaxPacketMsgPayloadSize: 1024, // 1 kB SendRate: 5120000, // 5 mB/s diff --git a/config/toml.go b/config/toml.go index 60ce15de..c85609b6 100644 --- a/config/toml.go +++ b/config/toml.go @@ -124,6 +124,8 @@ grpc_laddr = "{{ .RPC.GRPCListenAddress }}" # If you want to accept more significant number than the default, make sure # you increase your OS limits. # 0 - unlimited. +# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} +# 1024 - 40 - 10 - 50 = 924 = ~900 grpc_max_open_connections = {{ .RPC.GRPCMaxOpenConnections }} # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool @@ -134,6 +136,8 @@ unsafe = {{ .RPC.Unsafe }} # If you want to accept more significant number than the default, make sure # you increase your OS limits. # 0 - unlimited. +# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} +# 1024 - 40 - 10 - 50 = 924 = ~900 max_open_connections = {{ .RPC.MaxOpenConnections }} ##### peer to peer configuration options ##### @@ -166,8 +170,11 @@ addr_book_strict = {{ .P2P.AddrBookStrict }} # Time to wait before flushing messages out on the connection, in ms flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }} -# Maximum number of peers to connect to -max_num_peers = {{ .P2P.MaxNumPeers }} +# Maximum number of inbound peers +max_num_inbound_peers = {{ .P2P.MaxNumInboundPeers }} + +# Maximum number of outbound peers to connect to, excluding persistent peers +max_num_outbound_peers = {{ .P2P.MaxNumOutboundPeers }} # Maximum size of a message packet payload, in bytes max_packet_msg_payload_size = {{ .P2P.MaxPacketMsgPayloadSize }} diff --git a/docs/spec/reactors/pex/pex.md b/docs/spec/reactors/pex/pex.md index 0f13c0cb..9e00101a 100644 --- a/docs/spec/reactors/pex/pex.md +++ b/docs/spec/reactors/pex/pex.md @@ -15,6 +15,9 @@ we will not put them in the address book or gossip them to others. All peers except private peers and peers coming from them are tracked using the address book. +The rest of our peers are only distinguished by being either +inbound (they dialed our public address) or outbound (we dialed them). + ## Discovery Peer discovery begins with a list of seeds. @@ -26,15 +29,15 @@ and will attempt to maintain persistent connections with them. If the connection we will redial every 5s for a few minutes, then switch to an exponential backoff schedule, and after about a day of trying, stop dialing the peer. -So long as we have less than `MinNumOutboundPeers`, we periodically request additional peers +So long as we have less than `MaxNumOutboundPeers`, we periodically request additional peers from each of our own. If sufficient time goes by and we still can't find enough peers, we try the seeds again. ## Listening Peers listen on a configurable ListenAddr that they self-report in their -NodeInfo during handshakes with other peers. Peers accept up to (MaxNumPeers - -MinNumOutboundPeers) incoming peers. +NodeInfo during handshakes with other peers. Peers accept up to +`MaxNumInboundPeers` incoming peers. ## Address Book @@ -73,10 +76,11 @@ a trust metric (see below), but it's best to start with something simple. ## Select Peers to Dial -When we need more peers, we pick them randomly from the addrbook with some -configurable bias for unvetted peers. The bias should be lower when we have fewer peers -and can increase as we obtain more, ensuring that our first peers are more trustworthy, -but always giving us the chance to discover new good peers. +When we need more peers, we pick addresses randomly from the addrbook with some +configurable bias for unvetted peers. The bias should be lower when we have +fewer peers and can increase as we obtain more, ensuring that our first peers +are more trustworthy, but always giving us the chance to discover new good +peers. We track the last time we dialed a peer and the number of unsuccessful attempts we've made. If too many attempts are made, we mark the peer as bad. @@ -85,7 +89,8 @@ Connection attempts are made with exponential backoff (plus jitter). Because the selection process happens every `ensurePeersPeriod`, we might not end up dialing a peer for much longer than the backoff duration. -If we fail to connect to the peer after 16 tries (with exponential backoff), we remove from address book completely. +If we fail to connect to the peer after 16 tries (with exponential backoff), we +remove from address book completely. ## Select Peers to Exchange diff --git a/docs/tendermint-core/configuration.md b/docs/tendermint-core/configuration.md index ab2d7cc1..6fb72c44 100644 --- a/docs/tendermint-core/configuration.md +++ b/docs/tendermint-core/configuration.md @@ -77,6 +77,8 @@ grpc_laddr = "" # If you want to accept more significant number than the default, make sure # you increase your OS limits. # 0 - unlimited. +# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} +# 1024 - 40 - 10 - 50 = 924 = ~900 grpc_max_open_connections = 900 # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool @@ -87,7 +89,9 @@ unsafe = false # If you want to accept more significant number than the default, make sure # you increase your OS limits. # 0 - unlimited. -max_open_connections = 450 +# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} +# 1024 - 40 - 10 - 50 = 924 = ~900 +max_open_connections = 900 ##### peer to peer configuration options ##### [p2p] @@ -113,8 +117,11 @@ addr_book_strict = true # Time to wait before flushing messages out on the connection, in ms flush_throttle_timeout = 100 -# Maximum number of peers to connect to -max_num_peers = 50 +# Maximum number of inbound peers +max_num_inbound_peers = 40 + +# Maximum number of outbound peers to connect to +max_num_outbound_peers = 10 # Maximum size of a message packet payload, in bytes max_packet_msg_payload_size = 1024 diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 288cb0d1..b3c50a18 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -31,8 +31,7 @@ const ( maxMsgSize = maxAddressSize * maxGetSelection // ensure we have enough peers - defaultEnsurePeersPeriod = 30 * time.Second - defaultMinNumOutboundPeers = p2p.DefaultMinNumOutboundPeers + defaultEnsurePeersPeriod = 30 * time.Second // Seed/Crawler constants @@ -362,7 +361,7 @@ func (r *PEXReactor) ensurePeersRoutine() { func (r *PEXReactor) ensurePeers() { var ( out, in, dial = r.Switch.NumPeers() - numToDial = defaultMinNumOutboundPeers - (out + dial) + numToDial = r.Switch.MaxNumOutboundPeers() - (out + dial) ) r.Logger.Info( "Ensure peers", diff --git a/p2p/switch.go b/p2p/switch.go index da94fa4b..b26e1147 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -26,10 +26,6 @@ const ( // ie. 3**10 = 16hrs reconnectBackOffAttempts = 10 reconnectBackOffBaseSeconds = 3 - - // keep at least this many outbound peers - // TODO: move to config - DefaultMinNumOutboundPeers = 10 ) //----------------------------------------------------------------------------- @@ -268,6 +264,11 @@ func (sw *Switch) NumPeers() (outbound, inbound, dialing int) { return } +// MaxNumOutboundPeers returns a maximum number of outbound peers. +func (sw *Switch) MaxNumOutboundPeers() int { + return sw.config.MaxNumOutboundPeers +} + // Peers returns the set of peers that are connected to the switch. func (sw *Switch) Peers() IPeerSet { return sw.peers @@ -491,11 +492,15 @@ func (sw *Switch) listenerRoutine(l Listener) { break } - // ignore connection if we already have enough - // leave room for MinNumOutboundPeers - maxPeers := sw.config.MaxNumPeers - DefaultMinNumOutboundPeers - if maxPeers <= sw.peers.Size() { - sw.Logger.Info("Ignoring inbound connection: already have enough peers", "address", inConn.RemoteAddr().String(), "numPeers", sw.peers.Size(), "max", maxPeers) + // Ignore connection if we already have enough peers. + _, in, _ := sw.NumPeers() + if in >= sw.config.MaxNumInboundPeers { + sw.Logger.Info( + "Ignoring inbound connection: already have enough inbound peers", + "address", inConn.RemoteAddr().String(), + "have", in, + "max", sw.config.MaxNumInboundPeers, + ) inConn.Close() continue }