Randomized the retry delay

network/builder_test.go

@@ -79,14 +79,8 @@ func TestBuildGetPeerList(t *testing.T) {
 
 func TestBuildPeerList(t *testing.T) {
 	ips := []utils.IPDesc{
-		utils.IPDesc{
-			IP:   net.IPv6loopback,
-			Port: 12345,
-		},
-		utils.IPDesc{
-			IP:   net.IPv6loopback,
-			Port: 54321,
-		},
+		{IP: net.IPv6loopback, Port: 12345},
+		{IP: net.IPv6loopback, Port: 54321},
 	}
 
 	msg, err := TestBuilder.PeerList(ips)

network/commands.go

@@ -188,22 +188,22 @@ const (
 var (
 	Messages = map[Op][]Field{
 		// Handshake:
-		GetVersion:  []Field{},
-		Version:     []Field{NetworkID, NodeID, MyTime, IP, VersionStr},
-		GetPeerList: []Field{},
-		PeerList:    []Field{Peers},
+		GetVersion:  {},
+		Version:     {NetworkID, NodeID, MyTime, IP, VersionStr},
+		GetPeerList: {},
+		PeerList:    {Peers},
 		// Bootstrapping:
-		GetAcceptedFrontier: []Field{ChainID, RequestID},
-		AcceptedFrontier:    []Field{ChainID, RequestID, ContainerIDs},
-		GetAccepted:         []Field{ChainID, RequestID, ContainerIDs},
-		Accepted:            []Field{ChainID, RequestID, ContainerIDs},
-		GetAncestors:        []Field{ChainID, RequestID, ContainerID},
-		MultiPut:            []Field{ChainID, RequestID, MultiContainerBytes},
+		GetAcceptedFrontier: {ChainID, RequestID},
+		AcceptedFrontier:    {ChainID, RequestID, ContainerIDs},
+		GetAccepted:         {ChainID, RequestID, ContainerIDs},
+		Accepted:            {ChainID, RequestID, ContainerIDs},
+		GetAncestors:        {ChainID, RequestID, ContainerID},
+		MultiPut:            {ChainID, RequestID, MultiContainerBytes},
 		// Consensus:
-		Get:       []Field{ChainID, RequestID, ContainerID},
-		Put:       []Field{ChainID, RequestID, ContainerID, ContainerBytes},
-		PushQuery: []Field{ChainID, RequestID, ContainerID, ContainerBytes},
-		PullQuery: []Field{ChainID, RequestID, ContainerID},
-		Chits:     []Field{ChainID, RequestID, ContainerIDs},
+		Get:       {ChainID, RequestID, ContainerID},
+		Put:       {ChainID, RequestID, ContainerID, ContainerBytes},
+		PushQuery: {ChainID, RequestID, ContainerID, ContainerBytes},
+		PullQuery: {ChainID, RequestID, ContainerID},
+		Chits:     {ChainID, RequestID, ContainerIDs},
 	}
 )

network/network.go

@@ -27,11 +27,14 @@ import (
 	"github.com/ava-labs/gecko/version"
 )
 
+// reasonable default values
 const (
 	defaultInitialReconnectDelay                     = time.Second
 	defaultMaxReconnectDelay                         = time.Hour
 	DefaultMaxMessageSize                     uint32 = 1 << 21
 	defaultSendQueueSize                             = 1 << 10
+	defaultMaxNetworkPendingSendBytes                = 1 << 28 // 256MB
+	defaultNetworkPendingSendBytesToRateLimit        = defaultMaxNetworkPendingSendBytes / 4
 	defaultMaxClockDifference                        = time.Minute
 	defaultPeerListGossipSpacing                     = time.Minute
 	defaultPeerListGossipSize                        = 100
@@ -106,6 +109,8 @@ type network struct {
 	maxReconnectDelay                  time.Duration
 	maxMessageSize                     uint32
 	sendQueueSize                      int
+	maxNetworkPendingSendBytes         int
+	networkPendingSendBytesToRateLimit int
 	maxClockDifference                 time.Duration
 	peerListGossipSpacing              time.Duration
 	peerListGossipSize                 int
@@ -164,6 +169,8 @@ func NewDefaultNetwork(
 		defaultMaxReconnectDelay,
 		DefaultMaxMessageSize,
 		defaultSendQueueSize,
+		defaultMaxNetworkPendingSendBytes,
+		defaultNetworkPendingSendBytesToRateLimit,
 		defaultMaxClockDifference,
 		defaultPeerListGossipSpacing,
 		defaultPeerListGossipSize,
@@ -193,6 +200,8 @@ func NewNetwork(
 	maxReconnectDelay time.Duration,
 	maxMessageSize uint32,
 	sendQueueSize int,
+	maxNetworkPendingSendBytes int,
+	networkPendingSendBytesToRateLimit int,
 	maxClockDifference time.Duration,
 	peerListGossipSpacing time.Duration,
 	peerListGossipSize int,
@@ -219,6 +228,8 @@ func NewNetwork(
 		maxReconnectDelay:                  maxReconnectDelay,
 		maxMessageSize:                     maxMessageSize,
 		sendQueueSize:                      sendQueueSize,
+		maxNetworkPendingSendBytes:         maxNetworkPendingSendBytes,
+		networkPendingSendBytesToRateLimit: networkPendingSendBytesToRateLimit,
 		maxClockDifference:                 maxClockDifference,
 		peerListGossipSpacing:              peerListGossipSpacing,
 		peerListGossipSize:                 peerListGossipSize,
@@ -230,7 +241,7 @@ func NewNetwork(
 		disconnectedIPs: make(map[string]struct{}),
 		connectedIPs:    make(map[string]struct{}),
 		retryDelay:      make(map[string]time.Duration),
-		myIPs:           map[string]struct{}{ip.String(): struct{}{}},
+		myIPs:           map[string]struct{}{ip.String(): {}},
 		peers:           make(map[[20]byte]*peer),
 	}
 	net.initialize(registerer)
@@ -738,11 +749,12 @@ func (n *network) connectTo(ip utils.IPDesc) {
 
 		if delay == 0 {
 			delay = n.initialReconnectDelay
-		} else {
-			delay *= 2
 		}
+
+		delay = time.Duration(float64(delay) * (1 + rand.Float64()))
 		if delay > n.maxReconnectDelay {
-			delay = n.maxReconnectDelay
+			// set the timeout to [.75, 1) * maxReconnectDelay
+			delay = time.Duration(float64(n.maxReconnectDelay) * (3 + rand.Float64()) / 4)
 		}
 
 		n.stateLock.Lock()