Randomized the retry delay

network/builder_test.go

@@ -79,14 +79,8 @@ func TestBuildGetPeerList(t *testing.T) {
 
 func TestBuildPeerList(t *testing.T) {
 	ips := []utils.IPDesc{
-		utils.IPDesc{
+		{IP: net.IPv6loopback, Port: 12345},
-			IP:   net.IPv6loopback,
+		{IP: net.IPv6loopback, Port: 54321},
-			Port: 12345,
-		},
-		utils.IPDesc{
-			IP:   net.IPv6loopback,
-			Port: 54321,
-		},
 	}
 
 	msg, err := TestBuilder.PeerList(ips)

network/commands.go

@@ -188,22 +188,22 @@ const (
 var (
 	Messages = map[Op][]Field{
 		// Handshake:
-		GetVersion:  []Field{},
+		GetVersion:  {},
-		Version:     []Field{NetworkID, NodeID, MyTime, IP, VersionStr},
+		Version:     {NetworkID, NodeID, MyTime, IP, VersionStr},
-		GetPeerList: []Field{},
+		GetPeerList: {},
-		PeerList:    []Field{Peers},
+		PeerList:    {Peers},
 		// Bootstrapping:
-		GetAcceptedFrontier: []Field{ChainID, RequestID},
+		GetAcceptedFrontier: {ChainID, RequestID},
-		AcceptedFrontier:    []Field{ChainID, RequestID, ContainerIDs},
+		AcceptedFrontier:    {ChainID, RequestID, ContainerIDs},
-		GetAccepted:         []Field{ChainID, RequestID, ContainerIDs},
+		GetAccepted:         {ChainID, RequestID, ContainerIDs},
-		Accepted:            []Field{ChainID, RequestID, ContainerIDs},
+		Accepted:            {ChainID, RequestID, ContainerIDs},
-		GetAncestors:        []Field{ChainID, RequestID, ContainerID},
+		GetAncestors:        {ChainID, RequestID, ContainerID},
-		MultiPut:            []Field{ChainID, RequestID, MultiContainerBytes},
+		MultiPut:            {ChainID, RequestID, MultiContainerBytes},
 		// Consensus:
-		Get:       []Field{ChainID, RequestID, ContainerID},
+		Get:       {ChainID, RequestID, ContainerID},
-		Put:       []Field{ChainID, RequestID, ContainerID, ContainerBytes},
+		Put:       {ChainID, RequestID, ContainerID, ContainerBytes},
-		PushQuery: []Field{ChainID, RequestID, ContainerID, ContainerBytes},
+		PushQuery: {ChainID, RequestID, ContainerID, ContainerBytes},
-		PullQuery: []Field{ChainID, RequestID, ContainerID},
+		PullQuery: {ChainID, RequestID, ContainerID},
-		Chits:     []Field{ChainID, RequestID, ContainerIDs},
+		Chits:     {ChainID, RequestID, ContainerIDs},
 	}
 )

network/network.go

@@ -27,18 +27,21 @@ import (
 	"github.com/ava-labs/gecko/version"
 )
 
+// reasonable default values
 const (
-	defaultInitialReconnectDelay               = time.Second
+	defaultInitialReconnectDelay                     = time.Second
-	defaultMaxReconnectDelay                   = time.Hour
+	defaultMaxReconnectDelay                         = time.Hour
-	DefaultMaxMessageSize               uint32 = 1 << 21
+	DefaultMaxMessageSize                     uint32 = 1 << 21
-	defaultSendQueueSize                       = 1 << 10
+	defaultSendQueueSize                             = 1 << 10
-	defaultMaxClockDifference                  = time.Minute
+	defaultMaxNetworkPendingSendBytes                = 1 << 28 // 256MB
-	defaultPeerListGossipSpacing               = time.Minute
+	defaultNetworkPendingSendBytesToRateLimit        = defaultMaxNetworkPendingSendBytes / 4
-	defaultPeerListGossipSize                  = 100
+	defaultMaxClockDifference                        = time.Minute
-	defaultPeerListStakerGossipFraction        = 2
+	defaultPeerListGossipSpacing                     = time.Minute
-	defaultGetVersionTimeout                   = 2 * time.Second
+	defaultPeerListGossipSize                        = 100
-	defaultAllowPrivateIPs                     = true
+	defaultPeerListStakerGossipFraction              = 2
-	defaultGossipSize                          = 50
+	defaultGetVersionTimeout                         = 2 * time.Second
+	defaultAllowPrivateIPs                           = true
+	defaultGossipSize                                = 50
 )
 
 // Network defines the functionality of the networking library.
@@ -102,17 +105,19 @@ type network struct {
 	clock         timer.Clock
 	lastHeartbeat int64
 
-	initialReconnectDelay        time.Duration
+	initialReconnectDelay              time.Duration
-	maxReconnectDelay            time.Duration
+	maxReconnectDelay                  time.Duration
-	maxMessageSize               uint32
+	maxMessageSize                     uint32
-	sendQueueSize                int
+	sendQueueSize                      int
-	maxClockDifference           time.Duration
+	maxNetworkPendingSendBytes         int
-	peerListGossipSpacing        time.Duration
+	networkPendingSendBytesToRateLimit int
-	peerListGossipSize           int
+	maxClockDifference                 time.Duration
-	peerListStakerGossipFraction int
+	peerListGossipSpacing              time.Duration
-	getVersionTimeout            time.Duration
+	peerListGossipSize                 int
-	allowPrivateIPs              bool
+	peerListStakerGossipFraction       int
-	gossipSize                   int
+	getVersionTimeout                  time.Duration
+	allowPrivateIPs                    bool
+	gossipSize                         int
 
 	executor timer.Executor
 
@@ -164,6 +169,8 @@ func NewDefaultNetwork(
 		defaultMaxReconnectDelay,
 		DefaultMaxMessageSize,
 		defaultSendQueueSize,
+		defaultMaxNetworkPendingSendBytes,
+		defaultNetworkPendingSendBytesToRateLimit,
 		defaultMaxClockDifference,
 		defaultPeerListGossipSpacing,
 		defaultPeerListGossipSize,
@@ -193,6 +200,8 @@ func NewNetwork(
 	maxReconnectDelay time.Duration,
 	maxMessageSize uint32,
 	sendQueueSize int,
+	maxNetworkPendingSendBytes int,
+	networkPendingSendBytesToRateLimit int,
 	maxClockDifference time.Duration,
 	peerListGossipSpacing time.Duration,
 	peerListGossipSize int,
@@ -202,35 +211,37 @@ func NewNetwork(
 	gossipSize int,
 ) Network {
 	net := &network{
-		log:                          log,
+		log:                                log,
-		id:                           id,
+		id:                                 id,
-		ip:                           ip,
+		ip:                                 ip,
-		networkID:                    networkID,
+		networkID:                          networkID,
-		version:                      version,
+		version:                            version,
-		parser:                       parser,
+		parser:                             parser,
-		listener:                     listener,
+		listener:                           listener,
-		dialer:                       dialer,
+		dialer:                             dialer,
-		serverUpgrader:               serverUpgrader,
+		serverUpgrader:                     serverUpgrader,
-		clientUpgrader:               clientUpgrader,
+		clientUpgrader:                     clientUpgrader,
-		vdrs:                         vdrs,
+		vdrs:                               vdrs,
-		router:                       router,
+		router:                             router,
-		nodeID:                       rand.Uint32(),
+		nodeID:                             rand.Uint32(),
-		initialReconnectDelay:        initialReconnectDelay,
+		initialReconnectDelay:              initialReconnectDelay,
-		maxReconnectDelay:            maxReconnectDelay,
+		maxReconnectDelay:                  maxReconnectDelay,
-		maxMessageSize:               maxMessageSize,
+		maxMessageSize:                     maxMessageSize,
-		sendQueueSize:                sendQueueSize,
+		sendQueueSize:                      sendQueueSize,
-		maxClockDifference:           maxClockDifference,
+		maxNetworkPendingSendBytes:         maxNetworkPendingSendBytes,
-		peerListGossipSpacing:        peerListGossipSpacing,
+		networkPendingSendBytesToRateLimit: networkPendingSendBytesToRateLimit,
-		peerListGossipSize:           peerListGossipSize,
+		maxClockDifference:                 maxClockDifference,
-		peerListStakerGossipFraction: peerListStakerGossipFraction,
+		peerListGossipSpacing:              peerListGossipSpacing,
-		getVersionTimeout:            getVersionTimeout,
+		peerListGossipSize:                 peerListGossipSize,
-		allowPrivateIPs:              allowPrivateIPs,
+		peerListStakerGossipFraction:       peerListStakerGossipFraction,
-		gossipSize:                   gossipSize,
+		getVersionTimeout:                  getVersionTimeout,
+		allowPrivateIPs:                    allowPrivateIPs,
+		gossipSize:                         gossipSize,
 
 		disconnectedIPs: make(map[string]struct{}),
 		connectedIPs:    make(map[string]struct{}),
 		retryDelay:      make(map[string]time.Duration),
-		myIPs:           map[string]struct{}{ip.String(): struct{}{}},
+		myIPs:           map[string]struct{}{ip.String(): {}},
 		peers:           make(map[[20]byte]*peer),
 	}
 	net.initialize(registerer)
@@ -738,11 +749,12 @@ func (n *network) connectTo(ip utils.IPDesc) {
 
 		if delay == 0 {
 			delay = n.initialReconnectDelay
-		} else {
-			delay *= 2
 		}
+
+		delay = time.Duration(float64(delay) * (1 + rand.Float64()))
 		if delay > n.maxReconnectDelay {
-			delay = n.maxReconnectDelay
+			// set the timeout to [.75, 1) * maxReconnectDelay
+			delay = time.Duration(float64(n.maxReconnectDelay) * (3 + rand.Float64()) / 4)
 		}
 
 		n.stateLock.Lock()