node: override the p2p advertised address

When behind a private network such as in the case of a guardian running
in Kubernetes, or behind a NAT, the default p2p setup doesn't work. It
looks at all addresses that it is listening on and advertises them on p2p
as the addresses for contacting the guardian. This patch is the first
step towards allowing specifying a custom ip address to be advertised on
the gossip p2p network.

For example, a guardian running inside kubernetes can post a reserved ip
address of the incoming ingress/load balancer that sends the traffic in
to their guardian.

node/cmd/guardiand/node.go

@@ -233,6 +233,9 @@ var (
 	gatewayRelayerContract      *string
 	gatewayRelayerKeyPath       *string
 	gatewayRelayerKeyPassPhrase *string
+
+	// This is the externally reachable address advertized over gossip for guardian p2p and ccq p2p.
+	gossipAdvertiseAddress *string
 )
 
 func init() {
@@ -416,6 +419,7 @@ func init() {
 	ccqP2pBootstrap = NodeCmd.Flags().String("ccqP2pBootstrap", "", "CCQ P2P bootstrap peers (optional for mainnet or testnet, overrides default, required for unsafeDevMode)")
 	ccqAllowedPeers = NodeCmd.Flags().String("ccqAllowedPeers", "", "CCQ allowed P2P peers (comma-separated)")
 	ccqBackfillCache = NodeCmd.Flags().Bool("ccqBackfillCache", true, "Should EVM chains backfill CCQ timestamp cache on startup")
+	gossipAdvertiseAddress = NodeCmd.Flags().String("gossipAdvertiseAddress", "", "External IP to advertize on Guardian and CCQ p2p (use if behind a NAT or running in k8s)")
 
 	gatewayRelayerContract = NodeCmd.Flags().String("gatewayRelayerContract", "", "Address of the smart contract on wormchain to receive relayed VAAs")
 	gatewayRelayerKeyPath = NodeCmd.Flags().String("gatewayRelayerKeyPath", "", "Path to gateway relayer private key for signing transactions")
@@ -1683,7 +1687,7 @@ func runNode(cmd *cobra.Command, args []string) {
 		node.GuardianOptionGatewayRelayer(*gatewayRelayerContract, gatewayRelayerWormchainConn),
 		node.GuardianOptionQueryHandler(*ccqEnabled, *ccqAllowedRequesters),
 		node.GuardianOptionAdminService(*adminSocketPath, ethRPC, ethContract, rpcMap),
-		node.GuardianOptionP2P(p2pKey, *p2pNetworkID, *p2pBootstrap, *nodeName, *disableHeartbeatVerify, *p2pPort, *ccqP2pBootstrap, *ccqP2pPort, *ccqAllowedPeers, ibc.GetFeatures),
+		node.GuardianOptionP2P(p2pKey, *p2pNetworkID, *p2pBootstrap, *nodeName, *disableHeartbeatVerify, *p2pPort, *ccqP2pBootstrap, *ccqP2pPort, *ccqAllowedPeers, *gossipAdvertiseAddress, ibc.GetFeatures),
 		node.GuardianOptionStatusServer(*statusAddr),
 		node.GuardianOptionProcessor(),
 	}

node/pkg/node/node_test.go

@@ -190,7 +190,7 @@ func mockGuardianRunnable(t testing.TB, gs []*mockGuardian, mockGuardianIndex ui
 			GuardianOptionNoAccountant(), // disable accountant
 			GuardianOptionGovernor(true),
 			GuardianOptionGatewayRelayer("", nil), // disable gateway relayer
-			GuardianOptionP2P(gs[mockGuardianIndex].p2pKey, networkID, bootstrapPeers, nodeName, false, cfg.p2pPort, "", 0, "", func() string { return "" }),
+			GuardianOptionP2P(gs[mockGuardianIndex].p2pKey, networkID, bootstrapPeers, nodeName, false, cfg.p2pPort, "", 0, "", "", func() string { return "" }),
 			GuardianOptionPublicRpcSocket(cfg.publicSocket, publicRpcLogDetail),
 			GuardianOptionPublicrpcTcpService(cfg.publicRpc, publicRpcLogDetail),
 			GuardianOptionPublicWeb(cfg.publicWeb, cfg.publicSocket, "", false, ""),

node/pkg/node/options.go

@@ -39,7 +39,7 @@ type GuardianOption struct {
 
 // GuardianOptionP2P configures p2p networking.
 // Dependencies: Accountant, Governor
-func GuardianOptionP2P(p2pKey libp2p_crypto.PrivKey, networkId string, bootstrapPeers string, nodeName string, disableHeartbeatVerify bool, port uint, ccqBootstrapPeers string, ccqPort uint, ccqAllowedPeers string, ibcFeaturesFunc func() string) *GuardianOption {
+func GuardianOptionP2P(p2pKey libp2p_crypto.PrivKey, networkId, bootstrapPeers, nodeName string, disableHeartbeatVerify bool, port uint, ccqBootstrapPeers string, ccqPort uint, ccqAllowedPeers, gossipAdvertiseAddress string, ibcFeaturesFunc func() string) *GuardianOption {
 	return &GuardianOption{
 		name:         "p2p",
 		dependencies: []string{"accountant", "governor", "gateway-relayer"},
@@ -52,6 +52,11 @@ func GuardianOptionP2P(p2pKey libp2p_crypto.PrivKey, networkId string, bootstrap
 				components.SignedHeartbeatLogLevel = zapcore.InfoLevel
 			}
 
+			// Add the gossip advertisement address if it was specified
+			if gossipAdvertiseAddress != "" {
+				components.GossipAdvertiseAddress = gossipAdvertiseAddress
+			}
+
 			g.runnables["p2p"] = p2p.Run(
 				g.obsvC,
 				g.obsvReqC.writeC,

node/pkg/p2p/ccq_p2p.go

@@ -88,6 +88,9 @@ func (ccq *ccqP2p) run(
 	}
 	components.Port = port
 
+	// Pass the gossip advertize address through to NewHost() if it was defined
+	components.GossipAdvertiseAddress = ccq.p2pComponents.GossipAdvertiseAddress
+
 	ccq.h, err = NewHost(ccq.logger, ctx, networkID, bootstrapPeers, components, priv)
 	if err != nil {
 		return fmt.Errorf("failed to create p2p: %w", err)

node/pkg/p2p/p2p.go

@@ -107,6 +107,8 @@ type Components struct {
 	SignedHeartbeatLogLevel zapcore.Level
 	// GossipParams is used to configure the GossipSub instance used by the Guardian.
 	GossipParams pubsub.GossipSubParams
+	// GossipAdvertiseAddress is an override for the external IP advertised via p2p to other peers.
+	GossipAdvertiseAddress string
 }
 
 func (f *Components) ListeningAddresses() []string {
@@ -205,6 +207,26 @@ func NewHost(logger *zap.Logger, ctx context.Context, networkID string, bootstra
 			components.ListeningAddresses()...,
 		),
 
+		// Takes the multiaddrs we are listening on and returns the multiaddrs to advertise to the network to
+		// connect to. Allows overriding the announce address for nodes running behind a NAT or in kubernetes
+		libp2p.AddrsFactory(func(addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {
+			if components.GossipAdvertiseAddress != "" {
+				addr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/%d", components.GossipAdvertiseAddress, components.Port))
+				if err != nil {
+					// If the multiaddr is specified incorrectly, blow up
+					logger.Fatal("error with the specified gossip address",
+						zap.String("gossip_advertise_address", components.GossipAdvertiseAddress),
+						zap.Error(err),
+					)
+				}
+				logger.Info("Overriding the advertised p2p address",
+					zap.String("GossipAdvertiseAddress", addr.String()),
+				)
+				return []multiaddr.Multiaddr{addr}
+			}
+			return addrs
+		}),
+
 		// Enable TLS security as the only security protocol.
 		libp2p.Security(libp2ptls.ID, libp2ptls.New),