536 lines
16 KiB
Go
536 lines
16 KiB
Go
package p2p
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/certusone/wormhole/node/pkg/accounting"
|
|
node_common "github.com/certusone/wormhole/node/pkg/common"
|
|
"github.com/certusone/wormhole/node/pkg/governor"
|
|
"github.com/certusone/wormhole/node/pkg/version"
|
|
"github.com/ethereum/go-ethereum/common"
|
|
ethcrypto "github.com/ethereum/go-ethereum/crypto"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
"github.com/wormhole-foundation/wormhole/sdk/vaa"
|
|
|
|
"github.com/libp2p/go-libp2p/core/peer"
|
|
"github.com/multiformats/go-multiaddr"
|
|
|
|
"github.com/libp2p/go-libp2p"
|
|
dht "github.com/libp2p/go-libp2p-kad-dht"
|
|
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
|
"github.com/libp2p/go-libp2p/core/crypto"
|
|
"github.com/libp2p/go-libp2p/core/host"
|
|
"github.com/libp2p/go-libp2p/core/protocol"
|
|
"github.com/libp2p/go-libp2p/core/routing"
|
|
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
|
|
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
|
|
libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
|
|
"go.uber.org/zap"
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1"
|
|
"github.com/certusone/wormhole/node/pkg/supervisor"
|
|
)
|
|
|
|
var (
|
|
p2pHeartbeatsSent = promauto.NewCounter(
|
|
prometheus.CounterOpts{
|
|
Name: "wormhole_p2p_heartbeats_sent_total",
|
|
Help: "Total number of p2p heartbeats sent",
|
|
})
|
|
p2pMessagesSent = promauto.NewCounter(
|
|
prometheus.CounterOpts{
|
|
Name: "wormhole_p2p_broadcast_messages_sent_total",
|
|
Help: "Total number of p2p pubsub broadcast messages sent",
|
|
})
|
|
p2pMessagesReceived = promauto.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Name: "wormhole_p2p_broadcast_messages_received_total",
|
|
Help: "Total number of p2p pubsub broadcast messages received",
|
|
}, []string{"type"})
|
|
)
|
|
|
|
var heartbeatMessagePrefix = []byte("heartbeat|")
|
|
|
|
var signedObservationRequestPrefix = []byte("signed_observation_request|")
|
|
|
|
func heartbeatDigest(b []byte) common.Hash {
|
|
return ethcrypto.Keccak256Hash(append(heartbeatMessagePrefix, b...))
|
|
}
|
|
|
|
func signedObservationRequestDigest(b []byte) common.Hash {
|
|
return ethcrypto.Keccak256Hash(append(signedObservationRequestPrefix, b...))
|
|
}
|
|
|
|
func Run(
|
|
obsvC chan *gossipv1.SignedObservation,
|
|
obsvReqC chan *gossipv1.ObservationRequest,
|
|
obsvReqSendC chan *gossipv1.ObservationRequest,
|
|
sendC chan []byte,
|
|
signedInC chan *gossipv1.SignedVAAWithQuorum,
|
|
priv crypto.PrivKey,
|
|
gk *ecdsa.PrivateKey,
|
|
gst *node_common.GuardianSetState,
|
|
port uint,
|
|
networkID string,
|
|
bootstrapPeers string,
|
|
nodeName string,
|
|
disableHeartbeatVerify bool,
|
|
rootCtxCancel context.CancelFunc,
|
|
acct *accounting.Accounting,
|
|
gov *governor.ChainGovernor,
|
|
signedGovCfg chan *gossipv1.SignedChainGovernorConfig,
|
|
signedGovSt chan *gossipv1.SignedChainGovernorStatus,
|
|
) func(ctx context.Context) error {
|
|
return func(ctx context.Context) (re error) {
|
|
logger := supervisor.Logger(ctx)
|
|
|
|
mgr, err := connmgr.NewConnManager(
|
|
100, // LowWater
|
|
400, // HighWater,
|
|
connmgr.WithGracePeriod(time.Minute),
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create p2p connection manager: %w", err)
|
|
}
|
|
|
|
h, err := libp2p.New(
|
|
// Use the keypair we generated
|
|
libp2p.Identity(priv),
|
|
|
|
// Multiple listen addresses
|
|
libp2p.ListenAddrStrings(
|
|
// Listen on QUIC only.
|
|
// https://github.com/libp2p/go-libp2p/issues/688
|
|
fmt.Sprintf("/ip4/0.0.0.0/udp/%d/quic", port),
|
|
fmt.Sprintf("/ip6/::/udp/%d/quic", port),
|
|
),
|
|
|
|
// Enable TLS security as the only security protocol.
|
|
libp2p.Security(libp2ptls.ID, libp2ptls.New),
|
|
|
|
// Enable QUIC transport as the only transport.
|
|
libp2p.Transport(libp2pquic.NewTransport),
|
|
|
|
// Let's prevent our peer from having too many
|
|
// connections by attaching a connection manager.
|
|
libp2p.ConnectionManager(mgr),
|
|
|
|
// Let this host use the DHT to find other hosts
|
|
libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) {
|
|
logger.Info("Connecting to bootstrap peers", zap.String("bootstrap_peers", bootstrapPeers))
|
|
bootstrappers := make([]peer.AddrInfo, 0)
|
|
for _, addr := range strings.Split(bootstrapPeers, ",") {
|
|
if addr == "" {
|
|
continue
|
|
}
|
|
ma, err := multiaddr.NewMultiaddr(addr)
|
|
if err != nil {
|
|
logger.Error("Invalid bootstrap address", zap.String("peer", addr), zap.Error(err))
|
|
continue
|
|
}
|
|
pi, err := peer.AddrInfoFromP2pAddr(ma)
|
|
if err != nil {
|
|
logger.Error("Invalid bootstrap address", zap.String("peer", addr), zap.Error(err))
|
|
continue
|
|
}
|
|
if pi.ID == h.ID() {
|
|
logger.Info("We're a bootstrap node")
|
|
continue
|
|
}
|
|
bootstrappers = append(bootstrappers, *pi)
|
|
}
|
|
// TODO(leo): Persistent data store (i.e. address book)
|
|
idht, err := dht.New(ctx, h, dht.Mode(dht.ModeServer),
|
|
// This intentionally makes us incompatible with the global IPFS DHT
|
|
dht.ProtocolPrefix(protocol.ID("/"+networkID)),
|
|
dht.BootstrapPeers(bootstrappers...),
|
|
)
|
|
return idht, err
|
|
}),
|
|
)
|
|
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
defer func() {
|
|
// TODO: libp2p cannot be cleanly restarted (https://github.com/libp2p/go-libp2p/issues/992)
|
|
logger.Error("p2p routine has exited, cancelling root context...", zap.Error(re))
|
|
rootCtxCancel()
|
|
}()
|
|
|
|
topic := fmt.Sprintf("%s/%s", networkID, "broadcast")
|
|
|
|
logger.Info("Subscribing pubsub topic", zap.String("topic", topic))
|
|
ps, err := pubsub.NewGossipSub(ctx, h)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
th, err := ps.Join(topic)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to join topic: %w", err)
|
|
}
|
|
|
|
sub, err := th.Subscribe()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to subscribe topic: %w", err)
|
|
}
|
|
|
|
logger.Info("Node has been started", zap.String("peer_id", h.ID().String()),
|
|
zap.String("addrs", fmt.Sprintf("%v", h.Addrs())))
|
|
|
|
bootTime := time.Now()
|
|
|
|
// Periodically run guardian state set cleanup.
|
|
go func() {
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
gst.Cleanup()
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
// Disable heartbeat when no node name is provided (spy mode)
|
|
if nodeName == "" {
|
|
return
|
|
}
|
|
|
|
ctr := int64(0)
|
|
tick := time.NewTicker(15 * time.Second)
|
|
defer tick.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
DefaultRegistry.mu.Lock()
|
|
networks := make([]*gossipv1.Heartbeat_Network, 0, len(DefaultRegistry.networkStats))
|
|
for _, v := range DefaultRegistry.networkStats {
|
|
errCtr := DefaultRegistry.GetErrorCount(vaa.ChainID(v.Id))
|
|
v.ErrorCount = errCtr
|
|
networks = append(networks, v)
|
|
}
|
|
|
|
features := make([]string, 0)
|
|
if gov != nil {
|
|
features = append(features, "governor")
|
|
}
|
|
if acct != nil {
|
|
features = append(features, acct.FeatureString())
|
|
}
|
|
|
|
heartbeat := &gossipv1.Heartbeat{
|
|
NodeName: nodeName,
|
|
Counter: ctr,
|
|
Timestamp: time.Now().UnixNano(),
|
|
Networks: networks,
|
|
Version: version.Version(),
|
|
GuardianAddr: DefaultRegistry.guardianAddress,
|
|
BootTimestamp: bootTime.UnixNano(),
|
|
Features: features,
|
|
}
|
|
|
|
ourAddr := ethcrypto.PubkeyToAddress(gk.PublicKey)
|
|
if err := gst.SetHeartbeat(ourAddr, h.ID(), heartbeat); err != nil {
|
|
panic(err)
|
|
}
|
|
collectNodeMetrics(ourAddr, h.ID(), heartbeat)
|
|
|
|
if gov != nil {
|
|
gov.CollectMetrics(heartbeat, sendC, gk, ourAddr)
|
|
}
|
|
|
|
b, err := proto.Marshal(heartbeat)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
DefaultRegistry.mu.Unlock()
|
|
|
|
// Sign the heartbeat using our node's guardian key.
|
|
digest := heartbeatDigest(b)
|
|
sig, err := ethcrypto.Sign(digest.Bytes(), gk)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
msg := gossipv1.GossipMessage{Message: &gossipv1.GossipMessage_SignedHeartbeat{
|
|
SignedHeartbeat: &gossipv1.SignedHeartbeat{
|
|
Heartbeat: b,
|
|
Signature: sig,
|
|
GuardianAddr: ourAddr.Bytes(),
|
|
}}}
|
|
|
|
b, err = proto.Marshal(&msg)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
err = th.Publish(ctx, b)
|
|
if err != nil {
|
|
logger.Warn("failed to publish heartbeat message", zap.Error(err))
|
|
}
|
|
|
|
p2pHeartbeatsSent.Inc()
|
|
ctr += 1
|
|
}
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case msg := <-sendC:
|
|
err := th.Publish(ctx, msg)
|
|
p2pMessagesSent.Inc()
|
|
if err != nil {
|
|
logger.Error("failed to publish message from queue", zap.Error(err))
|
|
}
|
|
case msg := <-obsvReqSendC:
|
|
b, err := proto.Marshal(msg)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// Sign the observation request using our node's guardian key.
|
|
digest := signedObservationRequestDigest(b)
|
|
sig, err := ethcrypto.Sign(digest.Bytes(), gk)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
sReq := &gossipv1.SignedObservationRequest{
|
|
ObservationRequest: b,
|
|
Signature: sig,
|
|
GuardianAddr: ethcrypto.PubkeyToAddress(gk.PublicKey).Bytes(),
|
|
}
|
|
|
|
envelope := &gossipv1.GossipMessage{
|
|
Message: &gossipv1.GossipMessage_SignedObservationRequest{
|
|
SignedObservationRequest: sReq}}
|
|
|
|
b, err = proto.Marshal(envelope)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// Send to local observation request queue (the loopback message is ignored)
|
|
obsvReqC <- msg
|
|
|
|
err = th.Publish(ctx, b)
|
|
p2pMessagesSent.Inc()
|
|
if err != nil {
|
|
logger.Error("failed to publish observation request", zap.Error(err))
|
|
} else {
|
|
logger.Info("published signed observation request", zap.Any("signed_observation_request", sReq))
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
for {
|
|
envelope, err := sub.Next(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to receive pubsub message: %w", err)
|
|
}
|
|
|
|
var msg gossipv1.GossipMessage
|
|
err = proto.Unmarshal(envelope.Data, &msg)
|
|
if err != nil {
|
|
logger.Info("received invalid message",
|
|
zap.Binary("data", envelope.Data),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
p2pMessagesReceived.WithLabelValues("invalid").Inc()
|
|
continue
|
|
}
|
|
|
|
if envelope.GetFrom() == h.ID() {
|
|
logger.Debug("received message from ourselves, ignoring",
|
|
zap.Any("payload", msg.Message))
|
|
p2pMessagesReceived.WithLabelValues("loopback").Inc()
|
|
continue
|
|
}
|
|
|
|
logger.Debug("received message",
|
|
zap.Any("payload", msg.Message),
|
|
zap.Binary("raw", envelope.Data),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
|
|
switch m := msg.Message.(type) {
|
|
case *gossipv1.GossipMessage_SignedHeartbeat:
|
|
s := m.SignedHeartbeat
|
|
gs := gst.Get()
|
|
if gs == nil {
|
|
// No valid guardian set yet - dropping heartbeat
|
|
logger.Debug("skipping heartbeat - no guardian set",
|
|
zap.Any("value", s),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
break
|
|
}
|
|
if heartbeat, err := processSignedHeartbeat(envelope.GetFrom(), s, gs, gst, disableHeartbeatVerify); err != nil {
|
|
p2pMessagesReceived.WithLabelValues("invalid_heartbeat").Inc()
|
|
logger.Debug("invalid signed heartbeat received",
|
|
zap.Error(err),
|
|
zap.Any("payload", msg.Message),
|
|
zap.Any("value", s),
|
|
zap.Binary("raw", envelope.Data),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
} else {
|
|
p2pMessagesReceived.WithLabelValues("valid_heartbeat").Inc()
|
|
logger.Debug("valid signed heartbeat received",
|
|
zap.Any("value", heartbeat),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
}
|
|
case *gossipv1.GossipMessage_SignedObservation:
|
|
obsvC <- m.SignedObservation
|
|
p2pMessagesReceived.WithLabelValues("observation").Inc()
|
|
case *gossipv1.GossipMessage_SignedVaaWithQuorum:
|
|
signedInC <- m.SignedVaaWithQuorum
|
|
p2pMessagesReceived.WithLabelValues("signed_vaa_with_quorum").Inc()
|
|
case *gossipv1.GossipMessage_SignedObservationRequest:
|
|
s := m.SignedObservationRequest
|
|
gs := gst.Get()
|
|
if gs == nil {
|
|
logger.Debug("dropping SignedObservationRequest - no guardian set",
|
|
zap.Any("value", s),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
break
|
|
}
|
|
r, err := processSignedObservationRequest(s, gs)
|
|
if err != nil {
|
|
p2pMessagesReceived.WithLabelValues("invalid_signed_observation_request").Inc()
|
|
logger.Debug("invalid signed observation request received",
|
|
zap.Error(err),
|
|
zap.Any("payload", msg.Message),
|
|
zap.Any("value", s),
|
|
zap.Binary("raw", envelope.Data),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
} else {
|
|
p2pMessagesReceived.WithLabelValues("signed_observation_request").Inc()
|
|
logger.Info("valid signed observation request received",
|
|
zap.Any("value", r),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
|
|
obsvReqC <- r
|
|
}
|
|
case *gossipv1.GossipMessage_SignedChainGovernorConfig:
|
|
logger.Debug("cgov: received config message")
|
|
if signedGovCfg != nil {
|
|
signedGovCfg <- m.SignedChainGovernorConfig
|
|
}
|
|
case *gossipv1.GossipMessage_SignedChainGovernorStatus:
|
|
logger.Debug("cgov: received status message")
|
|
if signedGovSt != nil {
|
|
signedGovSt <- m.SignedChainGovernorStatus
|
|
}
|
|
default:
|
|
p2pMessagesReceived.WithLabelValues("unknown").Inc()
|
|
logger.Warn("received unknown message type (running outdated software?)",
|
|
zap.Any("payload", msg.Message),
|
|
zap.Binary("raw", envelope.Data),
|
|
zap.String("from", envelope.GetFrom().String()))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func processSignedHeartbeat(from peer.ID, s *gossipv1.SignedHeartbeat, gs *node_common.GuardianSet, gst *node_common.GuardianSetState, disableVerify bool) (*gossipv1.Heartbeat, error) {
|
|
envelopeAddr := common.BytesToAddress(s.GuardianAddr)
|
|
idx, ok := gs.KeyIndex(envelopeAddr)
|
|
var pk common.Address
|
|
if !ok {
|
|
if !disableVerify {
|
|
return nil, fmt.Errorf("invalid message: %s not in guardian set", envelopeAddr)
|
|
}
|
|
} else {
|
|
pk = gs.Keys[idx]
|
|
}
|
|
|
|
digest := heartbeatDigest(s.Heartbeat)
|
|
|
|
// SECURITY: see whitepapers/0009_guardian_key.md
|
|
if len(heartbeatMessagePrefix)+len(s.Heartbeat) < 34 {
|
|
return nil, fmt.Errorf("invalid message: too short")
|
|
}
|
|
|
|
pubKey, err := ethcrypto.Ecrecover(digest.Bytes(), s.Signature)
|
|
if err != nil {
|
|
return nil, errors.New("failed to recover public key")
|
|
}
|
|
|
|
signerAddr := common.BytesToAddress(ethcrypto.Keccak256(pubKey[1:])[12:])
|
|
if pk != signerAddr && !disableVerify {
|
|
return nil, fmt.Errorf("invalid signer: %v", signerAddr)
|
|
}
|
|
|
|
var h gossipv1.Heartbeat
|
|
err = proto.Unmarshal(s.Heartbeat, &h)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal heartbeat: %w", err)
|
|
}
|
|
|
|
// Store verified heartbeat in global guardian set state.
|
|
if err := gst.SetHeartbeat(signerAddr, from, &h); err != nil {
|
|
return nil, fmt.Errorf("failed to store in guardian set state: %w", err)
|
|
}
|
|
|
|
collectNodeMetrics(signerAddr, from, &h)
|
|
|
|
return &h, nil
|
|
}
|
|
|
|
func processSignedObservationRequest(s *gossipv1.SignedObservationRequest, gs *node_common.GuardianSet) (*gossipv1.ObservationRequest, error) {
|
|
envelopeAddr := common.BytesToAddress(s.GuardianAddr)
|
|
idx, ok := gs.KeyIndex(envelopeAddr)
|
|
var pk common.Address
|
|
if !ok {
|
|
return nil, fmt.Errorf("invalid message: %s not in guardian set", envelopeAddr)
|
|
} else {
|
|
pk = gs.Keys[idx]
|
|
}
|
|
|
|
// SECURITY: see whitepapers/0009_guardian_key.md
|
|
if len(signedObservationRequestPrefix)+len(s.ObservationRequest) < 34 {
|
|
return nil, fmt.Errorf("invalid observation request: too short")
|
|
}
|
|
|
|
digest := signedObservationRequestDigest(s.ObservationRequest)
|
|
|
|
pubKey, err := ethcrypto.Ecrecover(digest.Bytes(), s.Signature)
|
|
if err != nil {
|
|
return nil, errors.New("failed to recover public key")
|
|
}
|
|
|
|
signerAddr := common.BytesToAddress(ethcrypto.Keccak256(pubKey[1:])[12:])
|
|
if pk != signerAddr {
|
|
return nil, fmt.Errorf("invalid signer: %v", signerAddr)
|
|
}
|
|
|
|
var h gossipv1.ObservationRequest
|
|
err = proto.Unmarshal(s.ObservationRequest, &h)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal observation request: %w", err)
|
|
}
|
|
|
|
// TODO: implement per-guardian rate limiting
|
|
|
|
return &h, nil
|
|
}
|