2023-05-03 15:32:24 -07:00
//nolint:unparam // this will be refactored in https://github.com/wormhole-foundation/wormhole/pull/1953
2020-10-29 02:13:14 -07:00
package processor
import (
"context"
"encoding/hex"
"fmt"
2022-04-08 16:37:23 -07:00
"time"
2021-08-30 07:19:00 -07:00
node_common "github.com/certusone/wormhole/node/pkg/common"
2021-09-13 06:03:26 -07:00
"github.com/certusone/wormhole/node/pkg/db"
2021-10-01 00:33:03 -07:00
"github.com/mr-tron/base58"
2021-01-24 08:20:04 -08:00
"github.com/prometheus/client_golang/prometheus"
2021-07-23 07:06:35 -07:00
"github.com/prometheus/client_golang/prometheus/promauto"
2020-10-29 02:13:14 -07:00
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"go.uber.org/zap"
2023-07-15 16:30:51 -07:00
"go.uber.org/zap/zapcore"
2020-10-29 02:13:14 -07:00
2021-08-26 01:35:09 -07:00
gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1"
2022-08-18 01:52:36 -07:00
"github.com/wormhole-foundation/wormhole/sdk/vaa"
2020-10-29 02:13:14 -07:00
)
2021-01-24 08:20:04 -08:00
var (
2021-07-23 07:06:35 -07:00
observationsReceivedTotal = promauto . NewCounter (
2021-01-24 08:20:04 -08:00
prometheus . CounterOpts {
Name : "wormhole_observations_received_total" ,
Help : "Total number of raw VAA observations received from gossip" ,
} )
2021-07-23 07:06:35 -07:00
observationsReceivedByGuardianAddressTotal = promauto . NewCounterVec (
2021-01-24 08:20:04 -08:00
prometheus . CounterOpts {
Name : "wormhole_observations_signed_by_guardian_total" ,
Help : "Total number of signed and verified VAA observations grouped by guardian address" ,
} , [ ] string { "addr" } )
2021-07-23 07:06:35 -07:00
observationsFailedTotal = promauto . NewCounterVec (
2021-01-24 08:20:04 -08:00
prometheus . CounterOpts {
Name : "wormhole_observations_verification_failures_total" ,
Help : "Total number of observations verification failure, grouped by failure reason" ,
} , [ ] string { "cause" } )
2021-07-23 07:06:35 -07:00
observationsUnknownTotal = promauto . NewCounter (
2021-01-24 08:20:04 -08:00
prometheus . CounterOpts {
2021-07-21 10:46:10 -07:00
Name : "wormhole_observations_unknown_total" ,
Help : "Total number of verified observations we haven't seen ourselves" ,
2021-01-24 08:20:04 -08:00
} )
)
2023-07-15 16:33:54 -07:00
// signaturesToVaaFormat converts a map[common.Address][]byte (processor state format) to []*vaa.Signature (VAA format) given a set of keys gsKeys
// It also returns a bool array indicating which key in gsKeys had a signature
// The processor state format is used for effeciently storing signatures during aggregation while the VAA format is more efficient for on-chain verification.
func signaturesToVaaFormat ( signatures map [ common . Address ] [ ] byte , gsKeys [ ] common . Address ) ( [ ] * vaa . Signature , [ ] bool ) {
// Aggregate all valid signatures into a list of vaa.Signature and construct signed VAA.
agg := make ( [ ] bool , len ( gsKeys ) )
var sigs [ ] * vaa . Signature
for i , a := range gsKeys {
sig , ok := signatures [ a ]
if ok {
var bs [ 65 ] byte
if n := copy ( bs [ : ] , sig ) ; n != 65 {
panic ( fmt . Sprintf ( "invalid sig len: %d" , n ) )
}
sigs = append ( sigs , & vaa . Signature {
Index : uint8 ( i ) ,
Signature : bs ,
} )
}
agg [ i ] = ok
}
return sigs , agg
}
2020-10-29 02:13:15 -07:00
// handleObservation processes a remote VAA observation, verifies it, checks whether the VAA has met quorum,
// and assembles and submits a valid VAA if possible.
2023-07-19 08:23:16 -07:00
func ( p * Processor ) handleObservation ( ctx context . Context , obs * node_common . MsgWithTimeStamp [ gossipv1 . SignedObservation ] ) {
2020-10-29 02:13:14 -07:00
// SECURITY: at this point, observations received from the p2p network are fully untrusted (all fields!)
//
// Note that observations are never tied to the (verified) p2p identity key - the p2p network
// identity is completely decoupled from the guardian identity, p2p is just transport.
2023-07-19 08:23:16 -07:00
m := obs . Msg
2021-01-24 08:51:21 -08:00
hash := hex . EncodeToString ( m . Hash )
2023-07-11 19:07:14 -07:00
s := p . state . signatures [ hash ]
2023-09-25 06:43:40 -07:00
if s != nil && s . submitted {
// already submitted; ignoring additional signatures for it.
2023-07-11 19:07:14 -07:00
return
}
2021-01-24 08:51:21 -08:00
2023-07-11 19:07:14 -07:00
if p . logger . Core ( ) . Enabled ( zapcore . DebugLevel ) {
p . logger . Debug ( "received observation" ,
zap . String ( "digest" , hash ) ,
zap . String ( "signature" , hex . EncodeToString ( m . Signature ) ) ,
zap . String ( "addr" , hex . EncodeToString ( m . Addr ) ) ,
zap . String ( "txhash" , hex . EncodeToString ( m . TxHash ) ) ,
zap . String ( "txhash_b58" , base58 . Encode ( m . TxHash ) ) ,
zap . String ( "message_id" , m . MessageId ) ,
)
}
2020-10-29 02:13:14 -07:00
2021-01-24 08:20:04 -08:00
observationsReceivedTotal . Inc ( )
2020-10-29 02:13:14 -07:00
// Verify the Guardian's signature. This verifies that m.Signature matches m.Hash and recovers
// the public key that was used to sign the payload.
pk , err := crypto . Ecrecover ( m . Hash , m . Signature )
if err != nil {
2021-01-20 15:35:52 -08:00
p . logger . Warn ( "failed to verify signature on observation" ,
2021-01-24 08:51:21 -08:00
zap . String ( "digest" , hash ) ,
2020-10-29 02:13:14 -07:00
zap . String ( "signature" , hex . EncodeToString ( m . Signature ) ) ,
zap . String ( "addr" , hex . EncodeToString ( m . Addr ) ) ,
zap . Error ( err ) )
2021-01-24 08:20:04 -08:00
observationsFailedTotal . WithLabelValues ( "invalid_signature" ) . Inc ( )
2020-10-29 02:13:14 -07:00
return
}
// Verify that m.Addr matches the public key that signed m.Hash.
their_addr := common . BytesToAddress ( m . Addr )
signer_pk := common . BytesToAddress ( crypto . Keccak256 ( pk [ 1 : ] ) [ 12 : ] )
if their_addr != signer_pk {
2021-01-20 15:35:52 -08:00
p . logger . Info ( "invalid observation - address does not match pubkey" ,
2021-01-24 08:51:21 -08:00
zap . String ( "digest" , hash ) ,
2020-10-29 02:13:14 -07:00
zap . String ( "signature" , hex . EncodeToString ( m . Signature ) ) ,
zap . String ( "addr" , hex . EncodeToString ( m . Addr ) ) ,
zap . String ( "pk" , signer_pk . Hex ( ) ) )
2021-01-24 08:20:04 -08:00
observationsFailedTotal . WithLabelValues ( "pubkey_mismatch" ) . Inc ( )
2020-10-29 02:13:14 -07:00
return
}
2021-01-24 08:51:21 -08:00
// Determine which guardian set to use. The following cases are possible:
//
2022-06-02 23:32:16 -07:00
// - We have already seen the message and generated ourObservation. In this case, use the guardian set valid at the time,
2021-01-24 08:51:21 -08:00
// even if the guardian set was updated. Old guardian sets remain valid for longer than aggregation state,
2021-07-21 10:46:10 -07:00
// and the guardians in the old set stay online and observe and sign messages for the transition period.
2021-01-24 08:51:21 -08:00
//
2021-07-21 10:46:10 -07:00
// - We have not yet seen the message. In this case, we assume the latest guardian set because that's what
// we will store once we do see the message.
2021-01-24 08:51:21 -08:00
//
2021-07-21 10:46:10 -07:00
// This ensures that during a guardian set update, a node which observed a given message with either the old
2021-01-24 08:51:21 -08:00
// or the new guardian set can achieve consensus, since both the old and the new set would achieve consensus,
2021-07-21 10:46:10 -07:00
// assuming that 2/3+ of the old and the new guardian set have seen the message and will periodically attempt
2021-01-24 08:51:21 -08:00
// to retransmit their observations such that nodes who initially dropped the signature will get a 2nd chance.
//
// During an update, vaaState.signatures can contain signatures from *both* guardian sets.
//
2021-08-30 07:19:00 -07:00
var gs * node_common . GuardianSet
2023-07-11 19:07:14 -07:00
if s != nil && s . gs != nil {
gs = s . gs
2021-01-24 08:51:21 -08:00
} else {
gs = p . gs
}
2021-03-23 06:07:47 -07:00
// We haven't yet observed the trusted guardian set on Ethereum, and therefore, it's impossible to verify it.
// May as well not have received it/been offline - drop it and wait for the guardian set.
if gs == nil {
p . logger . Warn ( "dropping observations since we haven't initialized our guardian set yet" ,
2021-08-09 15:38:19 -07:00
zap . String ( "digest" , hash ) ,
2021-03-23 06:07:47 -07:00
zap . String ( "their_addr" , their_addr . Hex ( ) ) ,
)
observationsFailedTotal . WithLabelValues ( "uninitialized_guardian_set" ) . Inc ( )
return
}
2021-01-24 08:51:21 -08:00
// Verify that m.Addr is included in the guardian set. If it's not, drop the message. In case it's us
// who have the outdated guardian set, we'll just wait for the message to be retransmitted eventually.
_ , ok := gs . KeyIndex ( their_addr )
2020-10-29 02:13:14 -07:00
if ! ok {
2021-10-12 05:50:22 -07:00
p . logger . Debug ( "received observation by unknown guardian - is our guardian set outdated?" ,
2021-08-09 15:38:19 -07:00
zap . String ( "digest" , hash ) ,
2020-10-29 02:13:14 -07:00
zap . String ( "their_addr" , their_addr . Hex ( ) ) ,
2021-01-24 08:51:21 -08:00
zap . Uint32 ( "index" , gs . Index ) ,
2023-07-11 19:07:14 -07:00
//zap.Any("keys", gs.KeysAsHexStrings()),
2020-10-29 02:13:14 -07:00
)
2021-01-24 08:20:04 -08:00
observationsFailedTotal . WithLabelValues ( "unknown_guardian" ) . Inc ( )
2020-10-29 02:13:14 -07:00
return
}
2020-11-20 13:35:00 -08:00
// Hooray! Now, we have verified all fields on SignedObservation and know that it includes
2020-10-29 02:13:14 -07:00
// a valid signature by an active guardian. We still don't fully trust them, as they may be
// byzantine, but now we know who we're dealing with.
2021-01-24 08:20:04 -08:00
// We can now count events by guardian without worry about cardinality explosions:
observationsReceivedByGuardianAddressTotal . WithLabelValues ( their_addr . Hex ( ) ) . Inc ( )
2020-10-29 02:13:14 -07:00
// []byte isn't hashable in a map. Paying a small extra cost for encoding for easier debugging.
2023-07-11 19:07:14 -07:00
if s == nil {
2021-01-24 08:20:04 -08:00
// We haven't yet seen this event ourselves, and therefore do not know what the VAA looks like.
2020-10-29 02:13:14 -07:00
// However, we have established that a valid guardian has signed it, and therefore we can
// already start aggregating signatures for it.
//
2021-07-21 10:46:10 -07:00
// A malicious guardian can potentially DoS this by creating fake observations at a faster rate than they decay,
2020-11-27 10:08:13 -08:00
// leading to a slow out-of-memory crash. We do not attempt to automatically mitigate spam attacks with valid
// signatures - such byzantine behavior would be plainly visible and would be dealt with by kicking them.
2021-07-21 10:46:10 -07:00
observationsUnknownTotal . Inc ( )
2021-01-24 08:20:04 -08:00
2023-07-11 19:07:14 -07:00
s = & state {
2020-10-29 02:13:14 -07:00
firstObserved : time . Now ( ) ,
2023-07-14 08:01:47 -07:00
nextRetry : time . Now ( ) . Add ( nextRetryDuration ( 0 ) ) ,
2020-10-29 02:13:14 -07:00
signatures : map [ common . Address ] [ ] byte { } ,
2021-01-25 11:12:28 -08:00
source : "unknown" ,
2020-10-29 02:13:14 -07:00
}
2023-07-11 19:07:14 -07:00
p . state . signatures [ hash ] = s
2020-10-29 02:13:14 -07:00
}
2023-07-11 19:07:14 -07:00
s . signatures [ their_addr ] = m . Signature
2020-10-29 02:13:14 -07:00
2023-07-11 19:07:14 -07:00
if s . ourObservation != nil {
2022-06-02 23:32:16 -07:00
// We have made this observation on chain!
2020-10-29 02:13:14 -07:00
2022-10-21 10:32:24 -07:00
quorum := vaa . CalculateQuorum ( len ( gs . Keys ) )
2020-10-29 02:13:14 -07:00
2023-07-15 16:33:54 -07:00
// Check if we have more signatures than required for quorum.
// s.signatures may contain signatures from multiple guardian sets during guardian set updates
// Hence, if len(s.signatures) < quorum, then there is definitely no quorum and we can return early to save additional computation,
// but if len(s.signatures) >= quorum, there is not necessarily quorum for the active guardian set.
// We will later check for quorum again after assembling the VAA for a particular guardian set.
if len ( s . signatures ) < quorum {
// no quorum yet, we're done here
p . logger . Debug ( "quorum not yet met" ,
zap . String ( "digest" , hash ) ,
zap . String ( "messageId" , m . MessageId ) ,
)
return
}
2020-10-29 02:13:14 -07:00
2023-07-15 16:33:54 -07:00
// Now we *may* have quorum, depending on the guardian set in use.
// Let's construct the VAA and check if we actually have quorum.
sigsVaaFormat , agg := signaturesToVaaFormat ( s . signatures , gs . Keys )
if p . logger . Level ( ) . Enabled ( zapcore . DebugLevel ) {
p . logger . Debug ( "aggregation state for observation" , // 1.3M out of 3M info messages / hour / guardian
zap . String ( "digest" , hash ) ,
zap . Any ( "set" , gs . KeysAsHexStrings ( ) ) ,
zap . Uint32 ( "index" , gs . Index ) ,
zap . Bools ( "aggregation" , agg ) ,
zap . Int ( "required_sigs" , quorum ) ,
zap . Int ( "have_sigs" , len ( sigsVaaFormat ) ) ,
zap . Bool ( "quorum" , len ( sigsVaaFormat ) >= quorum ) ,
)
}
if len ( sigsVaaFormat ) >= quorum && ! s . submitted {
// we have reached quorum *with the active guardian set*
s . ourObservation . HandleQuorum ( sigsVaaFormat , hash , p )
2020-10-29 02:13:14 -07:00
} else {
2023-07-05 12:02:29 -07:00
p . logger . Debug ( "quorum not met or already submitted, doing nothing" , // 1.2M out of 3M info messages / hour / guardian
2020-10-29 02:13:14 -07:00
zap . String ( "digest" , hash ) )
}
2020-11-19 03:53:19 -08:00
} else {
2023-07-05 12:02:29 -07:00
p . logger . Debug ( "we have not yet seen this observation - temporarily storing signature" , // 175K out of 3M info messages / hour / guardian
2023-07-15 16:33:54 -07:00
zap . String ( "digest" , hash ) )
2020-11-19 03:53:19 -08:00
2020-10-29 02:13:14 -07:00
}
2023-07-19 08:23:16 -07:00
observationTotalDelay . Observe ( float64 ( time . Since ( obs . Timestamp ) . Microseconds ( ) ) )
2020-10-29 02:13:14 -07:00
}
2021-09-13 06:03:26 -07:00
2022-10-06 05:57:25 -07:00
func ( p * Processor ) handleInboundSignedVAAWithQuorum ( ctx context . Context , m * gossipv1 . SignedVAAWithQuorum ) {
2021-09-13 06:03:26 -07:00
v , err := vaa . Unmarshal ( m . Vaa )
if err != nil {
p . logger . Warn ( "received invalid VAA in SignedVAAWithQuorum message" ,
zap . Error ( err ) , zap . Any ( "message" , m ) )
return
}
2023-06-30 07:38:08 -07:00
// Check if we already store this VAA
2023-07-15 16:30:51 -07:00
if p . haveSignedVAA ( * db . VaaIDFromVAA ( v ) ) {
if p . logger . Level ( ) . Enabled ( zapcore . DebugLevel ) {
p . logger . Debug ( "ignored SignedVAAWithQuorum message for VAA we already stored" ,
zap . String ( "vaaID" , string ( db . VaaIDFromVAA ( v ) . Bytes ( ) ) ) ,
)
}
2023-06-30 07:38:08 -07:00
return
}
2021-09-13 06:03:26 -07:00
if p . gs == nil {
p . logger . Warn ( "dropping SignedVAAWithQuorum message since we haven't initialized our guardian set yet" ,
2023-07-11 19:07:14 -07:00
zap . String ( "digest" , hex . EncodeToString ( v . SigningDigest ( ) . Bytes ( ) ) ) ,
2021-09-13 06:03:26 -07:00
zap . Any ( "message" , m ) ,
)
return
}
2022-04-11 16:14:41 -07:00
// Check if guardianSet doesn't have any keys
if len ( p . gs . Keys ) == 0 {
p . logger . Warn ( "dropping SignedVAAWithQuorum message since we have a guardian set without keys" ,
2023-07-11 19:07:14 -07:00
zap . String ( "digest" , hex . EncodeToString ( v . SigningDigest ( ) . Bytes ( ) ) ) ,
2022-04-11 16:14:41 -07:00
zap . Any ( "message" , m ) ,
)
return
}
2022-10-21 10:32:24 -07:00
if err := v . Verify ( p . gs . Keys ) ; err != nil {
p . logger . Warn ( "dropping SignedVAAWithQuorum message because it failed verification: " + err . Error ( ) )
2021-09-13 06:03:26 -07:00
return
}
// We now established that:
// - all signatures on the VAA are valid
// - the signature's addresses match the node's current guardian set
// - enough signatures are present for the VAA to reach quorum
// Store signed VAA in database.
2023-07-11 19:07:14 -07:00
if p . logger . Level ( ) . Enabled ( zapcore . DebugLevel ) {
p . logger . Debug ( "storing inbound signed VAA with quorum" ,
zap . String ( "digest" , hex . EncodeToString ( v . SigningDigest ( ) . Bytes ( ) ) ) ,
zap . Any ( "vaa" , v ) ,
zap . String ( "bytes" , hex . EncodeToString ( m . Vaa ) ) ,
zap . String ( "message_id" , v . MessageID ( ) ) )
}
2021-09-13 06:03:26 -07:00
2022-09-27 06:14:52 -07:00
if err := p . storeSignedVAA ( v ) ; err != nil {
2021-09-13 06:03:26 -07:00
p . logger . Error ( "failed to store signed VAA" , zap . Error ( err ) )
return
}
}