From 6ab6824d82e4c349e401ccd615cbe42838dae9b1 Mon Sep 17 00:00:00 2001 From: Mariano <9205080+marianososto@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:43:53 -0300 Subject: [PATCH] [1520] Retrieve the feeUSD from coingecko in tx-tracker [WIP] (#1541) * start implementing retrieving the feeUSD from coingecko * add missing pricesApi in the initialization of tx-tracker service * add missing initialization * add missing cfg for coingecko properties * fix solana in url * replace coingecko api call with notional redis client * add native token list in commons * add decimal places * change feeUSD to Raw.GasPrice * add check for mainnet * replace with sdk const for more coupling * replace pricesApi with notionalCache * undo changes on pricesApi * undo more stuff * remove lines * remove unused timestamp * remove unused props * remove * fix indent * fix import * change * add cache configs * change token_address to native * fixes * fix BNB coingeckoID * fix polygon coingecko_id * change to lowercase * adjust type FeeDoc in operations endpoint * change Blast native token * pr review changes * code-review fixes * comment wormchain * rename var * add comment to wormchain gas token map entry --- api/handlers/operations/model.go | 5 +- common/domain/chainid.go | 11 ++++ common/domain/manual_mainnet_tokens.go | 55 ++++++++++++++++++++ deploy/tx-tracker/env/production-mainnet.env | 1 + deploy/tx-tracker/env/production-testnet.env | 1 + deploy/tx-tracker/env/staging-mainnet.env | 1 + deploy/tx-tracker/env/staging-testnet.env | 1 + deploy/tx-tracker/tx-tracker-service.yaml | 12 +++++ tx-tracker/chains/api_evm.go | 38 +++++++++----- tx-tracker/chains/api_solana.go | 22 ++++++-- tx-tracker/chains/chains.go | 16 ++++-- tx-tracker/chains/util.go | 10 ++++ tx-tracker/cmd/backfiller/vaas.go | 18 +++++-- tx-tracker/cmd/service/run.go | 18 +++++-- tx-tracker/config/structs.go | 22 +++++--- tx-tracker/consumer/consumer.go | 13 +++-- tx-tracker/consumer/repository.go | 6 ++- tx-tracker/consumer/source_processor.go | 5 +- tx-tracker/consumer/target_processor.go | 47 ++++++++++++----- tx-tracker/http/vaa/controller.go | 18 ++++--- 20 files changed, 256 insertions(+), 64 deletions(-) diff --git a/api/handlers/operations/model.go b/api/handlers/operations/model.go index 48324b4f..dc746539 100644 --- a/api/handlers/operations/model.go +++ b/api/handlers/operations/model.go @@ -76,7 +76,10 @@ type AttributeDoc struct { } type FeeDoc struct { - Fee string `bson:"fee" json:"fee"` + Fee string `bson:"fee" json:"fee"` + RawFee map[string]any `bson:"rawFee" json:"rawFee"` + GasTokenNotional string `bson:"gasTokenNotional" json:"gasTokenNotional"` + FeeUSD string `bson:"feeUSD" json:"feeUSD"` } // DestinationTx represents a destination transaction. diff --git a/common/domain/chainid.go b/common/domain/chainid.go index e40f6622..6c038610 100644 --- a/common/domain/chainid.go +++ b/common/domain/chainid.go @@ -31,6 +31,8 @@ var ( // NFT Bridge "0000000000000000000000000000000000000000000000000000000000000005": "0x1bdffae984043833ed7fe223f7af7a3f8902d04129b14f801823e64827da7130", } + + gasTokenList = GasTokenList() ) var allChainIDs = make(map[sdk.ChainID]bool) @@ -399,3 +401,12 @@ func encodeBech32(hrp string, data []byte) (string, error) { return bech32.Encode(hrp, aligned) } + +func GetGasTokenMetadata(chainID sdk.ChainID) *TokenMetadata { + for i := 0; i < len(gasTokenList); i++ { + if gasTokenList[i].TokenChain == chainID { + return &gasTokenList[i] + } + } + return nil +} diff --git a/common/domain/manual_mainnet_tokens.go b/common/domain/manual_mainnet_tokens.go index 24303897..c8cb8a6a 100644 --- a/common/domain/manual_mainnet_tokens.go +++ b/common/domain/manual_mainnet_tokens.go @@ -1,5 +1,7 @@ package domain +import sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" + // manualMainnetTokenList returns a list of tokens that are not generated automatically. func manualMainnetTokenList() []TokenMetadata { return []TokenMetadata{ @@ -32,9 +34,62 @@ func manualMainnetTokenList() []TokenMetadata { // mainnetTokenList returns a list of all tokens on the mainnet. func mainnetTokenList() []TokenMetadata { res := append(generatedMainnetTokenList(), manualMainnetTokenList()...) + res = append(res, GasTokenList()...) return append(res, unknownTokenList()...) } +// GasTokenList : gas tokens are the ones used to pay gas fees on the respective chains, they don't belong to a contract address. +func GasTokenList() []TokenMetadata { + const nativeTokenAddress = "0000000000000000000000000000000000000000000000000000000000000000" + return []TokenMetadata{ + {TokenChain: sdk.ChainIDSolana, TokenAddress: nativeTokenAddress, Symbol: "SOL", CoingeckoID: "solana", Decimals: 9}, + {TokenChain: sdk.ChainIDEthereum, TokenAddress: nativeTokenAddress, Symbol: "ETH", CoingeckoID: "ethereum", Decimals: 18}, + {TokenChain: sdk.ChainIDTerra, TokenAddress: nativeTokenAddress, Symbol: "LUNA", CoingeckoID: "terra-luna", Decimals: 6}, + {TokenChain: sdk.ChainIDBSC, TokenAddress: nativeTokenAddress, Symbol: "BNB", CoingeckoID: "binancecoin", Decimals: 18}, + {TokenChain: sdk.ChainIDPolygon, TokenAddress: nativeTokenAddress, Symbol: "MATIC", CoingeckoID: "matic-network", Decimals: 18}, + {TokenChain: sdk.ChainIDAvalanche, TokenAddress: nativeTokenAddress, Symbol: "AVAX", CoingeckoID: "avalanche-2", Decimals: 18}, + {TokenChain: sdk.ChainIDOasis, TokenAddress: nativeTokenAddress, Symbol: "ROSE", CoingeckoID: "oasis-network", Decimals: 18}, + {TokenChain: sdk.ChainIDAlgorand, TokenAddress: nativeTokenAddress, Symbol: "ALGO", CoingeckoID: "algorand", Decimals: 6}, + {TokenChain: sdk.ChainIDAurora, TokenAddress: nativeTokenAddress, Symbol: "AOA", CoingeckoID: "aurora", Decimals: 18}, + {TokenChain: sdk.ChainIDFantom, TokenAddress: nativeTokenAddress, Symbol: "FTM", CoingeckoID: "fantom", Decimals: 18}, + {TokenChain: sdk.ChainIDKarura, TokenAddress: nativeTokenAddress, Symbol: "KAR", CoingeckoID: "karura", Decimals: 12}, + {TokenChain: sdk.ChainIDAcala, TokenAddress: nativeTokenAddress, Symbol: "ACA", CoingeckoID: "acala", Decimals: 18}, + {TokenChain: sdk.ChainIDKlaytn, TokenAddress: nativeTokenAddress, Symbol: "KLAY", CoingeckoID: "klay-token", Decimals: 18}, + {TokenChain: sdk.ChainIDCelo, TokenAddress: nativeTokenAddress, Symbol: "CELO", CoingeckoID: "celo", Decimals: 18}, + {TokenChain: sdk.ChainIDNear, TokenAddress: nativeTokenAddress, Symbol: "NEAR", CoingeckoID: "near", Decimals: 24}, + {TokenChain: sdk.ChainIDMoonbeam, TokenAddress: nativeTokenAddress, Symbol: "GLMR", CoingeckoID: "moonbeam", Decimals: 18}, + {TokenChain: sdk.ChainIDTerra2, TokenAddress: nativeTokenAddress, Symbol: "LUNA", CoingeckoID: "terra-luna-2", Decimals: 6}, + {TokenChain: sdk.ChainIDInjective, TokenAddress: nativeTokenAddress, Symbol: "INJ", CoingeckoID: "injective-protocol", Decimals: 18}, + {TokenChain: sdk.ChainIDOsmosis, TokenAddress: nativeTokenAddress, Symbol: "OSMO", CoingeckoID: "osmosis", Decimals: 6}, + {TokenChain: sdk.ChainIDSui, TokenAddress: nativeTokenAddress, Symbol: "SUI", CoingeckoID: "sui", Decimals: 9}, + {TokenChain: sdk.ChainIDAptos, TokenAddress: nativeTokenAddress, Symbol: "APT", CoingeckoID: "aptos", Decimals: 8}, + {TokenChain: sdk.ChainIDArbitrum, TokenAddress: nativeTokenAddress, Symbol: "ARB", CoingeckoID: "arbitrum", Decimals: 18}, + {TokenChain: sdk.ChainIDOptimism, TokenAddress: nativeTokenAddress, Symbol: "OP", CoingeckoID: "optimism", Decimals: 18}, + {TokenChain: sdk.ChainIDGnosis, TokenAddress: nativeTokenAddress, Symbol: "GNO", CoingeckoID: "gnosis", Decimals: 18}, + {TokenChain: sdk.ChainIDXpla, TokenAddress: nativeTokenAddress, Symbol: "XPLA", CoingeckoID: "xpla", Decimals: 18}, + {TokenChain: sdk.ChainIDBtc, TokenAddress: nativeTokenAddress, Symbol: "BTC", CoingeckoID: "bitcoin", Decimals: 8}, + {TokenChain: sdk.ChainIDBase, TokenAddress: nativeTokenAddress, Symbol: "ETH", CoingeckoID: "ethereum", Decimals: 18}, + {TokenChain: sdk.ChainIDSei, TokenAddress: nativeTokenAddress, Symbol: "SEI", CoingeckoID: "sei-network", Decimals: 6}, + {TokenChain: sdk.ChainIDRootstock, TokenAddress: nativeTokenAddress, Symbol: "RSK", CoingeckoID: "rootstock", Decimals: 18}, + {TokenChain: sdk.ChainIDScroll, TokenAddress: nativeTokenAddress, Symbol: "ETH", CoingeckoID: "ethereum", Decimals: 18}, + {TokenChain: sdk.ChainIDMantle, TokenAddress: nativeTokenAddress, Symbol: "MNT", CoingeckoID: "mantle", Decimals: 18}, + {TokenChain: sdk.ChainIDBlast, TokenAddress: nativeTokenAddress, Symbol: "ETH", CoingeckoID: "ethereum", Decimals: 18}, + {TokenChain: sdk.ChainIDXLayer, TokenAddress: nativeTokenAddress, Symbol: "XLYR", CoingeckoID: "xlayer", Decimals: 18}, + {TokenChain: sdk.ChainIDLinea, TokenAddress: nativeTokenAddress, Symbol: "ETH", CoingeckoID: "ethereum", Decimals: 18}, + {TokenChain: sdk.ChainIDBerachain, TokenAddress: nativeTokenAddress, Symbol: "BERA", CoingeckoID: "berachain-bera", Decimals: 18}, + //{TokenChain: sdk.ChainIDWormchain, TokenAddress: nativeTokenAddress, Symbol: "WORM", CoingeckoID: "wormchain", Decimals: 18}, // Currently Wormchain doesn't charge gas fees to wormhole messages. This may change in the future: https://docs.wormhole.com/wormhole/explore-wormhole/gateway + {TokenChain: sdk.ChainIDCosmoshub, TokenAddress: nativeTokenAddress, Symbol: "ATOM", CoingeckoID: "cosmos", Decimals: 6}, + {TokenChain: sdk.ChainIDEvmos, TokenAddress: nativeTokenAddress, Symbol: "EVMOS", CoingeckoID: "evmos", Decimals: 18}, + {TokenChain: sdk.ChainIDKujira, TokenAddress: nativeTokenAddress, Symbol: "KUJI", CoingeckoID: "kujira", Decimals: 6}, + {TokenChain: sdk.ChainIDNeutron, TokenAddress: nativeTokenAddress, Symbol: "NEUT", CoingeckoID: "neutron-3", Decimals: 6}, + {TokenChain: sdk.ChainIDCelestia, TokenAddress: nativeTokenAddress, Symbol: "TIA", CoingeckoID: "celestia", Decimals: 6}, + {TokenChain: sdk.ChainIDStargaze, TokenAddress: nativeTokenAddress, Symbol: "STARS", CoingeckoID: "stargaze", Decimals: 6}, + {TokenChain: sdk.ChainIDSeda, TokenAddress: nativeTokenAddress, Symbol: "SEDA", CoingeckoID: "seda-2", Decimals: 18}, + {TokenChain: sdk.ChainIDDymension, TokenAddress: nativeTokenAddress, Symbol: "DYM", CoingeckoID: "dymension", Decimals: 18}, + {TokenChain: sdk.ChainIDProvenance, TokenAddress: nativeTokenAddress, Symbol: "HASH", CoingeckoID: "provenance-blockchain", Decimals: 9}, + } +} + func unknownTokenList() []TokenMetadata { return []TokenMetadata{ {TokenChain: 23, TokenAddress: "00000000000000000000000007dd5beaffb65b8ff2e575d500bdf324a05295dc", Symbol: "arbi", CoingeckoID: "arbipad", Decimals: 18}, diff --git a/deploy/tx-tracker/env/production-mainnet.env b/deploy/tx-tracker/env/production-mainnet.env index cbd73111..73304c46 100644 --- a/deploy/tx-tracker/env/production-mainnet.env +++ b/deploy/tx-tracker/env/production-mainnet.env @@ -17,6 +17,7 @@ SQS_AWS_REGION= P2P_NETWORK=mainnet AWS_IAM_ROLE= METRICS_ENABLED=true +NOTIONAL_CACHE_CHANNEL=WORMSCAN:NOTIONAL ACALA_BASE_URL=https://eth-rpc-acala.aca-api.network ACALA_REQUESTS_PER_MINUTE=12 diff --git a/deploy/tx-tracker/env/production-testnet.env b/deploy/tx-tracker/env/production-testnet.env index 480af5df..1bcf1444 100644 --- a/deploy/tx-tracker/env/production-testnet.env +++ b/deploy/tx-tracker/env/production-testnet.env @@ -17,6 +17,7 @@ SQS_AWS_REGION= P2P_NETWORK=testnet AWS_IAM_ROLE= METRICS_ENABLED=true +NOTIONAL_CACHE_CHANNEL=WORMSCAN:NOTIONAL ACALA_BASE_URL=https://acala-dev.aca-dev.network/eth/http ACALA_REQUESTS_PER_MINUTE=12 diff --git a/deploy/tx-tracker/env/staging-mainnet.env b/deploy/tx-tracker/env/staging-mainnet.env index 42fb847d..e8c94ab4 100644 --- a/deploy/tx-tracker/env/staging-mainnet.env +++ b/deploy/tx-tracker/env/staging-mainnet.env @@ -17,6 +17,7 @@ SQS_AWS_REGION= P2P_NETWORK=mainnet AWS_IAM_ROLE= METRICS_ENABLED=true +NOTIONAL_CACHE_CHANNEL=WORMSCAN:NOTIONAL ACALA_BASE_URL=https://eth-rpc-acala.aca-api.network ACALA_REQUESTS_PER_MINUTE=12 diff --git a/deploy/tx-tracker/env/staging-testnet.env b/deploy/tx-tracker/env/staging-testnet.env index 2532e810..17dac132 100644 --- a/deploy/tx-tracker/env/staging-testnet.env +++ b/deploy/tx-tracker/env/staging-testnet.env @@ -17,6 +17,7 @@ SQS_AWS_REGION= P2P_NETWORK=testnet AWS_IAM_ROLE= METRICS_ENABLED=true +NOTIONAL_CACHE_CHANNEL=WORMSCAN:NOTIONAL ACALA_BASE_URL=https://acala-dev.aca-dev.network/eth/http ACALA_REQUESTS_PER_MINUTE=12 diff --git a/deploy/tx-tracker/tx-tracker-service.yaml b/deploy/tx-tracker/tx-tracker-service.yaml index d03bd970..1fd7e254 100644 --- a/deploy/tx-tracker/tx-tracker-service.yaml +++ b/deploy/tx-tracker/tx-tracker-service.yaml @@ -75,6 +75,18 @@ spec: value: "/opt/tx-tracker/rpc-provider.json" - name: CONSUMER_WORKERS_SIZE value: "1" + - name: NOTIONAL_CACHE_CHANNEL + value: {{ .NOTIONAL_CACHE_CHANNEL }} + - name: NOTIONAL_CACHE_URL + valueFrom: + configMapKeyRef: + name: config + key: redis-uri + - name: NOTIONAL_CACHE_PREFIX + valueFrom: + configMapKeyRef: + name: config + key: redis-prefix image: {{ .IMAGE_NAME }} imagePullPolicy: Always livenessProbe: diff --git a/tx-tracker/chains/api_evm.go b/tx-tracker/chains/api_evm.go index 63a10101..c9cbefc0 100644 --- a/tx-tracker/chains/api_evm.go +++ b/tx-tracker/chains/api_evm.go @@ -3,14 +3,15 @@ package chains import ( "context" "fmt" - "math/big" - "strings" - "github.com/shopspring/decimal" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" + "github.com/wormhole-foundation/wormhole-explorer/common/domain" "github.com/wormhole-foundation/wormhole-explorer/common/pool" "github.com/wormhole-foundation/wormhole-explorer/txtracker/internal/metrics" sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" "go.uber.org/zap" + "math/big" + "strings" ) const ( @@ -35,7 +36,9 @@ type ethGetTransactionReceiptResponse struct { } type apiEvm struct { - chainId sdk.ChainID + chainId sdk.ChainID + notionalCache *notional.NotionalCache + p2pNetwork string } func (e *apiEvm) FetchEvmTx( @@ -75,12 +78,23 @@ func (e *apiEvm) FetchEvmTx( zap.Error(err), zap.String("txHash", txHash), zap.String("chainId", e.chainId.String())) - } else if fee == "" { + } else if fee == nil { txDetail.FeeDetail = nil } else { - txDetail.FeeDetail.Fee = fee + txDetail.FeeDetail.Fee = fee.String() + if e.p2pNetwork == domain.P2pMainNet { + gasPrice, errGasPrice := GetGasTokenNotional(e.chainId, e.notionalCache) + if errGasPrice != nil { + logger.Error("Failed to get gas price", + zap.Error(errGasPrice), + zap.String("chainId", e.chainId.String()), + zap.String("txHash", txHash)) + } else { + txDetail.FeeDetail.GasTokenNotional = gasPrice.NotionalUsd.String() + txDetail.FeeDetail.FeeUSD = gasPrice.NotionalUsd.Mul(*fee).String() + } + } } - } return txDetail, err @@ -176,17 +190,17 @@ func (e *apiEvm) fetchEvmTxReceiptByTxHash( }, nil } -func EvmCalculateFee(chainID sdk.ChainID, gasUsed string, effectiveGasPrice string) (string, error) { +func EvmCalculateFee(chainID sdk.ChainID, gasUsed string, effectiveGasPrice string) (*decimal.Decimal, error) { //ignore if the blockchain is L2 if chainID == sdk.ChainIDBase || chainID == sdk.ChainIDOptimism || chainID == sdk.ChainIDScroll { - return "", nil + return nil, nil } // get decimal gasUsed gs := new(big.Int) _, ok := gs.SetString(gasUsed, 0) if !ok { - return "", fmt.Errorf("failed to convert gasUsed to big.Int") + return nil, fmt.Errorf("failed to convert gasUsed to big.Int") } decimalGasUsed := decimal.NewFromBigInt(gs, 0) @@ -194,12 +208,12 @@ func EvmCalculateFee(chainID sdk.ChainID, gasUsed string, effectiveGasPrice stri gp := new(big.Int) _, ok = gp.SetString(effectiveGasPrice, 0) if !ok { - return "", fmt.Errorf("failed to convert gasPrice to big.Int") + return nil, fmt.Errorf("failed to convert gasPrice to big.Int") } decimalGasPrice := decimal.NewFromBigInt(gp, 0) // calculate gasUsed * (gasPrice / 1e18) decimalFee := decimalGasUsed.Mul(decimalGasPrice) decimalFee = decimalFee.DivRound(decimal.NewFromInt(1e18), 18) - return decimalFee.String(), nil + return &decimalFee, nil } diff --git a/tx-tracker/chains/api_solana.go b/tx-tracker/chains/api_solana.go index dee0409a..c10a0d38 100644 --- a/tx-tracker/chains/api_solana.go +++ b/tx-tracker/chains/api_solana.go @@ -4,6 +4,8 @@ import ( "context" "encoding/hex" "fmt" + notional "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" + "github.com/wormhole-foundation/wormhole-explorer/common/domain" "time" "github.com/mr-tron/base58" @@ -58,7 +60,9 @@ type getTransactionConfig struct { } type apiSolana struct { - timestamp *time.Time + timestamp *time.Time + notionalCache *notional.NotionalCache + p2pNetwork string } func (a *apiSolana) FetchSolanaTx( @@ -92,6 +96,16 @@ func (a *apiSolana) FetchSolanaTx( } } + if txDetail.FeeDetail != nil && txDetail.FeeDetail.Fee != "" && a.p2pNetwork == domain.P2pMainNet { + gasPrice, errGasPrice := GetGasTokenNotional(sdk.ChainIDSolana, a.notionalCache) + if errGasPrice != nil { + logger.Error("Failed to get gas price", zap.Error(errGasPrice), zap.String("chainId", sdk.ChainIDSolana.String()), zap.String("txHash", txHash)) + } else { + txDetail.FeeDetail.GasTokenNotional = gasPrice.NotionalUsd.String() + txDetail.FeeDetail.FeeUSD = gasPrice.NotionalUsd.Mul(decimal.RequireFromString(txDetail.FeeDetail.Fee)).String() + } + } + return txDetail, err } @@ -196,15 +210,15 @@ func (a *apiSolana) fetchSolanaTx( "fee": fmt.Sprintf("%d", *response.Meta.Fee), }, } - feeDetail.Fee = SolanaCalculateFee(*response.Meta.Fee) + feeDetail.Fee = SolanaCalculateFee(*response.Meta.Fee).String() txDetail.FeeDetail = feeDetail } return &txDetail, nil } -func SolanaCalculateFee(fee uint64) string { +func SolanaCalculateFee(fee uint64) decimal.Decimal { rawFee := decimal.NewFromUint64(fee) calculatedFee := rawFee.DivRound(decimal.NewFromInt(1e9), 9) - return calculatedFee.String() + return calculatedFee } diff --git a/tx-tracker/chains/chains.go b/tx-tracker/chains/chains.go index de1bcac3..97fd3d2a 100644 --- a/tx-tracker/chains/chains.go +++ b/tx-tracker/chains/chains.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + notional "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "time" "github.com/wormhole-foundation/wormhole-explorer/common/pool" @@ -29,8 +30,10 @@ type TxDetail struct { } type FeeDetail struct { - Fee string `bson:"fee" json:"fee"` - RawFee map[string]string `bson:"rawFee" json:"rawFee"` + Fee string `bson:"fee" json:"fee"` + RawFee map[string]string `bson:"rawFee" json:"rawFee"` + GasTokenNotional string `bson:"gasTokenNotional" json:"gasTokenNotional"` + FeeUSD string `bson:"feeUSD" json:"feeUSD"` } type AttributeTxDetail struct { @@ -48,13 +51,16 @@ func FetchTx( p2pNetwork string, m metrics.Metrics, logger *zap.Logger, + notionalCache *notional.NotionalCache, ) (*TxDetail, error) { // Decide which RPC/API service to use based on chain ID var fetchFunc func(ctx context.Context, pool *pool.Pool, txHash string, metrics metrics.Metrics, logger *zap.Logger) (*TxDetail, error) switch chainId { case sdk.ChainIDSolana: apiSolana := &apiSolana{ - timestamp: timestamp, + timestamp: timestamp, + notionalCache: notionalCache, + p2pNetwork: p2pNetwork, } fetchFunc = apiSolana.FetchSolanaTx case sdk.ChainIDAlgorand: @@ -95,7 +101,9 @@ func FetchTx( sdk.ChainIDMantle, sdk.ChainIDPolygonSepolia: // polygon amoy apiEvm := &apiEvm{ - chainId: chainId, + chainId: chainId, + notionalCache: notionalCache, + p2pNetwork: p2pNetwork, } fetchFunc = apiEvm.FetchEvmTx case sdk.ChainIDWormchain: diff --git a/tx-tracker/chains/util.go b/tx-tracker/chains/util.go index d5010c5f..b6660126 100644 --- a/tx-tracker/chains/util.go +++ b/tx-tracker/chains/util.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" + "github.com/wormhole-foundation/wormhole-explorer/common/domain" "io" "net/http" "strings" @@ -145,3 +147,11 @@ func FormatTxHashByChain(chainId sdk.ChainID, txHash string) string { return txHash } } + +func GetGasTokenNotional(chainID sdk.ChainID, notionalCache *notional.NotionalCache) (notional.PriceData, error) { + nativeToken := domain.GetGasTokenMetadata(chainID) + if nativeToken == nil { + return notional.PriceData{}, fmt.Errorf("gas token not found for chain %s", chainID) + } + return notionalCache.Get(nativeToken.GetTokenID()) +} diff --git a/tx-tracker/cmd/backfiller/vaas.go b/tx-tracker/cmd/backfiller/vaas.go index 98ca6211..05f3e6e1 100644 --- a/tx-tracker/cmd/backfiller/vaas.go +++ b/tx-tracker/cmd/backfiller/vaas.go @@ -3,6 +3,8 @@ package backfiller import ( "context" "errors" + "github.com/go-redis/redis/v8" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "log" "sync" "sync/atomic" @@ -103,6 +105,16 @@ func RunByVaas(backfillerConfig *VaasBackfiller) { // create a consumer repository. globalTrxRepository := consumer.NewRepository(logger, db.Database) + redisClient := redis.NewClient(&redis.Options{Addr: cfg.NotionalCacheURL}) + notionalCache, errCache := notional.NewNotionalCache(ctx, redisClient, cfg.NotionalCachePrefix, cfg.NotionalCacheChannel, logger) + if errCache != nil { + logger.Fatal("Failed to create notional cache", zap.Error(errCache)) + } + errCache = notionalCache.Init(ctx) + if errCache != nil { + logger.Fatal("Failed to initialize notional cache", zap.Error(errCache)) + } + query := repository.VaaQuery{ StartTime: &startTime, EndTime: &endTime, @@ -142,7 +154,7 @@ func RunByVaas(backfillerConfig *VaasBackfiller) { processedDocumentsSuccess: &quantityConsumedSuccess, processedDocumentsWithError: &quantityConsumedWithError, } - go processVaa(ctx, &p) + go processVaa(ctx, &p, notionalCache) } logger.Info("Waiting for all workers to finish...") @@ -195,7 +207,7 @@ func getVaas(ctx context.Context, logger *zap.Logger, pagination repository.Pagi } } -func processVaa(ctx context.Context, params *vaasBackfillerParams) { +func processVaa(ctx context.Context, params *vaasBackfillerParams, cache *notional.NotionalCache) { // Main loop: fetch global txs and process them metrics := metrics.NewDummyMetrics() defer params.wg.Done() @@ -226,7 +238,7 @@ func processVaa(ctx context.Context, params *vaasBackfillerParams) { Metrics: metrics, DisableDBUpsert: params.disableDBUpsert, } - _, err := consumer.ProcessSourceTx(ctx, params.logger, params.rpcPool, params.wormchainRpcPool, params.repository, &p, params.p2pNetwork) + _, err := consumer.ProcessSourceTx(ctx, params.logger, params.rpcPool, params.wormchainRpcPool, params.repository, &p, params.p2pNetwork, cache) if err != nil { if errors.Is(err, consumer.ErrAlreadyProcessed) { params.logger.Info("Source tx was already processed", zap.String("vaaId", v.ID)) diff --git a/tx-tracker/cmd/service/run.go b/tx-tracker/cmd/service/run.go index 98b63110..24662d32 100644 --- a/tx-tracker/cmd/service/run.go +++ b/tx-tracker/cmd/service/run.go @@ -3,6 +3,8 @@ package service import ( "context" "errors" + "github.com/go-redis/redis/v8" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "log" "os" "os/signal" @@ -64,8 +66,18 @@ func Run() { repository := consumer.NewRepository(logger, db.Database) vaaRepository := vaa.NewRepository(db.Database, logger) + redisClient := redis.NewClient(&redis.Options{Addr: cfg.NotionalCacheURL}) + notionalCache, errCache := notional.NewNotionalCache(rootCtx, redisClient, cfg.NotionalCachePrefix, cfg.NotionalCacheChannel, logger) + if errCache != nil { + logger.Fatal("Failed to create notional cache", zap.Error(errCache)) + } + errCache = notionalCache.Init(rootCtx) + if errCache != nil { + logger.Fatal("Failed to initialize notional cache", zap.Error(errCache)) + } + // create controller - vaaController := vaa.NewController(rpcPool, wormchainRpcPool, vaaRepository, repository, cfg.P2pNetwork, logger) + vaaController := vaa.NewController(rpcPool, wormchainRpcPool, vaaRepository, repository, cfg.P2pNetwork, logger, notionalCache) // start serving /health and /ready endpoints healthChecks, err := makeHealthChecks(rootCtx, cfg, db.Database) @@ -77,12 +89,12 @@ func Run() { // create and start a pipeline consumer. vaaConsumeFunc := newVAAConsumeFunc(rootCtx, cfg, metrics, logger) - vaaConsumer := consumer.New(vaaConsumeFunc, rpcPool, wormchainRpcPool, rootCtx, logger, repository, metrics, cfg.P2pNetwork, cfg.ConsumerWorkersSize) + vaaConsumer := consumer.New(vaaConsumeFunc, rpcPool, wormchainRpcPool, logger, repository, metrics, cfg.P2pNetwork, cfg.ConsumerWorkersSize, notionalCache) vaaConsumer.Start(rootCtx) // create and start a notification consumer. notificationConsumeFunc := newNotificationConsumeFunc(rootCtx, cfg, metrics, logger) - notificationConsumer := consumer.New(notificationConsumeFunc, rpcPool, wormchainRpcPool, rootCtx, logger, repository, metrics, cfg.P2pNetwork, cfg.ConsumerWorkersSize) + notificationConsumer := consumer.New(notificationConsumeFunc, rpcPool, wormchainRpcPool, logger, repository, metrics, cfg.P2pNetwork, cfg.ConsumerWorkersSize, notionalCache) notificationConsumer.Start(rootCtx) logger.Info("Started wormhole-explorer-tx-tracker") diff --git a/tx-tracker/config/structs.go b/tx-tracker/config/structs.go index b00e6615..23c49736 100644 --- a/tx-tracker/config/structs.go +++ b/tx-tracker/config/structs.go @@ -16,14 +16,17 @@ import ( type ServiceSettings struct { // MonitoringPort defines the TCP port for the /health and /ready endpoints. - MonitoringPort string `split_words:"true" default:"8000"` - Environment string `split_words:"true" required:"true"` - LogLevel string `split_words:"true" default:"INFO"` - PprofEnabled bool `split_words:"true" default:"false"` - MetricsEnabled bool `split_words:"true" default:"false"` - P2pNetwork string `split_words:"true" required:"true"` - RpcProviderPath string `split_words:"true" required:"false"` - ConsumerWorkersSize int `split_words:"true" default:"10"` + MonitoringPort string `split_words:"true" default:"8000"` + Environment string `split_words:"true" required:"true"` + LogLevel string `split_words:"true" default:"INFO"` + PprofEnabled bool `split_words:"true" default:"false"` + MetricsEnabled bool `split_words:"true" default:"false"` + P2pNetwork string `split_words:"true" required:"true"` + RpcProviderPath string `split_words:"true" required:"false"` + ConsumerWorkersSize int `split_words:"true" default:"10"` + NotionalCacheURL string `split_words:"true" required:"true"` + NotionalCachePrefix string `split_words:"true" required:"true"` + NotionalCacheChannel string `split_words:"true" required:"true"` AwsSettings MongodbSettings *RpcProviderSettings `required:"false"` @@ -35,6 +38,9 @@ type ServiceSettings struct { type RpcProviderSettingsJson struct { RpcProviders []ChainRpcProviderSettings `json:"rpcProviders"` WormchainRpcProviders []ChainRpcProviderSettings `json:"wormchainRpcProviders"` + NotionalCacheURL string `json:"notional_cache_url"` + NotionalCachePrefix string `json:"notional_cache_prefix"` + NotionalCacheChannel string `json:"notional_cache_channel"` } type ChainRpcProviderSettings struct { diff --git a/tx-tracker/consumer/consumer.go b/tx-tracker/consumer/consumer.go index 1b016564..51f29d42 100644 --- a/tx-tracker/consumer/consumer.go +++ b/tx-tracker/consumer/consumer.go @@ -3,6 +3,7 @@ package consumer import ( "context" "errors" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "time" "github.com/wormhole-foundation/wormhole-explorer/common/pool" @@ -24,19 +25,19 @@ type Consumer struct { metrics metrics.Metrics p2pNetwork string workersSize int + notionalCache *notional.NotionalCache } // New creates a new vaa consumer. -func New( - consumeFunc queue.ConsumeFunc, +func New(consumeFunc queue.ConsumeFunc, rpcPool map[vaa.ChainID]*pool.Pool, wormchainRpcPool map[vaa.ChainID]*pool.Pool, - ctx context.Context, logger *zap.Logger, repository *Repository, metrics metrics.Metrics, p2pNetwork string, workersSize int, + notionalCache *notional.NotionalCache, ) *Consumer { c := Consumer{ @@ -48,6 +49,7 @@ func New( metrics: metrics, p2pNetwork: p2pNetwork, workersSize: workersSize, + notionalCache: notionalCache, } return &c @@ -118,7 +120,7 @@ func (c *Consumer) processSourceTx(ctx context.Context, msg queue.ConsumerMessag Source: event.Source, SentTimestamp: msg.SentTimestamp(), } - _, err := ProcessSourceTx(ctx, c.logger, c.rpcpool, c.wormchainRpcPool, c.repository, &p, c.p2pNetwork) + _, err := ProcessSourceTx(ctx, c.logger, c.rpcpool, c.wormchainRpcPool, c.repository, &p, c.p2pNetwork, c.notionalCache) // add vaa processing duration metrics c.metrics.AddVaaProcessedDuration(uint16(event.ChainID), time.Since(start).Seconds()) @@ -204,8 +206,9 @@ func (c *Consumer) processTargetTx(ctx context.Context, msg queue.ConsumerMessag EvmFee: evmFee, SolanaFee: solanaFee, Metrics: c.metrics, + P2pNetwork: c.p2pNetwork, } - err := ProcessTargetTx(ctx, c.logger, c.repository, &p) + err := ProcessTargetTx(ctx, c.logger, c.repository, &p, c.notionalCache) elapsedLog := zap.Uint64("elapsedTime", uint64(time.Since(start).Milliseconds())) if err != nil { diff --git a/tx-tracker/consumer/repository.go b/tx-tracker/consumer/repository.go index 5e9341f4..9aae9f98 100644 --- a/tx-tracker/consumer/repository.go +++ b/tx-tracker/consumer/repository.go @@ -32,8 +32,10 @@ type DestinationTx struct { } type FeeDetail struct { - Fee string `bson:"fee"` - RawFee map[string]string `bson:"rawFee"` + Fee string `bson:"fee"` + RawFee map[string]string `bson:"rawFee"` + GasTokenNotional string `bson:"gasTokenNotional" json:"gasTokenNotional"` + FeeUSD string `bson:"feeUSD" json:"feeUSD"` } // TargetTxUpdate represents a transaction document. diff --git a/tx-tracker/consumer/source_processor.go b/tx-tracker/consumer/source_processor.go index d983165f..50f80df6 100644 --- a/tx-tracker/consumer/source_processor.go +++ b/tx-tracker/consumer/source_processor.go @@ -3,6 +3,7 @@ package consumer import ( "context" "errors" + notionalCache "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "time" "github.com/wormhole-foundation/wormhole-explorer/common/domain" @@ -38,6 +39,7 @@ type ProcessSourceTxParams struct { Metrics metrics.Metrics SentTimestamp *time.Time DisableDBUpsert bool + P2pNetwork string } func ProcessSourceTx( @@ -48,6 +50,7 @@ func ProcessSourceTx( repository *Repository, params *ProcessSourceTxParams, p2pNetwork string, + notionalCache *notionalCache.NotionalCache, ) (*chains.TxDetail, error) { if !params.Overwrite { @@ -114,7 +117,7 @@ func ProcessSourceTx( } // Get transaction details from the emitter blockchain - txDetail, err = chains.FetchTx(ctx, rpcPool, wormchainRpcPool, params.ChainId, params.TxHash, params.Timestamp, p2pNetwork, params.Metrics, logger) + txDetail, err = chains.FetchTx(ctx, rpcPool, wormchainRpcPool, params.ChainId, params.TxHash, params.Timestamp, p2pNetwork, params.Metrics, logger, notionalCache) if err != nil { errHandleFetchTx := handleFetchTxError(ctx, logger, repository, params, err) if errHandleFetchTx == nil { diff --git a/tx-tracker/consumer/target_processor.go b/tx-tracker/consumer/target_processor.go index 8b873d47..7eb796b1 100644 --- a/tx-tracker/consumer/target_processor.go +++ b/tx-tracker/consumer/target_processor.go @@ -3,6 +3,8 @@ package consumer import ( "context" "errors" + "github.com/shopspring/decimal" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "strconv" "time" @@ -36,6 +38,7 @@ type ProcessTargetTxParams struct { EvmFee *EvmFee SolanaFee *SolanaFee Metrics metrics.Metrics + P2pNetwork string } type EvmFee struct { @@ -52,9 +55,10 @@ func ProcessTargetTx( logger *zap.Logger, repository *Repository, params *ProcessTargetTxParams, + notionalCache *notional.NotionalCache, ) error { - feeDetail := calculateFeeDetail(params, logger) + feeDetail := calculateFeeDetail(params, logger, notionalCache) txHash := domain.NormalizeTxHashByChainId(params.ChainID, params.TxHash) now := time.Now() @@ -119,8 +123,10 @@ func checkTxShouldBeUpdated(ctx context.Context, tx *TargetTxUpdate, repository } } -func calculateFeeDetail(params *ProcessTargetTxParams, logger *zap.Logger) *FeeDetail { +func calculateFeeDetail(params *ProcessTargetTxParams, logger *zap.Logger, notionalCache *notional.NotionalCache) *FeeDetail { + // calculate tx fee for evm redeemed tx. + var feeDetail *FeeDetail if params.EvmFee != nil { fee, err := chains.EvmCalculateFee(params.ChainID, params.EvmFee.GasUsed, params.EvmFee.EffectiveGasPrice) if err != nil { @@ -133,27 +139,40 @@ func calculateFeeDetail(params *ProcessTargetTxParams, logger *zap.Logger) *FeeD ) return nil } - if fee == "" { - return nil - } - return &FeeDetail{ - RawFee: map[string]string{ - "gasUsed": params.EvmFee.GasUsed, - "effectiveGasPrice": params.EvmFee.EffectiveGasPrice, - }, - Fee: fee, + if fee != nil { + feeDetail = &FeeDetail{ + RawFee: map[string]string{ + "gasUsed": params.EvmFee.GasUsed, + "effectiveGasPrice": params.EvmFee.EffectiveGasPrice, + }, + Fee: fee.String(), + } } } // calculate tx fee for solana redeemed tx. if params.SolanaFee != nil { fee := chains.SolanaCalculateFee(params.SolanaFee.Fee) - return &FeeDetail{ + feeDetail = &FeeDetail{ RawFee: map[string]string{ "fee": strconv.FormatUint(params.SolanaFee.Fee, 10), }, - Fee: fee, + Fee: fee.String(), } } - return nil + if feeDetail != nil && params.P2pNetwork == domain.P2pMainNet { + gasTokenPrice, errGasPrice := chains.GetGasTokenNotional(params.ChainID, notionalCache) + if errGasPrice != nil { + logger.Error("Failed to get gas price", + zap.Error(errGasPrice), + zap.String("chainId", params.ChainID.String()), + zap.String("txHash", params.TxHash), + ) + return feeDetail + } + feeDetail.GasTokenNotional = gasTokenPrice.NotionalUsd.String() + feeDetail.FeeUSD = gasTokenPrice.NotionalUsd.Mul(decimal.RequireFromString(feeDetail.Fee)).String() + } + + return feeDetail } diff --git a/tx-tracker/http/vaa/controller.go b/tx-tracker/http/vaa/controller.go index b74c17fa..bd97371d 100644 --- a/tx-tracker/http/vaa/controller.go +++ b/tx-tracker/http/vaa/controller.go @@ -2,10 +2,8 @@ package vaa import ( "encoding/hex" - "strconv" - "strings" - "github.com/gofiber/fiber/v2" + "github.com/wormhole-foundation/wormhole-explorer/common/client/cache/notional" "github.com/wormhole-foundation/wormhole-explorer/common/domain" "github.com/wormhole-foundation/wormhole-explorer/common/pool" "github.com/wormhole-foundation/wormhole-explorer/common/utils" @@ -13,6 +11,8 @@ import ( "github.com/wormhole-foundation/wormhole-explorer/txtracker/internal/metrics" sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" "go.uber.org/zap" + "strconv" + "strings" ) // Controller definition. @@ -24,10 +24,11 @@ type Controller struct { repository *consumer.Repository metrics metrics.Metrics p2pNetwork string + notionalCache *notional.NotionalCache } // NewController creates a Controller instance. -func NewController(rpcPool map[sdk.ChainID]*pool.Pool, wormchainRpcPool map[sdk.ChainID]*pool.Pool, vaaRepository *Repository, repository *consumer.Repository, p2pNetwork string, logger *zap.Logger) *Controller { +func NewController(rpcPool map[sdk.ChainID]*pool.Pool, wormchainRpcPool map[sdk.ChainID]*pool.Pool, vaaRepository *Repository, repository *consumer.Repository, p2pNetwork string, logger *zap.Logger, notionalCache *notional.NotionalCache) *Controller { return &Controller{ metrics: metrics.NewDummyMetrics(), rpcPool: rpcPool, @@ -35,7 +36,9 @@ func NewController(rpcPool map[sdk.ChainID]*pool.Pool, wormchainRpcPool map[sdk. vaaRepository: vaaRepository, repository: repository, p2pNetwork: p2pNetwork, - logger: logger} + logger: logger, + notionalCache: notionalCache, + } } func (c *Controller) Process(ctx *fiber.Ctx) error { @@ -70,9 +73,10 @@ func (c *Controller) Process(ctx *fiber.Ctx) error { IsVaaSigned: true, Metrics: c.metrics, Overwrite: true, + P2pNetwork: c.p2pNetwork, } - result, err := consumer.ProcessSourceTx(ctx.Context(), c.logger, c.rpcPool, c.wormchainRpcPool, c.repository, p, c.p2pNetwork) + result, err := consumer.ProcessSourceTx(ctx.Context(), c.logger, c.rpcPool, c.wormchainRpcPool, c.repository, p, c.p2pNetwork, c.notionalCache) if err != nil { return err } @@ -143,7 +147,7 @@ func (c *Controller) CreateTxHash(ctx *fiber.Ctx) error { DisableDBUpsert: true, } - result, err := consumer.ProcessSourceTx(ctx.Context(), c.logger, c.rpcPool, c.wormchainRpcPool, c.repository, p, c.p2pNetwork) + result, err := consumer.ProcessSourceTx(ctx.Context(), c.logger, c.rpcPool, c.wormchainRpcPool, c.repository, p, c.p2pNetwork, c.notionalCache) if err != nil { return err }