certusone
/
wormhole-explorer
mirror of https://github.com/certusone/wormhole-explorer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add supported tokens for testnet (#810) 

Update supported tokens for mainnet
This commit is contained in:
ftocal 2023-11-28 10:16:40 -03:00 committed by GitHub
parent 7c467f5267
commit 695fd0dcd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1143 additions and 939 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
analytics/cmd/main.go
View File

@ -72,14 +72,14 @@ func addVaaCountCommand(parent *cobra.Command) {
}
func addVaaVolumeFromFileCommand(parent *cobra.Command) {
var input, output, prices, vaaPayloadParserURL string
var input, output, prices, vaaPayloadParserURL, p2pNetwork string
//vaa-volume from csv file
vaaVolumeFileCmd := &cobra.Command{
Use: "file",
Short: "Generate volume metrics from a VAA csv file",
Run: func(_ *cobra.Command, _ []string) {
metrics.RunVaaVolumeFromFile(input, output, prices, vaaPayloadParserURL)
metrics.RunVaaVolumeFromFile(input, output, prices, vaaPayloadParserURL, p2pNetwork)
},
}
@ -96,17 +96,21 @@ func addVaaVolumeFromFileCommand(parent *cobra.Command) {
vaaVolumeFileCmd.Flags().StringVar(&vaaPayloadParserURL, "vaa-payload-parser-url", "", "VAA payload parser URL")
vaaVolumeFileCmd.MarkFlagRequired("vaa-payload-parser-url")
//p2p-network flag
vaaVolumeFileCmd.Flags().StringVar(&p2pNetwork, "p2p-network", "", "P2P network")
vaaVolumeFileCmd.MarkFlagRequired("p2p-network")
parent.AddCommand(vaaVolumeFileCmd)
}
func addVaaVolumeFromMongoCommand(parent *cobra.Command) {
var mongoUri, mongoDb, output, prices, vaaPayloadParserURL string
var mongoUri, mongoDb, output, prices, vaaPayloadParserURL, p2pNetwork string
//vaa-volume from MongoDB
vaaVolumeMongoCmd := &cobra.Command{
Use: "mongo",
Short: "Generate volume metrics from MongoDB",
Run: func(_ *cobra.Command, _ []string) {
metrics.RunVaaVolumeFromMongo(mongoUri, mongoDb, output, prices, vaaPayloadParserURL)
metrics.RunVaaVolumeFromMongo(mongoUri, mongoDb, output, prices, vaaPayloadParserURL, p2pNetwork)
},
}
@ -124,6 +128,10 @@ func addVaaVolumeFromMongoCommand(parent *cobra.Command) {
vaaVolumeMongoCmd.Flags().StringVar(&vaaPayloadParserURL, "vaa-payload-parser-url", "", "VAA payload parser URL")
vaaVolumeMongoCmd.MarkFlagRequired("vaa-payload-parser-url")
//p2p-network flag
vaaVolumeMongoCmd.Flags().StringVar(&p2pNetwork, "p2p-network", "", "P2P network")
vaaVolumeMongoCmd.MarkFlagRequired("p2p-network")
parent.AddCommand(vaaVolumeMongoCmd)
}
@ -141,16 +149,20 @@ func addVaaVolumeCommand(parent *cobra.Command) {
}
func addPricesCommand(root *cobra.Command) {
var output string
var output, p2pNetwork string
vaaCountCmd := &cobra.Command{
Use: "history",
Short: "Generate notional price history for symbol",
Run: func(_ *cobra.Command, _ []string) {
prices.RunPrices(output)
prices.RunPrices(output, p2pNetwork)
},
}
// output flag
vaaCountCmd.Flags().StringVar(&output, "output", "", "path to output file")
vaaCountCmd.MarkFlagRequired("output")
//p2p-network flag
vaaCountCmd.Flags().StringVar(&p2pNetwork, "p2p-network", "", "P2P network")
vaaCountCmd.MarkFlagRequired("p2p-network")
root.AddCommand(vaaCountCmd)
}

7
analytics/cmd/metrics/volume.go
View File

@ -22,15 +22,17 @@ type VaaConverter struct {
PriceCache *prices.CoinPricesCache
Metrics metrics.Metrics
GetTransferredTokenByVaa token.GetTransferredTokenByVaa
TokenProvider *domain.TokenProvider
}
func NewVaaConverter(priceCache *prices.CoinPricesCache, GetTransferredTokenByVaa token.GetTransferredTokenByVaa) *VaaConverter {
func NewVaaConverter(priceCache *prices.CoinPricesCache, GetTransferredTokenByVaa token.GetTransferredTokenByVaa, tokenProvider *domain.TokenProvider) *VaaConverter {
return &VaaConverter{
MissingTokens: make(map[sdk.Address]sdk.ChainID),
MissingTokensCounter: make(map[sdk.Address]int),
PriceCache: priceCache,
Metrics: metrics.NewNoopMetrics(),
GetTransferredTokenByVaa: GetTransferredTokenByVaa,
TokenProvider: tokenProvider,
}
}
@ -47,7 +49,7 @@ func (c *VaaConverter) Convert(ctx context.Context, vaaBytes []byte) (string, er
}
// Look up token metadata
tokenMetadata, ok := domain.GetTokenByAddress(transferredToken.TokenChain, transferredToken.TokenAddress.String())
tokenMetadata, ok := c.TokenProvider.GetTokenByAddress(transferredToken.TokenChain, transferredToken.TokenAddress.String())
if !ok {
// if not found, add to missing tokens
@ -74,6 +76,7 @@ func (c *VaaConverter) Convert(ctx context.Context, vaaBytes []byte) (string, er
},
Metrics: c.Metrics,
TransferredToken: transferredToken,
TokenProvider: c.TokenProvider,
}
var err error

8
analytics/cmd/metrics/volume_csv.go
View File

@ -12,6 +12,7 @@ import (
"github.com/wormhole-foundation/wormhole-explorer/analytics/cmd/token"
"github.com/wormhole-foundation/wormhole-explorer/analytics/prices"
"github.com/wormhole-foundation/wormhole-explorer/common/client/parser"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
"github.com/wormhole-foundation/wormhole-explorer/common/logger"
"go.uber.org/zap"
)
@ -22,7 +23,7 @@ type LineParser struct {
// read a csv file with VAAs and convert into a decoded csv file
// ready to upload to the database
func RunVaaVolumeFromFile(inputFile, outputFile, pricesFile, vaaPayloadParserURL string) {
func RunVaaVolumeFromFile(inputFile, outputFile, pricesFile, vaaPayloadParserURL, p2pNetwork string) {
ctx := context.Background()
// build logger
@ -39,6 +40,9 @@ func RunVaaVolumeFromFile(inputFile, outputFile, pricesFile, vaaPayloadParserURL
// create a token resolver
tokenResolver := token.NewTokenResolver(parserVAAAPIClient, logger)
// create a token provider
tokenProvider := domain.NewTokenProvider(p2pNetwork)
// open input file
f, err := os.Open(inputFile)
if err != nil {
@ -65,7 +69,7 @@ func RunVaaVolumeFromFile(inputFile, outputFile, pricesFile, vaaPayloadParserURL
logger.Info("loading historical prices...")
priceCache := prices.NewCoinPricesCache(pricesFile)
priceCache.InitCache()
converter := NewVaaConverter(priceCache, tokenResolver.GetTransferredTokenByVaa)
converter := NewVaaConverter(priceCache, tokenResolver.GetTransferredTokenByVaa, tokenProvider)
lp := NewLineParser(converter)
logger.Info("loaded historical prices")

8
analytics/cmd/metrics/volume_mongo.go
View File

@ -10,6 +10,7 @@ import (
"github.com/wormhole-foundation/wormhole-explorer/analytics/prices"
"github.com/wormhole-foundation/wormhole-explorer/common/client/parser"
"github.com/wormhole-foundation/wormhole-explorer/common/dbutil"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
"github.com/wormhole-foundation/wormhole-explorer/common/logger"
"github.com/wormhole-foundation/wormhole-explorer/common/repository"
"go.uber.org/zap"
@ -17,7 +18,7 @@ import (
// read a csv file with VAAs and convert into a decoded csv file
// ready to upload to the database
func RunVaaVolumeFromMongo(mongoUri, mongoDb, outputFile, pricesFile, vaaPayloadParserURL string) {
func RunVaaVolumeFromMongo(mongoUri, mongoDb, outputFile, pricesFile, vaaPayloadParserURL, p2pNetwork string) {
rootCtx := context.Background()
@ -44,6 +45,9 @@ func RunVaaVolumeFromMongo(mongoUri, mongoDb, outputFile, pricesFile, vaaPayload
// create a token resolver
tokenResolver := token.NewTokenResolver(parserVAAAPIClient, logger)
// create a token provider
tokenProvider := domain.NewTokenProvider(p2pNetwork)
// create missing tokens file
missingTokensFile := "missing_tokens.csv"
fmissingTokens, err := os.Create(missingTokensFile)
@ -63,7 +67,7 @@ func RunVaaVolumeFromMongo(mongoUri, mongoDb, outputFile, pricesFile, vaaPayload
logger.Info("loading historical prices...")
priceCache := prices.NewCoinPricesCache(pricesFile)
priceCache.InitCache()
converter := NewVaaConverter(priceCache, tokenResolver.GetTransferredTokenByVaa)
converter := NewVaaConverter(priceCache, tokenResolver.GetTransferredTokenByVaa, tokenProvider)
logger.Info("loaded historical prices")
endTime := time.Now()

6
analytics/cmd/prices/run.go
View File

@ -14,7 +14,7 @@ import (
// go througth the symbol list provided by wormhole
// and fetch the history from coingecko
// and save it to a file
func RunPrices(output string) {
func RunPrices(output, p2pNetwork string) {
// build logger
logger := logger.New("wormhole-explorer-analytics")
@ -29,7 +29,9 @@ func RunPrices(output string) {
}
defer pricesOutput.Close()
tokens := domain.GetAllTokens()
// create token provider
tokenProvider := domain.NewTokenProvider(p2pNetwork)
tokens := tokenProvider.GetAllTokens()
logger.Info("found tokens", zap.Int("count", len(tokens)))
for index, token := range tokens {
logger.Info("processing token",

6
analytics/cmd/service/run.go
View File

@ -27,6 +27,7 @@ import (
"github.com/wormhole-foundation/wormhole-explorer/common/client/parser"
sqs_client "github.com/wormhole-foundation/wormhole-explorer/common/client/sqs"
"github.com/wormhole-foundation/wormhole-explorer/common/dbutil"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
health "github.com/wormhole-foundation/wormhole-explorer/common/health"
"github.com/wormhole-foundation/wormhole-explorer/common/logger"
"go.mongodb.org/mongo-driver/mongo"
@ -97,10 +98,13 @@ func Run() {
// create a token resolver
tokenResolver := token.NewTokenResolver(parserVAAAPIClient, logger)
// create a token provider
tokenProvider := domain.NewTokenProvider(config.P2pNetwork)
// create a metrics instance
logger.Info("initializing metrics instance...")
metric, err := metric.New(rootCtx, db.Database, influxCli, config.InfluxOrganization, config.InfluxBucketInfinite,
config.InfluxBucket30Days, config.InfluxBucket24Hours, notionalCache, metrics, tokenResolver.GetTransferredTokenByVaa, logger)
config.InfluxBucket30Days, config.InfluxBucket24Hours, notionalCache, metrics, tokenResolver.GetTransferredTokenByVaa, tokenProvider, logger)
if err != nil {
logger.Fatal("failed to create metrics instance", zap.Error(err))
}

42
analytics/metric/metric.go
View File

@ -38,6 +38,7 @@ type Metric struct {
notionalCache wormscanNotionalCache.NotionalLocalCacheReadable
metrics metrics.Metrics
getTransferredTokenByVaa token.GetTransferredTokenByVaa
tokenProvider *domain.TokenProvider
logger *zap.Logger
}
@ -53,6 +54,7 @@ func New(
notionalCache wormscanNotionalCache.NotionalLocalCacheReadable,
metrics metrics.Metrics,
getTransferredTokenByVaa token.GetTransferredTokenByVaa,
tokenProvider *domain.TokenProvider,
logger *zap.Logger,
) (*Metric, error) {
@ -72,6 +74,7 @@ func New(
notionalCache: notionalCache,
metrics: metrics,
getTransferredTokenByVaa: getTransferredTokenByVaa,
tokenProvider: tokenProvider,
}
return &m, nil
}
@ -105,7 +108,7 @@ func (m *Metric) Push(ctx context.Context, params *Params) error {
if transferredToken != nil {
if isVaaSigned {
err3 = m.volumeMeasurement(ctx, params.Vaa, transferredToken.Clone())
err3 = m.volumeMeasurement(ctx, params, transferredToken.Clone())
}
err4 = upsertTransferPrices(
@ -122,6 +125,7 @@ func (m *Metric) Push(ctx context.Context, params *Params) error {
return priceData.NotionalUsd, nil
},
transferredToken.Clone(),
m.tokenProvider,
)
} else {
@ -138,6 +142,12 @@ func (m *Metric) Push(ctx context.Context, params *Params) error {
return fmt.Errorf("err1=%w, err2=%w, err3=%w err4=%w", err1, err2, err3, err4)
}
if params.Vaa.EmitterChain != sdk.ChainIDPythNet {
m.logger.Info("Transaction processed successfully",
zap.String("trackId", params.TrackID),
zap.String("vaaId", params.Vaa.MessageID()))
}
return nil
}
@ -172,8 +182,7 @@ func (m *Metric) vaaCountMeasurement(ctx context.Context, p *Params) error {
// Write the point to influx
err = m.apiBucket30Days.WritePoint(ctx, point)
if err != nil {
m.logger.Error("failed to write metric",
zap.String("trackId", p.TrackID),
m.logger.Error("Failed to write metric",
zap.String("measurement", point.Name()),
zap.Uint16("chain_id", uint16(p.Vaa.EmitterChain)),
zap.Error(err),
@ -211,7 +220,7 @@ func (m *Metric) vaaCountAllMessagesMeasurement(ctx context.Context, params *Par
// Write the point to influx
err := m.apiBucket24Hours.WritePoint(ctx, point)
if err != nil {
m.logger.Error("failed to write metric",
m.logger.Error("Failed to write metric",
zap.String("measurement", VaaAllMessagesMeasurement),
zap.Uint16("chain_id", uint16(params.Vaa.EmitterChain)),
zap.Error(err),
@ -225,12 +234,12 @@ func (m *Metric) vaaCountAllMessagesMeasurement(ctx context.Context, params *Par
}
// volumeMeasurement creates a new point for the `vaa_volume_v2` measurement.
func (m *Metric) volumeMeasurement(ctx context.Context, vaa *sdk.VAA, token *token.TransferredToken) error {
func (m *Metric) volumeMeasurement(ctx context.Context, params *Params, token *token.TransferredToken) error {
// Generate a data point for the volume metric
p := MakePointForVaaVolumeParams{
Logger: m.logger,
Vaa: vaa,
Vaa: params.Vaa,
TokenPriceFunc: func(tokenID string, timestamp time.Time) (decimal.Decimal, error) {
priceData, err := m.notionalCache.Get(tokenID)
@ -242,6 +251,7 @@ func (m *Metric) volumeMeasurement(ctx context.Context, vaa *sdk.VAA, token *tok
},
Metrics: m.metrics,
TransferredToken: token,
TokenProvider: m.tokenProvider,
}
point, err := MakePointForVaaVolume(&p)
if err != nil {
@ -258,8 +268,9 @@ func (m *Metric) volumeMeasurement(ctx context.Context, vaa *sdk.VAA, token *tok
m.metrics.IncFailedMeasurement(VaaVolumeMeasurement)
return err
}
m.logger.Info("Wrote a data point for the volume metric",
zap.String("vaaId", vaa.MessageID()),
m.logger.Debug("Wrote a data point for the volume metric",
zap.String("vaaId", params.Vaa.MessageID()),
zap.String("trackId", params.TrackID),
zap.String("measurement", point.Name()),
zap.Any("tags", point.TagList()),
zap.Any("fields", point.FieldList()),
@ -307,6 +318,9 @@ type MakePointForVaaVolumeParams struct {
// TransferredToken is the token that was transferred in the VAA.
TransferredToken *token.TransferredToken
// TokenProvider is used to obtain token metadata.
TokenProvider *domain.TokenProvider
}
// MakePointForVaaVolume builds the InfluxDB volume metric for a given VAA
@ -323,7 +337,7 @@ func MakePointForVaaVolume(params *MakePointForVaaVolumeParams) (*write.Point, e
// Do not generate this metric when the emitter chain is unset
if params.Vaa.EmitterChain.String() == sdk.ChainIDUnset.String() {
if params.Logger != nil {
params.Logger.Warn("emitter chain is unset",
params.Logger.Warn("Emitter chain is unset",
zap.String("vaaId", params.Vaa.MessageID()),
zap.Uint16("emitterChain", uint16(params.Vaa.EmitterChain)),
)
@ -359,7 +373,7 @@ func MakePointForVaaVolume(params *MakePointForVaaVolumeParams) (*write.Point, e
// Get the token metadata
//
// This is complementary data about the token that is not present in the VAA itself.
tokenMeta, ok := domain.GetTokenByAddress(params.TransferredToken.TokenChain, params.TransferredToken.TokenAddress.String())
tokenMeta, ok := params.TokenProvider.GetTokenByAddress(params.TransferredToken.TokenChain, params.TransferredToken.TokenAddress.String())
if !ok {
params.Metrics.IncMissingToken(params.TransferredToken.TokenChain.String(), params.TransferredToken.TokenAddress.String())
// We don't have metadata for this token, so we can't compute the volume-related fields
@ -370,6 +384,12 @@ func MakePointForVaaVolume(params *MakePointForVaaVolumeParams) (*write.Point, e
//
// Moreover, many flux queries depend on the existence of the `volume` field,
// and would break if we had measurements without it.
params.Logger.Warn("Cannot obtain this token",
zap.String("vaaId", params.Vaa.MessageID()),
zap.String("tokenAddress", params.TransferredToken.TokenAddress.String()),
zap.Uint16("tokenChain", uint16(params.TransferredToken.TokenChain)),
zap.Any("tokenMetadata", tokenMeta),
)
point.AddField("volume", uint64(0))
return point, nil
}
@ -391,7 +411,7 @@ func MakePointForVaaVolume(params *MakePointForVaaVolumeParams) (*write.Point, e
if err != nil {
params.Metrics.IncMissingNotional(tokenMeta.Symbol.String())
if params.Logger != nil {
params.Logger.Warn("failed to obtain notional for this token",
params.Logger.Warn("Failed to obtain notional for this token",
zap.String("vaaId", params.Vaa.MessageID()),
zap.String("tokenAddress", params.TransferredToken.TokenAddress.String()),
zap.Uint16("tokenChain", uint16(params.TransferredToken.TokenChain)),

3
analytics/metric/mongo.go
View File

@ -46,6 +46,7 @@ func upsertTransferPrices(
transferPrices *mongo.Collection,
tokenPriceFunc func(tokenID string, timestamp time.Time) (decimal.Decimal, error),
transferredToken *token.TransferredToken,
tokenProvider *domain.TokenProvider,
) error {
// Do not generate this metric for PythNet VAAs
@ -61,7 +62,7 @@ func upsertTransferPrices(
// Get the token metadata
//
// This is complementary data about the token that is not present in the VAA itself.
tokenMeta, ok := domain.GetTokenByAddress(transferredToken.TokenChain, transferredToken.TokenAddress.String())
tokenMeta, ok := tokenProvider.GetTokenByAddress(transferredToken.TokenChain, transferredToken.TokenAddress.String())
if !ok {
return nil
}

11
api/handlers/transactions/service.go
View File

@ -22,6 +22,7 @@ type Service struct {
cache cache.Cache
expiration time.Duration
supportedChainIDs map[vaa.ChainID]string
tokenProvider *domain.TokenProvider
logger *zap.Logger
}
@ -34,10 +35,10 @@ const (
)
// NewService create a new Service.
func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, logger *zap.Logger) *Service {
func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, logger *zap.Logger) *Service {
supportedChainIDs := domain.GetSupportedChainIDs()
return &Service{repo: repo, supportedChainIDs: supportedChainIDs,
cache: cache, expiration: expiration, logger: logger.With(zap.String("module", "TransactionService"))}
cache: cache, expiration: expiration, tokenProvider: tokenProvider, logger: logger.With(zap.String("module", "TransactionService"))}
}
// GetTransactionCount get the last transactions.
@ -104,7 +105,7 @@ func (s *Service) GetTokenByChainAndAddress(ctx context.Context, chainID vaa.Cha
}
//get token by contractID (chainID + tokenAddress)
tokenMetadata, ok := domain.GetTokenByAddress(chainID, tokenAddress.Hex())
tokenMetadata, ok := s.tokenProvider.GetTokenByAddress(chainID, tokenAddress.Hex())
if !ok {
return nil, errs.ErrNotFound
}
@ -159,3 +160,7 @@ func (s *Service) GetTransactionByID(
// Return matching document
return &output[0], nil
}
func (s *Service) GetTokenProvider() *domain.TokenProvider {
return s.tokenProvider
}

6
api/main.go
View File

@ -43,6 +43,7 @@ import (
wormscanCache "github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
vaaPayloadParser "github.com/wormhole-foundation/wormhole-explorer/common/client/parser"
"github.com/wormhole-foundation/wormhole-explorer/common/dbutil"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
xlogger "github.com/wormhole-foundation/wormhole-explorer/common/logger"
"github.com/wormhole-foundation/wormhole-explorer/common/utils"
sdk "github.com/wormhole-foundation/wormhole/sdk/vaa"
@ -154,6 +155,9 @@ func main() {
relaysRepo := relays.NewRepository(db.Database, rootLogger)
operationsRepo := operations.NewRepository(db.Database, rootLogger)
// create token provider
tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork)
// Set up services
rootLogger.Info("initializing services")
addressService := address.NewService(addressRepo, rootLogger)
@ -162,7 +166,7 @@ func main() {
governorService := governor.NewService(governorRepo, rootLogger)
infrastructureService := infrastructure.NewService(infrastructureRepo, rootLogger)
heartbeatsService := heartbeats.NewService(heartbeatsRepo, rootLogger)
transactionsService := transactions.NewService(transactionsRepo, cache, time.Duration(cfg.Cache.MetricExpiration)*time.Second, rootLogger)
transactionsService := transactions.NewService(transactionsRepo, cache, time.Duration(cfg.Cache.MetricExpiration)*time.Second, tokenProvider, rootLogger)
relaysService := relays.NewService(relaysRepo, rootLogger)
operationsService := operations.NewService(operationsRepo, rootLogger)

2
api/routes/wormscan/transactions/controller.go
View File

@ -169,7 +169,7 @@ func (c *Controller) GetTopAssets(ctx *fiber.Ctx) error {
}
// Look up the token symbol
tokenMeta, ok := domain.GetTokenByAddress(assetDTOs[i].TokenChain, assetDTOs[i].TokenAddress)
tokenMeta, ok := c.srv.GetTokenProvider().GetTokenByAddress(assetDTOs[i].TokenChain, assetDTOs[i].TokenAddress)
if ok {
asset.Symbol = tokenMeta.Symbol.String()
}

1765
common/domain/generated_mainnet_tokens.go
View File

File diff suppressed because it is too large Load Diff

16
common/domain/testnet_tokens.go Normal file
View File

@ -0,0 +1,16 @@
package domain
func manualTestnetTokenList() []TokenMetadata {
return []TokenMetadata{
{TokenChain: 1, TokenAddress: "069b8857feab8184fb687f634618c035dac439dc1aeb3b5598a0f00000000001", Symbol: "SOL", CoingeckoID: "wrapped-solana", Decimals: 9},
{TokenChain: 2, TokenAddress: "000000000000000000000000b4fbf271143f4fbf7b91a5ded31805e42b2208d6", Symbol: "WETH", CoingeckoID: "weth", Decimals: 18},
{TokenChain: 2, TokenAddress: "00000000000000000000000011fe4b6ae13d2a6055c8d9cf65c55bac32b5d844", Symbol: "DAI", CoingeckoID: "dai", Decimals: 18},
{TokenChain: 4, TokenAddress: "000000000000000000000000ae13d989dac2f0debff460ac112a837c89baa7cd", Symbol: "WBNB", CoingeckoID: "wbnb", Decimals: 18},
{TokenChain: 5, TokenAddress: "0000000000000000000000009c3c9283d3e44854697cd22d3faa240cfb032889", Symbol: "WMATIC", CoingeckoID: "wmatic", Decimals: 18},
{TokenChain: 6, TokenAddress: "0000000000000000000000005425890298aed601595a70ab815c96711a31bc65", Symbol: "USDC", CoingeckoID: "usd-coin", Decimals: 6},
{TokenChain: 6, TokenAddress: "000000000000000000000000d00ae08403b9bbb9124bb305c09058e32c39a48c", Symbol: "WAVAX", CoingeckoID: "wrapped-avax", Decimals: 18},
{TokenChain: 10, TokenAddress: "000000000000000000000000f1277d1ed8ad466beddf92ef448a132661956621", Symbol: "WFTM", CoingeckoID: "wrapped-fantom", Decimals: 18},
{TokenChain: 14, TokenAddress: "000000000000000000000000f194afdf50b03e69bd7d057c1aa9e10c9954e4c9", Symbol: "CELO", CoingeckoID: "celo", Decimals: 18},
{TokenChain: 16, TokenAddress: "000000000000000000000000d909178cc99d318e4d46e7e66a972955859670e1", Symbol: "GLMR", CoingeckoID: "wrapped-moonbeam", Decimals: 18},
{TokenChain: 21, TokenAddress: "587c29de216efd4219573e08a1f6964d4fa7cb714518c2c8a0f29abfa264327d", Symbol: "SUI", CoingeckoID: "sui", Decimals: 9}}
}

65
common/domain/tokens.go
View File

@ -24,20 +24,37 @@ type TokenMetadata struct {
Decimals int64
}
var (
tokenMetadata = generatedMainnetTokenList()
tokenMetadataByContractID = make(map[string]*TokenMetadata)
tokenMetadataByCoingeckoID = make(map[string]*TokenMetadata)
)
type TokenProvider struct {
p2pNetwork string
tokenMetadata []TokenMetadata
tokenMetadataByContractID map[string]*TokenMetadata
tokenMetadataByCoingeckoID map[string]*TokenMetadata
}
func (t *TokenMetadata) GetTokenID() string {
return fmt.Sprintf("%d/%s", t.TokenChain, t.TokenAddress)
}
func init() {
func makeContractID(tokenChain sdk.ChainID, tokenAddress string) string {
return fmt.Sprintf("%d-%s", tokenChain, tokenAddress)
}
func NewTokenProvider(p2pNetwork string) *TokenProvider {
var tokenMetadata []TokenMetadata
switch p2pNetwork {
case P2pMainNet:
tokenMetadata = generatedMainnetTokenList()
case P2pTestNet:
tokenMetadata = manualTestnetTokenList()
default:
panic(fmt.Sprintf("unknown p2p network: %s", p2pNetwork))
}
tokenMetadataByContractID := make(map[string]*TokenMetadata)
tokenMetadataByCoingeckoID := make(map[string]*TokenMetadata)
for i := range tokenMetadata {
// populate the map `tokenMetadataByCoingeckoID`
coingeckoID := tokenMetadata[i].CoingeckoID
if coingeckoID != "" {
@ -50,26 +67,28 @@ func init() {
tokenMetadataByContractID[contractID] = &tokenMetadata[i]
}
}
}
func makeContractID(tokenChain sdk.ChainID, tokenAddress string) string {
return fmt.Sprintf("%d-%s", tokenChain, tokenAddress)
return &TokenProvider{
p2pNetwork: p2pNetwork,
tokenMetadata: tokenMetadata,
tokenMetadataByContractID: tokenMetadataByContractID,
tokenMetadataByCoingeckoID: tokenMetadataByCoingeckoID,
}
}
// GetAllTokens returns a list of all tokens that exist in the database.
//
// The caller must not modify the `[]TokenMetadata` returned.
func GetAllTokens() []TokenMetadata {
return tokenMetadata
func (t *TokenProvider) GetAllTokens() []TokenMetadata {
return t.tokenMetadata
}
// GetAllCoingeckoIDs returns a list of all coingecko IDs that exist in the database.
func GetAllCoingeckoIDs() []string {
func (t *TokenProvider) GetAllCoingeckoIDs() []string {
// use a map to remove duplicates
uniqueIDs := make(map[string]bool, len(tokenMetadata))
for i := range tokenMetadata {
uniqueIDs[tokenMetadata[i].CoingeckoID] = true
uniqueIDs := make(map[string]bool, len(t.tokenMetadata))
for i := range t.tokenMetadata {
uniqueIDs[t.tokenMetadata[i].CoingeckoID] = true
}
// collect keys into a slice
@ -84,9 +103,9 @@ func GetAllCoingeckoIDs() []string {
// GetTokenByCoingeckoID returns information about a token identified by its coingecko ID.
//
// The caller must not modify the `*TokenMetadata` returned.
func GetTokenByCoingeckoID(coingeckoID string) (*TokenMetadata, bool) {
func (t *TokenProvider) GetTokenByCoingeckoID(coingeckoID string) (*TokenMetadata, bool) {
result, ok := tokenMetadataByCoingeckoID[coingeckoID]
result, ok := t.tokenMetadataByCoingeckoID[coingeckoID]
if !ok {
return nil, false
}
@ -97,14 +116,18 @@ func GetTokenByCoingeckoID(coingeckoID string) (*TokenMetadata, bool) {
// GetTokenByAddress returns information about a token identified by its original mint address.
//
// The caller must not modify the `*TokenMetadata` returned.
func GetTokenByAddress(tokenChain sdk.ChainID, tokenAddress string) (*TokenMetadata, bool) {
func (t *TokenProvider) GetTokenByAddress(tokenChain sdk.ChainID, tokenAddress string) (*TokenMetadata, bool) {
key := makeContractID(tokenChain, tokenAddress)
result, ok := tokenMetadataByContractID[key]
result, ok := t.tokenMetadataByContractID[key]
if !ok {
return nil, false
}
return result, true
}
func (t *TokenProvider) GetP2pNewtork() string {
return t.p2pNetwork
}

2
deploy/jobs/env/production-testnet.env vendored
View File

@ -6,7 +6,7 @@ RESOURCES_LIMITS_MEMORY=30Mi
RESOURCES_LIMITS_CPU=20m
RESOURCES_REQUESTS_MEMORY=15Mi
RESOURCES_REQUESTS_CPU=10m
P2P_NETWORK=mainnet
P2P_NETWORK=testnet
COINGECKO_URL=https://api.coingecko.com/api/v3
NOTIONAL_CHANNEL=WORMSCAN:NOTIONAL
LOG_LEVEL=INFO

2
deploy/jobs/env/staging-testnet.env vendored
View File

@ -6,7 +6,7 @@ RESOURCES_LIMITS_MEMORY=30Mi
RESOURCES_LIMITS_CPU=20m
RESOURCES_REQUESTS_MEMORY=15Mi
RESOURCES_REQUESTS_CPU=10m
P2P_NETWORK=mainnet
P2P_NETWORK=testnet
COINGECKO_URL=https://api.coingecko.com/api/v3
NOTIONAL_CHANNEL=WORMSCAN:NOTIONAL
LOG_LEVEL=INFO

9
jobs/cmd/main.go
View File

@ -7,6 +7,7 @@ import (
"github.com/go-redis/redis"
"github.com/wormhole-foundation/wormhole-explorer/common/dbutil"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
"github.com/wormhole-foundation/wormhole-explorer/common/logger"
"github.com/wormhole-foundation/wormhole-explorer/common/prices"
"github.com/wormhole-foundation/wormhole-explorer/jobs/config"
@ -67,8 +68,10 @@ func initNotionalJob(ctx context.Context, cfg *config.NotionalConfiguration, log
api := coingecko.NewCoingeckoAPI(cfg.CoingeckoURL)
// init redis client.
redisClient := redis.NewClient(&redis.Options{Addr: cfg.CacheURL})
// init token provider.
tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork)
// create notional job.
notionalJob := notional.NewNotionalJob(api, redisClient, cfg.CachePrefix, cfg.P2pNetwork, cfg.NotionalChannel, logger)
notionalJob := notional.NewNotionalJob(api, redisClient, cfg.CachePrefix, cfg.NotionalChannel, tokenProvider, logger)
return notionalJob
}
@ -81,7 +84,9 @@ func initTransferReportJob(ctx context.Context, cfg *config.TransferReportConfig
}
pricesCache := prices.NewCoinPricesCache(cfg.PricesPath)
pricesCache.InitCache()
return report.NewTransferReportJob(db.Database, cfg.PageSize, pricesCache, cfg.OutputPath, logger)
// init token provider.
tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork)
return report.NewTransferReportJob(db.Database, cfg.PageSize, pricesCache, cfg.OutputPath, tokenProvider, logger)
}
func handleExit() {

1
jobs/config/config.go
View File

@ -30,6 +30,7 @@ type TransferReportConfiguration struct {
PageSize int64 `env:"PAGE_SIZE,default=100"`
PricesPath string `env:"PRICES_PATH,required"`
OutputPath string `env:"OUTPUT_PATH,required"`
P2pNetwork string `env:"P2P_NETWORK,required"`
}
// New creates a default configuration with the values from .env file and environment variables.

12
jobs/internal/coingecko/coingecko.go
View File

@ -8,7 +8,6 @@ import (
"strings"
"github.com/shopspring/decimal"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
)
// CoingeckoAPI is a client for the coingecko API
@ -83,14 +82,3 @@ func chunkChainIds(slice []string, chunkSize int) [][]string {
}
return chunks
}
// GetChainIDs returns the coingecko chain ids for the given p2p network.
func GetChainIDs(p2pNetwork string) []string {
if p2pNetwork == domain.P2pMainNet {
return domain.GetAllCoingeckoIDs()
}
// TODO: define chains ids for testnet.
return []string{}
}

32
jobs/jobs/notional/notional.go
View File

@ -14,23 +14,23 @@ import (
// NotionalJob is the job to get the notional value of assets.
type NotionalJob struct {
coingeckoAPI *coingecko.CoingeckoAPI
cacheClient *redis.Client
cachePrefix string
cacheChannel string
p2pNetwork string
logger *zap.Logger
coingeckoAPI *coingecko.CoingeckoAPI
cacheClient *redis.Client
cachePrefix string
cacheChannel string
tokenProvider *domain.TokenProvider
logger *zap.Logger
}
// NewNotionalJob creates a new notional job.
func NewNotionalJob(api *coingecko.CoingeckoAPI, cacheClient *redis.Client, cachePrefix string, p2pNetwork, cacheChannel string, logger *zap.Logger) *NotionalJob {
func NewNotionalJob(api *coingecko.CoingeckoAPI, cacheClient *redis.Client, cachePrefix string, cacheChannel string, tokenProvider *domain.TokenProvider, logger *zap.Logger) *NotionalJob {
return &NotionalJob{
coingeckoAPI: api,
cacheClient: cacheClient,
cachePrefix: cachePrefix,
cacheChannel: formatChannel(cachePrefix, cacheChannel),
p2pNetwork: p2pNetwork,
logger: logger,
coingeckoAPI: api,
cacheClient: cacheClient,
cachePrefix: cachePrefix,
cacheChannel: formatChannel(cachePrefix, cacheChannel),
tokenProvider: tokenProvider,
logger: logger,
}
}
@ -38,9 +38,9 @@ func NewNotionalJob(api *coingecko.CoingeckoAPI, cacheClient *redis.Client, cach
func (j *NotionalJob) Run() error {
// get chains coingecko ids by p2p network.
chainIDs := coingecko.GetChainIDs(j.p2pNetwork)
chainIDs := j.tokenProvider.GetAllCoingeckoIDs()
if len(chainIDs) == 0 {
return fmt.Errorf("no chain ids found for p2p network %s", j.p2pNetwork)
return fmt.Errorf("no chain ids found for p2p network %s", j.tokenProvider.GetP2pNewtork())
}
// get notional value of assets.
@ -98,7 +98,7 @@ func (j *NotionalJob) convertToSymbols(m map[string]coingecko.NotionalUSD) map[s
w := make(map[string]notional.PriceData, len(m))
now := time.Now()
for _, v := range domain.GetAllTokens() {
for _, v := range j.tokenProvider.GetAllTokens() {
notionalUSD, ok := m[v.CoingeckoID]
if !ok {
j.logger.Info("skipping unknown coingecko ID", zap.String("coingeckoID", v.CoingeckoID))

17
jobs/jobs/report/transfer_report.go
View File

@ -19,11 +19,12 @@ import (
)
type TransferReportJob struct {
database *mongo.Database
pageSize int64
logger *zap.Logger
pricesCache *prices.CoinPricesCache
outputPath string
database *mongo.Database
pageSize int64
logger *zap.Logger
pricesCache *prices.CoinPricesCache
outputPath string
tokenProvider *domain.TokenProvider
}
type transactionResult struct {
@ -44,8 +45,8 @@ type transactionResult struct {
}
// NewTransferReportJob creates a new transfer report job.
func NewTransferReportJob(database *mongo.Database, pageSize int64, pricesCache *prices.CoinPricesCache, outputPath string, logger *zap.Logger) *TransferReportJob {
return &TransferReportJob{database: database, pageSize: pageSize, pricesCache: pricesCache, outputPath: outputPath, logger: logger}
func NewTransferReportJob(database *mongo.Database, pageSize int64, pricesCache *prices.CoinPricesCache, outputPath string, tokenProvider *domain.TokenProvider, logger *zap.Logger) *TransferReportJob {
return &TransferReportJob{database: database, pageSize: pageSize, pricesCache: pricesCache, outputPath: outputPath, tokenProvider: tokenProvider, logger: logger}
}
// Run runs the transfer report job.
@ -94,7 +95,7 @@ func (j *TransferReportJob) Run(ctx context.Context) error {
continue
}
m, ok := domain.GetTokenByAddress(sdk.ChainID(t.TokenChain), tokenAddress.String())
m, ok := j.tokenProvider.GetTokenByAddress(sdk.ChainID(t.TokenChain), tokenAddress.String())
if ok {
tokenPrice, err := j.pricesCache.GetPriceByTime(m.CoingeckoID, t.Timestamp)
if err != nil {

6
parser/cmd/backfiller/run.go
View File

@ -8,6 +8,7 @@ import (
"github.com/wormhole-foundation/wormhole-explorer/common/client/alert"
vaaPayloadParser "github.com/wormhole-foundation/wormhole-explorer/common/client/parser"
"github.com/wormhole-foundation/wormhole-explorer/common/dbutil"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
"github.com/wormhole-foundation/wormhole-explorer/common/logger"
"github.com/wormhole-foundation/wormhole-explorer/parser/config"
"github.com/wormhole-foundation/wormhole-explorer/parser/http/vaa"
@ -58,8 +59,11 @@ func Run(config *config.BackfillerConfiguration) {
parserRepository := parser.NewRepository(db.Database, logger)
vaaRepository := vaa.NewRepository(db.Database, logger)
// create a token provider
tokenProvider := domain.NewTokenProvider(config.P2pNetwork)
//create a processor
eventProcessor := processor.New(parserVAAAPIClient, parserRepository, alert.NewDummyClient(), metrics.NewDummyMetrics(), logger)
eventProcessor := processor.New(parserVAAAPIClient, parserRepository, alert.NewDummyClient(), metrics.NewDummyMetrics(), tokenProvider, logger)
logger.Info("Started wormhole-explorer-parser as backfiller")

5
parser/cmd/service/run.go
View File

@ -14,6 +14,7 @@ import (
"github.com/wormhole-foundation/wormhole-explorer/common/client/alert"
vaaPayloadParser "github.com/wormhole-foundation/wormhole-explorer/common/client/parser"
"github.com/wormhole-foundation/wormhole-explorer/common/dbutil"
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
"github.com/wormhole-foundation/wormhole-explorer/common/health"
"github.com/wormhole-foundation/wormhole-explorer/common/logger"
"github.com/wormhole-foundation/wormhole-explorer/parser/config"
@ -99,9 +100,11 @@ func Run() {
if err != nil {
logger.Fatal("failed to create health checks", zap.Error(err))
}
// create a token provider
tokenProvider := domain.NewTokenProvider(config.P2pNetwork)
//create a processor
processor := processor.New(parserVAAAPIClient, repository, alertClient, metrics, logger)
processor := processor.New(parserVAAAPIClient, repository, alertClient, metrics, tokenProvider, logger)
// create and start a vaaConsumer
vaaConsumer := consumer.New(vaaConsumeFunc, processor.Process, metrics, logger)

1
parser/config/config.go
View File

@ -48,6 +48,7 @@ type BackfillerConfiguration struct {
EndTime string `env:"END_TIME"`
PageSize int64 `env:"PAGE_SIZE,default=100"`
SortAsc bool `env:"SORT_ASC,default=false"`
P2pNetwork string `env:"P2P_NETWORK,required"`
}
// New creates a configuration with the values from .env file and environment variables.

26
parser/processor/processor.go
View File

@ -19,20 +19,22 @@ import (
)
type Processor struct {
parser vaaPayloadParser.ParserVAAAPIClient
repository *parser.Repository
alert alert.AlertClient
metrics metrics.Metrics
logger *zap.Logger
parser vaaPayloadParser.ParserVAAAPIClient
repository *parser.Repository
alert alert.AlertClient
metrics metrics.Metrics
tokenProvider *domain.TokenProvider
logger *zap.Logger
}
func New(parser vaaPayloadParser.ParserVAAAPIClient, repository *parser.Repository, alert alert.AlertClient, metrics metrics.Metrics, logger *zap.Logger) *Processor {
func New(parser vaaPayloadParser.ParserVAAAPIClient, repository *parser.Repository, alert alert.AlertClient, metrics metrics.Metrics, tokenProvider *domain.TokenProvider, logger *zap.Logger) *Processor {
return &Processor{
parser: parser,
repository: repository,
alert: alert,
metrics: metrics,
logger: logger,
parser: parser,
repository: repository,
alert: alert,
metrics: metrics,
tokenProvider: tokenProvider,
logger: logger,
}
}
@ -177,7 +179,7 @@ func (p *Processor) transformAmount(chainID sdk.ChainID, trackID, nativeAddress,
// Get the token metadata
//
// This is complementary data about the token that is not present in the VAA itself.
tokenMeta, ok := domain.GetTokenByAddress(sdk.ChainID(chainID), addr.String())
tokenMeta, ok := p.tokenProvider.GetTokenByAddress(sdk.ChainID(chainID), addr.String())
if !ok {
p.logger.Warn("Token metadata not found",
zap.String("trackId", trackID),