509 lines
15 KiB
Go
509 lines
15 KiB
Go
package p
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"math"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"cloud.google.com/go/bigtable"
|
|
"cloud.google.com/go/pubsub"
|
|
"cloud.google.com/go/storage"
|
|
"github.com/wormhole-foundation/wormhole/sdk/vaa"
|
|
)
|
|
|
|
// shared code for the various functions, primarily response formatting.
|
|
|
|
// client is a global Bigtable client, to avoid initializing a new client for
|
|
// every request.
|
|
var client *bigtable.Client
|
|
var clientOnce sync.Once
|
|
var tbl *bigtable.Table
|
|
|
|
var storageClient *storage.Client
|
|
var cacheBucketName string
|
|
var cacheBucket *storage.BucketHandle
|
|
|
|
var pubsubClient *pubsub.Client
|
|
var pubSubTokenTransferDetailsTopic *pubsub.Topic
|
|
|
|
var coinGeckoCoins = map[string][]CoinGeckoCoin{}
|
|
var solanaTokens = map[string]SolanaToken{}
|
|
|
|
var releaseDay = time.Date(2021, 9, 13, 0, 0, 0, 0, time.UTC)
|
|
|
|
var loadCache = true
|
|
|
|
// init runs during cloud function initialization. So, this will only run during an
|
|
// an instance's cold start.
|
|
// https://cloud.google.com/functions/docs/bestpractices/networking#accessing_google_apis
|
|
func init() {
|
|
defer timeTrack(time.Now(), "init")
|
|
clientOnce.Do(func() {
|
|
// Declare a separate err variable to avoid shadowing client.
|
|
var err error
|
|
project := os.Getenv("GCP_PROJECT")
|
|
instance := os.Getenv("BIGTABLE_INSTANCE")
|
|
client, err = bigtable.NewClient(context.Background(), project, instance)
|
|
if err != nil {
|
|
// http.Error(w, "Error initializing client", http.StatusInternalServerError)
|
|
log.Printf("bigtable.NewClient error: %v", err)
|
|
|
|
return
|
|
}
|
|
|
|
// create the topic that will be published to after decoding token transfer payloads
|
|
tokenTransferDetailsTopic := os.Getenv("PUBSUB_TOKEN_TRANSFER_DETAILS_TOPIC")
|
|
if tokenTransferDetailsTopic != "" {
|
|
var pubsubErr error
|
|
pubsubClient, pubsubErr = pubsub.NewClient(context.Background(), project)
|
|
if pubsubErr != nil {
|
|
log.Printf("pubsub.NewClient error: %v", pubsubErr)
|
|
return
|
|
}
|
|
pubSubTokenTransferDetailsTopic = pubsubClient.Topic(tokenTransferDetailsTopic)
|
|
// fetch the token lists once at start up
|
|
coinGeckoCoins = fetchCoinGeckoCoins()
|
|
solanaTokens = fetchSolanaTokenList()
|
|
}
|
|
})
|
|
tbl = client.Open("v2Events")
|
|
|
|
cacheBucketName = os.Getenv("CACHE_BUCKET")
|
|
if cacheBucketName != "" {
|
|
// Create storage client.
|
|
var err error
|
|
storageClient, err = storage.NewClient(context.Background())
|
|
if err != nil {
|
|
log.Fatalf("Failed to create storage client: %v", err)
|
|
}
|
|
cacheBucket = storageClient.Bucket(cacheBucketName)
|
|
}
|
|
|
|
tokenAllowlistFilePath := os.Getenv("TOKEN_ALLOWLIST")
|
|
if tokenAllowlistFilePath != "" {
|
|
loadJsonToInterface(context.Background(), tokenAllowlistFilePath, &sync.RWMutex{}, &tokenAllowlist)
|
|
}
|
|
|
|
loadCacheStr := os.Getenv("LOAD_CACHE")
|
|
if val, err := strconv.ParseBool(loadCacheStr); err == nil {
|
|
loadCache = val
|
|
log.Printf("loadCache set to %v\n", loadCache)
|
|
}
|
|
}
|
|
|
|
func timeTrack(start time.Time, name string) {
|
|
elapsed := time.Since(start)
|
|
log.Printf("%s took %s", name, elapsed)
|
|
}
|
|
|
|
// reads the specified file from the CACHE_BUCKET and unmarshals the json into the supplied interface.
|
|
func loadJsonToInterface(ctx context.Context, filePath string, mutex *sync.RWMutex, cacheMap interface{}) {
|
|
if cacheBucket == nil {
|
|
log.Println("no cacheBucket supplied, not going to read cache")
|
|
return
|
|
}
|
|
defer timeTrack(time.Now(), fmt.Sprintf("reading %v", filePath))
|
|
mutex.Lock()
|
|
defer mutex.Unlock()
|
|
|
|
reader, readErr := cacheBucket.Object(filePath).NewReader(ctx)
|
|
if readErr != nil {
|
|
log.Printf("Failed reading %v in GCS. err: %v", filePath, readErr)
|
|
return
|
|
}
|
|
defer reader.Close()
|
|
fileData, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
log.Printf("loadJsonToInterface: unable to read data. file %q: %v", filePath, err)
|
|
}
|
|
unmarshalErr := json.Unmarshal(fileData, &cacheMap)
|
|
if unmarshalErr != nil {
|
|
log.Printf("failed unmarshaling %v, err: %v", filePath, unmarshalErr)
|
|
}
|
|
}
|
|
|
|
// writes the supplied interface to the CACHE_BUCKET/filePath.
|
|
func persistInterfaceToJson(ctx context.Context, filePath string, mutex *sync.RWMutex, cacheMap interface{}) {
|
|
if cacheBucket == nil {
|
|
log.Println("no cacheBucket supplied, not going to persist cache")
|
|
return
|
|
}
|
|
defer timeTrack(time.Now(), fmt.Sprintf("writing %v", filePath))
|
|
mutex.Lock()
|
|
|
|
cacheBytes, marshalErr := json.MarshalIndent(cacheMap, "", " ")
|
|
if marshalErr != nil {
|
|
log.Fatal("failed marshaling cacheMap.", marshalErr)
|
|
}
|
|
wc := cacheBucket.Object(filePath).NewWriter(ctx)
|
|
reader := bytes.NewReader(cacheBytes)
|
|
if _, writeErr := io.Copy(wc, reader); writeErr != nil {
|
|
log.Printf("failed writing to file %v, err: %v", filePath, writeErr)
|
|
}
|
|
mutex.Unlock()
|
|
if err := wc.Close(); err != nil {
|
|
log.Printf("Writer.Close with error: %v", err)
|
|
}
|
|
}
|
|
|
|
var columnFamilies = []string{
|
|
"MessagePublication",
|
|
"QuorumState",
|
|
"TokenTransferPayload",
|
|
"AssetMetaPayload",
|
|
"NFTTransferPayload",
|
|
"TokenTransferDetails",
|
|
"ChainDetails",
|
|
}
|
|
var messagePubFam = columnFamilies[0]
|
|
var quorumStateFam = columnFamilies[1]
|
|
var transferPayloadFam = columnFamilies[2]
|
|
var metaPayloadFam = columnFamilies[3]
|
|
var nftPayloadFam = columnFamilies[4]
|
|
var transferDetailsFam = columnFamilies[5]
|
|
var chainDetailsFam = columnFamilies[6]
|
|
|
|
type (
|
|
// Summary is MessagePublication data & QuorumState data
|
|
Summary struct {
|
|
EmitterChain string
|
|
EmitterAddress string
|
|
Sequence string
|
|
InitiatingTxID string
|
|
Payload []byte
|
|
SignedVAABytes []byte
|
|
QuorumTime string
|
|
TransferDetails *TransferDetails
|
|
}
|
|
// Details is a Summary extended with all the post-processing ColumnFamilies
|
|
Details struct {
|
|
Summary
|
|
SignedVAA *vaa.VAA
|
|
TokenTransferPayload *TokenTransferPayload
|
|
AssetMetaPayload *AssetMetaPayload
|
|
NFTTransferPayload *NFTTransferPayload
|
|
ChainDetails *ChainDetails
|
|
}
|
|
// The following structs match the ColumnFamiles they are named after
|
|
TokenTransferPayload struct {
|
|
Amount string
|
|
OriginAddress string
|
|
OriginChain string
|
|
TargetAddress string
|
|
TargetChain string
|
|
}
|
|
AssetMetaPayload struct {
|
|
TokenAddress string
|
|
TokenChain string
|
|
Decimals string
|
|
Symbol string
|
|
Name string
|
|
CoinGeckoCoinId string
|
|
NativeAddress string
|
|
}
|
|
NFTTransferPayload struct {
|
|
OriginAddress string
|
|
OriginChain string
|
|
Symbol string
|
|
Name string
|
|
TokenId string
|
|
URI string
|
|
TargetAddress string
|
|
TargetChain string
|
|
}
|
|
TransferDetails struct {
|
|
Amount string
|
|
Decimals string
|
|
NotionalUSDStr string
|
|
TokenPriceUSDStr string
|
|
TransferTimestamp string
|
|
OriginSymbol string
|
|
OriginName string
|
|
OriginTokenAddress string
|
|
}
|
|
ChainDetails struct {
|
|
SenderAddress string
|
|
ReceiverAddress string
|
|
}
|
|
)
|
|
|
|
// ChainIDs to compute TVL/stats for
|
|
// Useful to exclude chains we don't want to compute TVL for which can improve performance
|
|
// (notably PythNet is excluded, ChainID 26)
|
|
var tvlChainIDs = []vaa.ChainID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 22, 28}
|
|
|
|
func chainIdStringToType(chainId string) vaa.ChainID {
|
|
switch chainId {
|
|
case "1":
|
|
return vaa.ChainIDSolana
|
|
case "2":
|
|
return vaa.ChainIDEthereum
|
|
case "3":
|
|
return vaa.ChainIDTerra
|
|
case "4":
|
|
return vaa.ChainIDBSC
|
|
case "5":
|
|
return vaa.ChainIDPolygon
|
|
case "6":
|
|
return vaa.ChainIDAvalanche
|
|
case "7":
|
|
return vaa.ChainIDOasis
|
|
case "8":
|
|
return vaa.ChainIDAlgorand
|
|
case "9":
|
|
return vaa.ChainIDAurora
|
|
case "10":
|
|
return vaa.ChainIDFantom
|
|
case "11":
|
|
return vaa.ChainIDKarura
|
|
case "12":
|
|
return vaa.ChainIDAcala
|
|
case "13":
|
|
return vaa.ChainIDKlaytn
|
|
case "14":
|
|
return vaa.ChainIDCelo
|
|
case "15":
|
|
return vaa.ChainIDNear
|
|
case "16":
|
|
return vaa.ChainIDMoonbeam
|
|
case "18":
|
|
return vaa.ChainIDTerra2
|
|
case "22":
|
|
return vaa.ChainIDAptos
|
|
case "28":
|
|
return vaa.ChainIDXpla
|
|
}
|
|
return vaa.ChainIDUnset
|
|
}
|
|
|
|
func chainIDToNumberString(c vaa.ChainID) string {
|
|
return strconv.FormatUint(uint64(c), 10)
|
|
}
|
|
|
|
func makeSummary(row bigtable.Row) *Summary {
|
|
summary := &Summary{}
|
|
if _, ok := row[messagePubFam]; ok {
|
|
|
|
for _, item := range row[messagePubFam] {
|
|
switch item.Column {
|
|
case "MessagePublication:InitiatingTxID":
|
|
summary.InitiatingTxID = string(item.Value)
|
|
case "MessagePublication:Payload":
|
|
summary.Payload = item.Value
|
|
case "MessagePublication:EmitterChain":
|
|
summary.EmitterChain = string(item.Value)
|
|
case "MessagePublication:EmitterAddress":
|
|
summary.EmitterAddress = string(item.Value)
|
|
case "MessagePublication:Sequence":
|
|
summary.Sequence = string(item.Value)
|
|
}
|
|
}
|
|
} else {
|
|
// Some rows have a QuorumState, but no MessagePublication,
|
|
// so populate Summary values from the rowKey.
|
|
keyParts := strings.Split(row.Key(), ":")
|
|
chainId := chainIdStringToType(keyParts[0])
|
|
summary.EmitterChain = chainId.String()
|
|
summary.EmitterAddress = keyParts[1]
|
|
seq := strings.TrimLeft(keyParts[2], "0")
|
|
if seq == "" {
|
|
seq = "0"
|
|
}
|
|
summary.Sequence = seq
|
|
}
|
|
if _, ok := row[quorumStateFam]; ok {
|
|
item := row[quorumStateFam][0]
|
|
summary.SignedVAABytes = item.Value
|
|
summary.QuorumTime = item.Timestamp.Time().String()
|
|
}
|
|
if _, ok := row[transferDetailsFam]; ok {
|
|
transferDetails := &TransferDetails{}
|
|
for _, item := range row[transferDetailsFam] {
|
|
switch item.Column {
|
|
case "TokenTransferDetails:Amount":
|
|
transferDetails.Amount = string(item.Value)
|
|
case "TokenTransferDetails:Decimals":
|
|
transferDetails.Decimals = string(item.Value)
|
|
case "TokenTransferDetails:NotionalUSDStr":
|
|
transferDetails.NotionalUSDStr = string(item.Value)
|
|
case "TokenTransferDetails:TokenPriceUSDStr":
|
|
transferDetails.TokenPriceUSDStr = string(item.Value)
|
|
case "TokenTransferDetails:TransferTimestamp":
|
|
transferDetails.TransferTimestamp = string(item.Value)
|
|
case "TokenTransferDetails:OriginSymbol":
|
|
transferDetails.OriginSymbol = string(item.Value)
|
|
case "TokenTransferDetails:OriginName":
|
|
transferDetails.OriginName = string(item.Value)
|
|
case "TokenTransferDetails:OriginTokenAddress":
|
|
transferDetails.OriginTokenAddress = string(item.Value)
|
|
}
|
|
}
|
|
summary.TransferDetails = transferDetails
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func makeDetails(row bigtable.Row) *Details {
|
|
deets := &Details{}
|
|
sum := makeSummary(row)
|
|
deets.Summary = Summary{
|
|
EmitterChain: sum.EmitterChain,
|
|
EmitterAddress: sum.EmitterAddress,
|
|
Sequence: sum.Sequence,
|
|
InitiatingTxID: sum.InitiatingTxID,
|
|
Payload: sum.Payload,
|
|
SignedVAABytes: sum.SignedVAABytes,
|
|
QuorumTime: sum.QuorumTime,
|
|
TransferDetails: sum.TransferDetails,
|
|
}
|
|
|
|
if _, ok := row[quorumStateFam]; ok {
|
|
item := row[quorumStateFam][0]
|
|
deets.SignedVAA, _ = vaa.Unmarshal(item.Value)
|
|
}
|
|
if _, ok := row[transferPayloadFam]; ok {
|
|
tokenTransferPayload := &TokenTransferPayload{}
|
|
for _, item := range row[transferPayloadFam] {
|
|
switch item.Column {
|
|
case "TokenTransferPayload:Amount":
|
|
tokenTransferPayload.Amount = string(item.Value)
|
|
case "TokenTransferPayload:OriginAddress":
|
|
tokenTransferPayload.OriginAddress = string(item.Value)
|
|
case "TokenTransferPayload:OriginChain":
|
|
tokenTransferPayload.OriginChain = string(item.Value)
|
|
case "TokenTransferPayload:TargetAddress":
|
|
tokenTransferPayload.TargetAddress = string(item.Value)
|
|
case "TokenTransferPayload:TargetChain":
|
|
tokenTransferPayload.TargetChain = string(item.Value)
|
|
}
|
|
}
|
|
deets.TokenTransferPayload = tokenTransferPayload
|
|
}
|
|
if _, ok := row[metaPayloadFam]; ok {
|
|
assetMetaPayload := &AssetMetaPayload{}
|
|
for _, item := range row[metaPayloadFam] {
|
|
switch item.Column {
|
|
case "AssetMetaPayload:TokenAddress":
|
|
assetMetaPayload.TokenAddress = string(item.Value)
|
|
case "AssetMetaPayload:TokenChain":
|
|
assetMetaPayload.TokenChain = string(item.Value)
|
|
case "AssetMetaPayload:Decimals":
|
|
assetMetaPayload.Decimals = string(item.Value)
|
|
case "AssetMetaPayload:Symbol":
|
|
assetMetaPayload.Symbol = string(item.Value)
|
|
case "AssetMetaPayload:Name":
|
|
assetMetaPayload.Name = string(item.Value)
|
|
case "AssetMetaPayload:CoinGeckoCoinId":
|
|
assetMetaPayload.CoinGeckoCoinId = string(item.Value)
|
|
case "AssetMetaPayload:NativeAddress":
|
|
assetMetaPayload.NativeAddress = string(item.Value)
|
|
}
|
|
}
|
|
deets.AssetMetaPayload = assetMetaPayload
|
|
}
|
|
if _, ok := row[nftPayloadFam]; ok {
|
|
nftTransferPayload := &NFTTransferPayload{}
|
|
for _, item := range row[nftPayloadFam] {
|
|
switch item.Column {
|
|
case "NFTTransferPayload:OriginAddress":
|
|
nftTransferPayload.OriginAddress = string(item.Value)
|
|
case "NFTTransferPayload:OriginChain":
|
|
nftTransferPayload.OriginChain = string(item.Value)
|
|
case "NFTTransferPayload:Symbol":
|
|
nftTransferPayload.Symbol = string(item.Value)
|
|
case "NFTTransferPayload:Name":
|
|
nftTransferPayload.Name = string(item.Value)
|
|
case "NFTTransferPayload:TokenId":
|
|
nftTransferPayload.TokenId = string(item.Value)
|
|
case "NFTTransferPayload:URI":
|
|
nftTransferPayload.URI = string(TrimUnicodeFromByteArray(item.Value))
|
|
case "NFTTransferPayload:TargetAddress":
|
|
nftTransferPayload.TargetAddress = string(item.Value)
|
|
case "NFTTransferPayload:TargetChain":
|
|
nftTransferPayload.TargetChain = string(item.Value)
|
|
}
|
|
}
|
|
deets.NFTTransferPayload = nftTransferPayload
|
|
}
|
|
if _, ok := row[chainDetailsFam]; ok {
|
|
chainDetails := &ChainDetails{}
|
|
for _, item := range row[chainDetailsFam] {
|
|
switch item.Column {
|
|
// TEMP - until we have this backfilled/populating for new messages
|
|
// case "ChainDetails:SenderAddress":
|
|
// chainDetails.SenderAddress = string(item.Value)
|
|
case "ChainDetails:ReceiverAddress":
|
|
chainDetails.ReceiverAddress = string(item.Value)
|
|
}
|
|
}
|
|
deets.ChainDetails = chainDetails
|
|
}
|
|
return deets
|
|
}
|
|
|
|
func roundToTwoDecimalPlaces(num float64) float64 {
|
|
return math.Round(num*100) / 100
|
|
}
|
|
func createCachePrefix(prefix string) string {
|
|
cachePrefix := prefix
|
|
if prefix == "" {
|
|
cachePrefix = "*"
|
|
}
|
|
return cachePrefix
|
|
}
|
|
|
|
// useCache allows overriding the cache for a given day.
|
|
// This is useful for debugging, to generate fresh data
|
|
func useCache(date string) bool {
|
|
skipDates := map[string]bool{
|
|
// for example, add to skip:
|
|
// "2022-02-01": true,
|
|
}
|
|
if _, ok := skipDates[date]; ok {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// tokens allowed in TVL calculation
|
|
var tokenAllowlist = map[string]map[string]string{}
|
|
|
|
// isTokenAllowed returns whether or not the token is in the allowlist along with its CoinGecko ID
|
|
func isTokenAllowed(chainId string, tokenAddress string) (bool, string) {
|
|
if tokenAddresses, ok := tokenAllowlist[chainId]; ok {
|
|
if coinGeckoCoinId, ok := tokenAddresses[tokenAddress]; ok {
|
|
return true, coinGeckoCoinId
|
|
}
|
|
}
|
|
return false, ""
|
|
}
|
|
|
|
// tokens with no trading activity recorded by exchanges integrated on CoinGecko since the specified date
|
|
var inactiveTokens = map[string]map[string]string{
|
|
chainIDToNumberString(vaa.ChainIDEthereum): {
|
|
"0x707f9118e33a9b8998bea41dd0d46f38bb963fc8": "2022-06-15", // Anchor bETH token
|
|
},
|
|
}
|
|
|
|
func isTokenActive(chainId string, tokenAddress string, date string) bool {
|
|
if deactivatedDates, ok := inactiveTokens[chainId]; ok {
|
|
if deactivatedDate, ok := deactivatedDates[tokenAddress]; ok {
|
|
return date < deactivatedDate
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func chainIDRowPrefix(chainId vaa.ChainID) string {
|
|
return fmt.Sprintf("%d:", chainId)
|
|
}
|