BigTable: improve token identification

Change-Id: I693e1ee8b55d797d739db9f9bcaad83c0422a477

commit-id:e0fb97b9

event_database/cloud_functions/external-data.go

@@ -5,9 +5,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"strings"
 	"time"
 
 	"net/http"
+
+	"github.com/certusone/wormhole/node/pkg/vaa"
 )
 
 const cgBaseUrl = "https://api.coingecko.com/api/v3/"
@@ -24,6 +27,9 @@ type CoinGeckoMarket [2]float64
 type CoinGeckoMarketRes struct {
 	Prices []CoinGeckoMarket `json:"prices"`
 }
+type CoinGeckoErrorRes struct {
+	Error string `json:"error"`
+}
 
 func fetchCoinGeckoCoins() map[string][]CoinGeckoCoin {
 	url := fmt.Sprintf("%vcoins/list", cgBaseUrl)
@@ -51,13 +57,114 @@ func fetchCoinGeckoCoins() map[string][]CoinGeckoCoin {
 	}
 	var geckoCoins = map[string][]CoinGeckoCoin{}
 	for _, coin := range parsed {
-		geckoCoins[coin.Symbol] = append(geckoCoins[coin.Symbol], coin)
+		symbol := strings.ToLower(coin.Symbol)
+		geckoCoins[symbol] = append(geckoCoins[symbol], coin)
 	}
 	return geckoCoins
 
 }
 
+func chainIdToCoinGeckoPlatform(chain vaa.ChainID) string {
+	switch chain {
+	case vaa.ChainIDSolana:
+		return "solana"
+	case vaa.ChainIDEthereum:
+		return "ethereum"
+	case vaa.ChainIDTerra:
+		return "terra"
+	case vaa.ChainIDBSC:
+		return "binance-smart-chain"
+	case vaa.ChainIDPolygon:
+		return "polygon-pos"
+	}
+	return ""
+}
+
+func fetchCoinGeckoCoinFromContract(chainId vaa.ChainID, address string) CoinGeckoCoin {
+	platform := chainIdToCoinGeckoPlatform(chainId)
+	url := fmt.Sprintf("%vcoins/%v/contract/%v", cgBaseUrl, platform, address)
+	req, reqErr := http.NewRequest("GET", url, nil)
+	if reqErr != nil {
+		log.Fatalf("failed contract request, err: %v\n", reqErr)
+	}
+
+	res, resErr := http.DefaultClient.Do(req)
+	if resErr != nil {
+		log.Fatalf("failed get contract response, err: %v\n", resErr)
+	}
+
+	defer res.Body.Close()
+	body, bodyErr := ioutil.ReadAll(res.Body)
+	if bodyErr != nil {
+		log.Fatalf("failed decoding contract body, err: %v\n", bodyErr)
+	}
+
+	var parsed CoinGeckoCoin
+
+	parseErr := json.Unmarshal(body, &parsed)
+	if parseErr != nil {
+		log.Printf("failed parsing body. err %v\n", parseErr)
+		var errRes CoinGeckoErrorRes
+		if err := json.Unmarshal(body, &errRes); err == nil {
+			if errRes.Error == "Could not find coin with the given id" {
+				log.Printf("Could not find CoinGecko coin by contract address, for chain %v, address, %v\n", chainId, address)
+			} else {
+				log.Println("Failed calling CoinGecko, got err", errRes.Error)
+			}
+		}
+	}
+
+	return parsed
+}
+
+func fetchCoinGeckoCoinId(chainId vaa.ChainID, address, symbol, name string) (coinId, foundSymbol, foundName string) {
+	// try coingecko, return if good
+	// if coingecko does not work, try chain-specific options
+
+	// initialize strings that will be returned if we find a symbol/name
+	// when looking up this token by contract address
+	newSymbol := ""
+	newName := ""
+
+	if symbol == "" && chainId == vaa.ChainIDSolana {
+		// try to lookup the symbol in solana token list, from the address
+		if token, ok := solanaTokens[address]; ok {
+			symbol = token.Symbol
+			name = token.Name
+			newSymbol = token.Symbol
+			newName = token.Name
+		}
+	}
+	if _, ok := coinGeckoCoins[strings.ToLower(symbol)]; ok {
+		tokens := coinGeckoCoins[strings.ToLower(symbol)]
+		if len(tokens) == 1 {
+			// only one match found for this symbol
+			return tokens[0].Id, newSymbol, newName
+		}
+		for _, token := range tokens {
+			if token.Name == name {
+				// found token by name match
+				return token.Id, newSymbol, newName
+			}
+			if strings.Contains(strings.ToLower(strings.ReplaceAll(name, " ", "")), strings.ReplaceAll(token.Id, "-", "")) {
+				// found token by id match
+				log.Println("found token by symbol and name match", name)
+				return token.Id, newSymbol, newName
+			}
+		}
+		// more than one symbol with this name, let contract lookup try
+	}
+	coin := fetchCoinGeckoCoinFromContract(chainId, address)
+	if coin.Id != "" {
+		return coin.Id, newSymbol, newName
+	}
+	// could not find a CoinGecko coin
+	return "", newSymbol, newName
+}
+
 func fetchCoinGeckoPrice(coinId string, timestamp time.Time) (float64, error) {
+	hourAgo := time.Now().Add(-time.Duration(1) * time.Hour)
+	withinLastHour := timestamp.After(hourAgo)
 	start, end := rangeFromTime(timestamp, 4)
 	url := fmt.Sprintf("%vcoins/%v/market_chart/range?vs_currency=usd&from=%v&to=%v", cgBaseUrl, coinId, start.Unix(), end.Unix())
 	req, reqErr := http.NewRequest("GET", url, nil)
@@ -81,13 +188,23 @@ func fetchCoinGeckoPrice(coinId string, timestamp time.Time) (float64, error) {
 	parseErr := json.Unmarshal(body, &parsed)
 	if parseErr != nil {
 		log.Printf("failed parsing body. err %v\n", parseErr)
+		var errRes CoinGeckoErrorRes
+		if err := json.Unmarshal(body, &errRes); err == nil {
+			log.Println("Failed calling CoinGecko, got err", errRes.Error)
+		}
 	}
 	if len(parsed.Prices) >= 1 {
-		numPrices := len(parsed.Prices)
+		var priceIndex int
-		middle := numPrices / 2
+		if withinLastHour {
-		// take the price in the middle of the range, as that should be
+			// use the last price in the list, latest price
-		// closest to the timestamp.
+			priceIndex = len(parsed.Prices) - 1
-		price := parsed.Prices[middle][1]
+		} else {
+			// use a price from the middle of the list, as that should be
+			// closest to the timestamp.
+			numPrices := len(parsed.Prices)
+			priceIndex = numPrices / 2
+		}
+		price := parsed.Prices[priceIndex][1]
 		fmt.Printf("found a price for %v! %v\n", coinId, price)
 		return price, nil
 	}

event_database/cloud_functions/process-transfer.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"math"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/certusone/wormhole/node/pkg/vaa"
@@ -25,38 +24,6 @@ var tokenAddressExceptions = map[string]string{
 	"010000000000000000000000000000000000000000000000000000756c756e61": "uluna",
 }
 
-func fetchTokenPrice(chain vaa.ChainID, symbol, address string, timestamp time.Time) (float64, string, string) {
-	// try coingecko, return if good
-	// if coingecko does not work, try chain-specific options
-
-	// initialize strings that will be returned if we find a symbol/name
-	// when looking up this token by contract address
-	foundSymbol := ""
-	foundName := ""
-
-	if symbol == "" && chain == vaa.ChainIDSolana {
-		// try to lookup the symbol in solana token list, from the address
-		if token, ok := solanaTokens[address]; ok {
-			symbol = token.Symbol
-			foundSymbol = token.Symbol
-			foundName = token.Name
-		}
-	}
-
-	coinGeckoId := ""
-	if _, ok := coinGeckoCoins[strings.ToLower(symbol)]; ok {
-		tokens := coinGeckoCoins[strings.ToLower(symbol)]
-		coinGeckoId = tokens[0].Id
-	}
-	if coinGeckoId != "" {
-		price, _ := fetchCoinGeckoPrice(coinGeckoId, timestamp)
-		if price != 0 {
-			return price, foundSymbol, foundName
-		}
-	}
-	return float64(0), foundSymbol, foundName
-}
-
 // returns a pair of dates before and after the input time.
 // useful for creating a time rage for querying historical price APIs.
 func rangeFromTime(t time.Time, hours int) (start time.Time, end time.Time) {
@@ -69,10 +36,10 @@ func transformHexAddressToNative(chain vaa.ChainID, address string) string {
 	case vaa.ChainIDSolana:
 		addr, err := hex.DecodeString(address)
 		if err != nil {
-			panic(fmt.Errorf("failed to decode solana string: %v", err))
+			log.Fatalf("failed to decode solana string: %v", err)
 		}
 		if len(addr) != 32 {
-			panic(fmt.Errorf("address must be 32 bytes. address: %v", address))
+			log.Fatalf("address must be 32 bytes. address: %v", address)
 		}
 		solPk := solana.PublicKeyFromBytes(addr[:])
 		return solPk.String()
@@ -174,6 +141,8 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error {
 	var decimals int
 	var symbol string
 	var name string
+	var coinId string
+	var nativeTokenAddress string
 	for _, item := range assetMetaRow[columnFamilies[3]] {
 		switch item.Column {
 		case "AssetMetaPayload:Decimals":
@@ -187,8 +156,17 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error {
 			symbol = string(item.Value)
 		case "AssetMetaPayload:Name":
 			name = string(item.Value)
+		case "AssetMetaPayload:CoinGeckoCoinId":
+			coinId = string(item.Value)
+		case "AssetMetaPayload:NativeAddress":
+			nativeTokenAddress = string(item.Value)
 		}
 	}
+	if coinId == "" {
+		log.Printf("no coinId for symbol: %v, nothing to lookup.\n", symbol)
+		// no coinId for this asset, cannot get price from coingecko.
+		return nil
+	}
 
 	// transfers created by the bridge UI will have at most 8 decimals.
 	if decimals > 8 {
@@ -203,11 +181,8 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error {
 	decAmount := amount[len(amount)-decimals:]
 	calculatedAmount := intAmount + "." + decAmount
 
-	nativeTokenAddress := transformHexAddressToNative(tokenChain, tokenAddress)
-
 	timestamp := signedVaa.Timestamp.UTC()
-
+	price, _ := fetchCoinGeckoPrice(coinId, timestamp)
-	price, foundSymbol, foundName := fetchTokenPrice(tokenChain, symbol, nativeTokenAddress, timestamp)
 
 	if price == 0 {
 		// no price found, don't save
@@ -215,14 +190,6 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error {
 		return nil
 	}
 
-	// update symbol and name if they are missing
-	if symbol == "" {
-		symbol = foundSymbol
-	}
-	if name == "" {
-		name = foundName
-	}
-
 	// convert the amount string so it can be used for math
 	amountFloat, convErr := strconv.ParseFloat(calculatedAmount, 64)
 	if convErr != nil {

event_database/cloud_functions/process-vaa.go

@@ -282,17 +282,37 @@ func ProcessVAA(ctx context.Context, m PubSubMessage) error {
 				return decodeErr
 			}
 
+			addressHex := hex.EncodeToString(payload.TokenAddress[:])
+			chainID := vaa.ChainID(payload.TokenChain)
+			nativeAddress := transformHexAddressToNative(chainID, addressHex)
+			name := string(TrimUnicodeFromByteArray(payload.Name[:]))
+			symbol := string(TrimUnicodeFromByteArray(payload.Symbol[:]))
+
+			// find the CoinGecko id of this token
+			coinGeckoCoinId, foundSymbol, foundName := fetchCoinGeckoCoinId(chainID, nativeAddress, symbol, name)
+
+			// populate the symbol & name if they were blank, and we found values
+			if symbol == "" && foundSymbol != "" {
+				symbol = foundSymbol
+			}
+			if name == "" && foundName != "" {
+				name = foundName
+			}
+
 			// save payload to bigtable
 			colFam := columnFamilies[3]
 			mutation := bigtable.NewMutation()
 			ts := bigtable.Now()
 
 			mutation.Set(colFam, "PayloadId", ts, []byte(fmt.Sprint(payload.PayloadId)))
-			mutation.Set(colFam, "TokenAddress", ts, []byte(hex.EncodeToString(payload.TokenAddress[:])))
+			mutation.Set(colFam, "TokenAddress", ts, []byte(addressHex))
 			mutation.Set(colFam, "TokenChain", ts, []byte(fmt.Sprint(payload.TokenChain)))
 			mutation.Set(colFam, "Decimals", ts, []byte(fmt.Sprint(payload.Decimals)))
-			mutation.Set(colFam, "Name", ts, TrimUnicodeFromByteArray(payload.Name[:]))
+			mutation.Set(colFam, "Name", ts, []byte(name))
-			mutation.Set(colFam, "Symbol", ts, TrimUnicodeFromByteArray(payload.Symbol[:]))
+			mutation.Set(colFam, "Symbol", ts, []byte(symbol))
+			mutation.Set(colFam, "CoinGeckoCoinId", ts, []byte(coinGeckoCoinId))
+			mutation.Set(colFam, "NativeAddress", ts, []byte(nativeAddress))
+
 			writeErr := writePayloadToBigTable(ctx, rowKey, colFam, mutation)
 			if writeErr != nil {
 				log.Println("wrote TokenTransferPayload to bigtable!", rowKey)