diff --git a/event_database/cloud_functions/external-data.go b/event_database/cloud_functions/external-data.go index 9f5c0e30..0665c3a1 100644 --- a/event_database/cloud_functions/external-data.go +++ b/event_database/cloud_functions/external-data.go @@ -5,9 +5,12 @@ import ( "fmt" "io/ioutil" "log" + "strings" "time" "net/http" + + "github.com/certusone/wormhole/node/pkg/vaa" ) const cgBaseUrl = "https://api.coingecko.com/api/v3/" @@ -24,6 +27,9 @@ type CoinGeckoMarket [2]float64 type CoinGeckoMarketRes struct { Prices []CoinGeckoMarket `json:"prices"` } +type CoinGeckoErrorRes struct { + Error string `json:"error"` +} func fetchCoinGeckoCoins() map[string][]CoinGeckoCoin { url := fmt.Sprintf("%vcoins/list", cgBaseUrl) @@ -51,13 +57,114 @@ func fetchCoinGeckoCoins() map[string][]CoinGeckoCoin { } var geckoCoins = map[string][]CoinGeckoCoin{} for _, coin := range parsed { - geckoCoins[coin.Symbol] = append(geckoCoins[coin.Symbol], coin) + symbol := strings.ToLower(coin.Symbol) + geckoCoins[symbol] = append(geckoCoins[symbol], coin) } return geckoCoins } +func chainIdToCoinGeckoPlatform(chain vaa.ChainID) string { + switch chain { + case vaa.ChainIDSolana: + return "solana" + case vaa.ChainIDEthereum: + return "ethereum" + case vaa.ChainIDTerra: + return "terra" + case vaa.ChainIDBSC: + return "binance-smart-chain" + case vaa.ChainIDPolygon: + return "polygon-pos" + } + return "" +} + +func fetchCoinGeckoCoinFromContract(chainId vaa.ChainID, address string) CoinGeckoCoin { + platform := chainIdToCoinGeckoPlatform(chainId) + url := fmt.Sprintf("%vcoins/%v/contract/%v", cgBaseUrl, platform, address) + req, reqErr := http.NewRequest("GET", url, nil) + if reqErr != nil { + log.Fatalf("failed contract request, err: %v\n", reqErr) + } + + res, resErr := http.DefaultClient.Do(req) + if resErr != nil { + log.Fatalf("failed get contract response, err: %v\n", resErr) + } + + defer res.Body.Close() + body, bodyErr := ioutil.ReadAll(res.Body) + if bodyErr != nil { + log.Fatalf("failed decoding contract body, err: %v\n", bodyErr) + } + + var parsed CoinGeckoCoin + + parseErr := json.Unmarshal(body, &parsed) + if parseErr != nil { + log.Printf("failed parsing body. err %v\n", parseErr) + var errRes CoinGeckoErrorRes + if err := json.Unmarshal(body, &errRes); err == nil { + if errRes.Error == "Could not find coin with the given id" { + log.Printf("Could not find CoinGecko coin by contract address, for chain %v, address, %v\n", chainId, address) + } else { + log.Println("Failed calling CoinGecko, got err", errRes.Error) + } + } + } + + return parsed +} + +func fetchCoinGeckoCoinId(chainId vaa.ChainID, address, symbol, name string) (coinId, foundSymbol, foundName string) { + // try coingecko, return if good + // if coingecko does not work, try chain-specific options + + // initialize strings that will be returned if we find a symbol/name + // when looking up this token by contract address + newSymbol := "" + newName := "" + + if symbol == "" && chainId == vaa.ChainIDSolana { + // try to lookup the symbol in solana token list, from the address + if token, ok := solanaTokens[address]; ok { + symbol = token.Symbol + name = token.Name + newSymbol = token.Symbol + newName = token.Name + } + } + if _, ok := coinGeckoCoins[strings.ToLower(symbol)]; ok { + tokens := coinGeckoCoins[strings.ToLower(symbol)] + if len(tokens) == 1 { + // only one match found for this symbol + return tokens[0].Id, newSymbol, newName + } + for _, token := range tokens { + if token.Name == name { + // found token by name match + return token.Id, newSymbol, newName + } + if strings.Contains(strings.ToLower(strings.ReplaceAll(name, " ", "")), strings.ReplaceAll(token.Id, "-", "")) { + // found token by id match + log.Println("found token by symbol and name match", name) + return token.Id, newSymbol, newName + } + } + // more than one symbol with this name, let contract lookup try + } + coin := fetchCoinGeckoCoinFromContract(chainId, address) + if coin.Id != "" { + return coin.Id, newSymbol, newName + } + // could not find a CoinGecko coin + return "", newSymbol, newName +} + func fetchCoinGeckoPrice(coinId string, timestamp time.Time) (float64, error) { + hourAgo := time.Now().Add(-time.Duration(1) * time.Hour) + withinLastHour := timestamp.After(hourAgo) start, end := rangeFromTime(timestamp, 4) url := fmt.Sprintf("%vcoins/%v/market_chart/range?vs_currency=usd&from=%v&to=%v", cgBaseUrl, coinId, start.Unix(), end.Unix()) req, reqErr := http.NewRequest("GET", url, nil) @@ -81,13 +188,23 @@ func fetchCoinGeckoPrice(coinId string, timestamp time.Time) (float64, error) { parseErr := json.Unmarshal(body, &parsed) if parseErr != nil { log.Printf("failed parsing body. err %v\n", parseErr) + var errRes CoinGeckoErrorRes + if err := json.Unmarshal(body, &errRes); err == nil { + log.Println("Failed calling CoinGecko, got err", errRes.Error) + } } if len(parsed.Prices) >= 1 { - numPrices := len(parsed.Prices) - middle := numPrices / 2 - // take the price in the middle of the range, as that should be - // closest to the timestamp. - price := parsed.Prices[middle][1] + var priceIndex int + if withinLastHour { + // use the last price in the list, latest price + priceIndex = len(parsed.Prices) - 1 + } else { + // use a price from the middle of the list, as that should be + // closest to the timestamp. + numPrices := len(parsed.Prices) + priceIndex = numPrices / 2 + } + price := parsed.Prices[priceIndex][1] fmt.Printf("found a price for %v! %v\n", coinId, price) return price, nil } diff --git a/event_database/cloud_functions/process-transfer.go b/event_database/cloud_functions/process-transfer.go index bdba6a22..de74f4c4 100644 --- a/event_database/cloud_functions/process-transfer.go +++ b/event_database/cloud_functions/process-transfer.go @@ -8,7 +8,6 @@ import ( "log" "math" "strconv" - "strings" "time" "github.com/certusone/wormhole/node/pkg/vaa" @@ -25,38 +24,6 @@ var tokenAddressExceptions = map[string]string{ "010000000000000000000000000000000000000000000000000000756c756e61": "uluna", } -func fetchTokenPrice(chain vaa.ChainID, symbol, address string, timestamp time.Time) (float64, string, string) { - // try coingecko, return if good - // if coingecko does not work, try chain-specific options - - // initialize strings that will be returned if we find a symbol/name - // when looking up this token by contract address - foundSymbol := "" - foundName := "" - - if symbol == "" && chain == vaa.ChainIDSolana { - // try to lookup the symbol in solana token list, from the address - if token, ok := solanaTokens[address]; ok { - symbol = token.Symbol - foundSymbol = token.Symbol - foundName = token.Name - } - } - - coinGeckoId := "" - if _, ok := coinGeckoCoins[strings.ToLower(symbol)]; ok { - tokens := coinGeckoCoins[strings.ToLower(symbol)] - coinGeckoId = tokens[0].Id - } - if coinGeckoId != "" { - price, _ := fetchCoinGeckoPrice(coinGeckoId, timestamp) - if price != 0 { - return price, foundSymbol, foundName - } - } - return float64(0), foundSymbol, foundName -} - // returns a pair of dates before and after the input time. // useful for creating a time rage for querying historical price APIs. func rangeFromTime(t time.Time, hours int) (start time.Time, end time.Time) { @@ -69,10 +36,10 @@ func transformHexAddressToNative(chain vaa.ChainID, address string) string { case vaa.ChainIDSolana: addr, err := hex.DecodeString(address) if err != nil { - panic(fmt.Errorf("failed to decode solana string: %v", err)) + log.Fatalf("failed to decode solana string: %v", err) } if len(addr) != 32 { - panic(fmt.Errorf("address must be 32 bytes. address: %v", address)) + log.Fatalf("address must be 32 bytes. address: %v", address) } solPk := solana.PublicKeyFromBytes(addr[:]) return solPk.String() @@ -174,6 +141,8 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error { var decimals int var symbol string var name string + var coinId string + var nativeTokenAddress string for _, item := range assetMetaRow[columnFamilies[3]] { switch item.Column { case "AssetMetaPayload:Decimals": @@ -187,8 +156,17 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error { symbol = string(item.Value) case "AssetMetaPayload:Name": name = string(item.Value) + case "AssetMetaPayload:CoinGeckoCoinId": + coinId = string(item.Value) + case "AssetMetaPayload:NativeAddress": + nativeTokenAddress = string(item.Value) } } + if coinId == "" { + log.Printf("no coinId for symbol: %v, nothing to lookup.\n", symbol) + // no coinId for this asset, cannot get price from coingecko. + return nil + } // transfers created by the bridge UI will have at most 8 decimals. if decimals > 8 { @@ -203,11 +181,8 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error { decAmount := amount[len(amount)-decimals:] calculatedAmount := intAmount + "." + decAmount - nativeTokenAddress := transformHexAddressToNative(tokenChain, tokenAddress) - timestamp := signedVaa.Timestamp.UTC() - - price, foundSymbol, foundName := fetchTokenPrice(tokenChain, symbol, nativeTokenAddress, timestamp) + price, _ := fetchCoinGeckoPrice(coinId, timestamp) if price == 0 { // no price found, don't save @@ -215,14 +190,6 @@ func ProcessTransfer(ctx context.Context, m PubSubMessage) error { return nil } - // update symbol and name if they are missing - if symbol == "" { - symbol = foundSymbol - } - if name == "" { - name = foundName - } - // convert the amount string so it can be used for math amountFloat, convErr := strconv.ParseFloat(calculatedAmount, 64) if convErr != nil { diff --git a/event_database/cloud_functions/process-vaa.go b/event_database/cloud_functions/process-vaa.go index 88401ceb..72c3c2e9 100644 --- a/event_database/cloud_functions/process-vaa.go +++ b/event_database/cloud_functions/process-vaa.go @@ -282,17 +282,37 @@ func ProcessVAA(ctx context.Context, m PubSubMessage) error { return decodeErr } + addressHex := hex.EncodeToString(payload.TokenAddress[:]) + chainID := vaa.ChainID(payload.TokenChain) + nativeAddress := transformHexAddressToNative(chainID, addressHex) + name := string(TrimUnicodeFromByteArray(payload.Name[:])) + symbol := string(TrimUnicodeFromByteArray(payload.Symbol[:])) + + // find the CoinGecko id of this token + coinGeckoCoinId, foundSymbol, foundName := fetchCoinGeckoCoinId(chainID, nativeAddress, symbol, name) + + // populate the symbol & name if they were blank, and we found values + if symbol == "" && foundSymbol != "" { + symbol = foundSymbol + } + if name == "" && foundName != "" { + name = foundName + } + // save payload to bigtable colFam := columnFamilies[3] mutation := bigtable.NewMutation() ts := bigtable.Now() mutation.Set(colFam, "PayloadId", ts, []byte(fmt.Sprint(payload.PayloadId))) - mutation.Set(colFam, "TokenAddress", ts, []byte(hex.EncodeToString(payload.TokenAddress[:]))) + mutation.Set(colFam, "TokenAddress", ts, []byte(addressHex)) mutation.Set(colFam, "TokenChain", ts, []byte(fmt.Sprint(payload.TokenChain))) mutation.Set(colFam, "Decimals", ts, []byte(fmt.Sprint(payload.Decimals))) - mutation.Set(colFam, "Name", ts, TrimUnicodeFromByteArray(payload.Name[:])) - mutation.Set(colFam, "Symbol", ts, TrimUnicodeFromByteArray(payload.Symbol[:])) + mutation.Set(colFam, "Name", ts, []byte(name)) + mutation.Set(colFam, "Symbol", ts, []byte(symbol)) + mutation.Set(colFam, "CoinGeckoCoinId", ts, []byte(coinGeckoCoinId)) + mutation.Set(colFam, "NativeAddress", ts, []byte(nativeAddress)) + writeErr := writePayloadToBigTable(ctx, rowKey, colFam, mutation) if writeErr != nil { log.Println("wrote TokenTransferPayload to bigtable!", rowKey)