lightwalletd/common/prices.go

package common

import (
	"encoding/gob"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/sirupsen/logrus"
)

var (
	// Map of all historical prices. Date as "yyyy-mm-dd" to price in USD
	historicalPrices map[string]float64 = make(map[string]float64)

	// The latest price
	latestPrice float64 = -1

	// Latest price was fetched at
	latestPriceTime time.Time

	// Mutex to guard both historical and latest price
	mutex sync.Mutex

	// Full path of the persistence file
	pricesFileName string
)

func fetchAPIPrice(url string, resultPath []string) (float64, error) {
	resp, err := http.Get(url)
	if err != nil {
		return -1, err
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return -1, err
	}

	var priceJSON map[string]interface{}
	json.Unmarshal(body, &priceJSON)

	for i := 0; i < len(resultPath); i++ {
		d, ok := priceJSON[resultPath[i]]
		if !ok {
			return -1, fmt.Errorf("API error: couldn't find '%s'", resultPath[i])
		}

		switch v := d.(type) {
		case float64:
			return v, nil
		case string:
			{
				price, err := strconv.ParseFloat(v, 64)
				return price, err
			}

		case map[string]interface{}:
			priceJSON = v
		}

	}

	return -1, errors.New("path didn't result in lookup")
}

func fetchCoinbasePrice() (float64, error) {
	return fetchAPIPrice("https://api.coinbase.com/v2/exchange-rates?currency=ZEC", []string{"data", "rates", "USD"})
}

func fetchCoinCapPrice() (float64, error) {
	return fetchAPIPrice("https://api.coincap.io/v2/rates/zcash", []string{"data", "rateUsd"})
}

func fetchBinancePrice() (float64, error) {
	return fetchAPIPrice("https://api.binance.com/api/v3/avgPrice?symbol=ZECUSDC", []string{"price"})
}

func fetchHistoricalCoingeckoPrice(ts time.Time) (float64, error) {
	dt := ts.Format("02-01-2006") // dd-mm-yyyy
	url := fmt.Sprintf("https://api.coingecko.com/api/v3/coins/zcash/history?date=%s", dt)

	return fetchAPIPrice(url, []string{"market_data", "current_price", "usd"})
}

// calcMedian calculates the median of a sorted slice of numbers
func calcMedian(inp []float64) (median float64) {
	// For even numbers we add the two middle numbers
	// and divide by two using the mean function above
	// For odd numbers we just use the middle number
	n := len(inp)
	if n%2 == 0 {
		return (inp[n/2-1] + inp[n/2]) / 2
	} else {
		return inp[n/2]
	}
}

// fetchPriceFromWebAPI will fetch prices from multiple places, discard outliers and return the
// concensus price. This function doesn't need the mutex.
func fetchPriceFromWebAPI() (float64, error) {
	// We'll fetch prices from all our endpoints, and use the median price from that
	priceProviders := []func() (float64, error){fetchBinancePrice, fetchCoinCapPrice, fetchCoinbasePrice}

	// Get all prices
	prices := make([]float64, 0)
	for _, provider := range priceProviders {
		price, err := provider()
		if err == nil {
			Log.WithFields(logrus.Fields{
				"method":   "CurrentPrice",
				"provider": runtime.FuncForPC(reflect.ValueOf(provider).Pointer()).Name(),
				"price":    price,
			}).Info("Service")
			prices = append(prices, price)
		} else {
			Log.WithFields(logrus.Fields{
				"method":   "CurrentPrice",
				"provider": runtime.FuncForPC(reflect.ValueOf(provider).Pointer()).Name(),
				"error":    err,
			}).Error("Service")
		}
	}
	if len(prices) == 0 {
		return -1, errors.New("no price providers are available")
	}
	sort.Float64s(prices)

	// Get the median price
	median := calcMedian(prices)

	// Discard all values that are more than 20% outside the median
	validPrices := make([]float64, 0)
	for _, price := range prices {
		if (math.Abs(price-median) / median) > 0.2 {
			Log.WithFields(logrus.Fields{
				"method": "CurrentPrice",
				"error":  fmt.Sprintf("Discarding price (%.2f) because too far away from median (%.2f", price, median),
			}).Error("Service")
		} else {
			validPrices = append(validPrices, price)
		}
	}

	// At least 2 (valid) providers are required; we don't want to depend on just one
	if len(validPrices) < 2 {
		return -1, errors.New("insufficient price providers are available")
	}

	// If we discarded too many, return an error
	if len(validPrices) < (len(prices)/2 + 1) {
		return -1, errors.New("not enough valid prices")
	}
	median = calcMedian(validPrices)
	if median <= 0 {
		return -1, errors.New("median price is <= 0")
	}
	return median, nil
}

func readHistoricalPricesFile() (map[string]float64, error) {
	f, err := os.Open(pricesFileName)
	if err != nil {
		Log.Errorf("Couldn't open file %s for writing: %v", pricesFileName, err)
		return nil, err
	}
	defer f.Close()

	j := gob.NewDecoder(f)
	var prices map[string]float64
	err = j.Decode(&prices)
	if err != nil {
		Log.Errorf("Couldn't decode historical prices: %v", err)
		return nil, err
	}

	Log.WithFields(logrus.Fields{
		"method":  "HistoricalPrice",
		"action":  "Read historical prices file",
		"records": len(prices),
	}).Info("Service")
	return prices, nil
}

func writeHistoricalPricesMap() {
	// Serialize the map to disk.
	f, err := os.OpenFile(pricesFileName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		Log.Errorf("Couldn't open file %s for writing: %v", pricesFileName, err)
		return
	}
	defer f.Close()

	j := gob.NewEncoder(f)
	if err = j.Encode(historicalPrices); err != nil {
		Log.Errorf("Couldn't encode historical prices: %v", err)
		return
	}
	Log.WithFields(logrus.Fields{
		"method": "HistoricalPrice",
		"action": "Wrote historical prices file",
	}).Info("Service")
}

// GetCurrentPrice is a top-level API, returns the latest price that we
// have fetched if no more than 3 hours old, else an error. An error
// should not occur unless we can't reach enough price oracles.
func GetCurrentPrice() (float64, error) {
	mutex.Lock()
	defer mutex.Unlock()

	if latestPriceTime.IsZero() {
		return -1, errors.New("starting up, prices not available yet")
	}

	// If the current price is too old, don't return it.
	if time.Since(latestPriceTime).Hours() > 3 {
		return -1, errors.New("price too old")
	}

	return latestPrice, nil
}

// return the time in YYYY-MM-DD string format
func day(t time.Time) string {
	return t.Format("2006-01-02")
}

// GetHistoricalPrice returns the price for the given day, but only
// accurate to day granularity.
func GetHistoricalPrice(ts time.Time) (float64, *time.Time, error) {
	dt := day(ts)
	canonicalTime, err := time.Parse("2006-01-02", dt)
	if err != nil {
		return -1, nil, err
	}
	mutex.Lock()
	defer mutex.Unlock()
	if val, ok := historicalPrices[dt]; ok {
		return val, &canonicalTime, nil
	}
	// Check if this is the same as the current latest price
	if latestPrice > 0 && day(latestPriceTime) == dt {
		return latestPrice, &canonicalTime, nil
	}

	// Fetch price from web API
	mutex.Unlock()
	price, err := fetchHistoricalCoingeckoPrice(ts)
	mutex.Lock()
	if err != nil {
		Log.Errorf("Couldn't read historical prices from Coingecko: %v", err)
		return -1, nil, err
	}
	if price <= 0 {
		Log.Errorf("historical prices from Coingecko <= 0")
		return -1, nil, errors.New("bad Coingecko result")
	}
	// add to our cache so we don't have to hit Coingecko again
	// for the same date
	addHistoricalPrice(price, ts)

	return price, &canonicalTime, nil
}

// Add a price entry for the given day both to our map
// and to the file (so we'll have it after a restart).
// This caching allows us to hit coingecko less often,
// and provides resilience when that site is down.
//
// There are two ways a historical price can be added:
//   - When a client calls GetZECPrice to get a past price
//   - When a new day begins, we'll save the previous day's price
//
func addHistoricalPrice(price float64, ts time.Time) {
	dt := day(ts)
	if _, ok := historicalPrices[dt]; !ok {
		// an entry for this day doesn't exist, add it
		historicalPrices[dt] = price
		Log.WithFields(logrus.Fields{
			"method": "HistoricalPrice",
			"action": "Add",
			"date":   dt,
			"price":  price,
		}).Info("Service")
		writeHistoricalPricesMap()
	}
}

// StartPriceFetcher starts a new thread that will fetch historical and current prices
func StartPriceFetcher(dbPath string, chainName string) {
	// Set the prices file name
	pricesFileName = filepath.Join(dbPath, chainName, "prices")

	// Read the historical prices if available
	mutex.Lock()
	if prices, err := readHistoricalPricesFile(); err != nil {
		Log.Errorf("Couldn't read historical prices, starting with empty map")
	} else {
		historicalPrices = prices
		Log.Infof("prices at start: %v", prices)
	}
	mutex.Unlock()

	// Fetch the current price every 15 mins
	go func() {
		for {
			price, err := fetchPriceFromWebAPI()
			if err != nil {
				Log.Errorf("Error getting prices from web APIs: %v", err)
			} else {
				Log.WithFields(logrus.Fields{
					"method": "CurrentPrice",
					"price":  price,
				}).Info("Service")

				mutex.Lock()
				// If the day has changed, save the previous day's
				// historical price. Historical prices are per-day.
				if day(latestPriceTime) != day(time.Now()) {
					if latestPrice > 0 {
						t := time.Unix(latestPriceTime.Unix(), 0)
						addHistoricalPrice(latestPrice, t)
					}
				}
				latestPrice = price
				latestPriceTime = time.Now()
				mutex.Unlock()
			}
			// price data 15 minutes out of date is probably okay
			time.Sleep(15 * time.Minute)
		}
	}()
}