From eacb7c3f06873f944022db1e6ab3aead78104634 Mon Sep 17 00:00:00 2001 From: Kevin Peters Date: Wed, 20 Jul 2022 16:22:18 +0000 Subject: [PATCH] cloud_functions: Added LOAD_CACHE env variable Can be set to false to effectively rebuild cache files. --- .../addresses-transferred-to-cumulative.go | 4 ++-- .../cloud_functions/addresses-transferred-to.go | 2 +- event_database/cloud_functions/nfts.go | 2 +- .../cloud_functions/notional-transferred-from.go | 9 +++++---- .../notional-transferred-to-cumulative.go | 4 ++-- .../cloud_functions/notional-transferred-to.go | 2 +- .../cloud_functions/notional-transferred.go | 2 +- .../cloud_functions/notional-tvl-cumulative.go | 11 ++++++----- event_database/cloud_functions/notional-tvl.go | 5 ++--- event_database/cloud_functions/recent.go | 2 +- event_database/cloud_functions/shared.go | 8 ++++++++ event_database/cloud_functions/totals.go | 2 +- event_database/functions_server/.vscode/launch.json | 3 +++ 13 files changed, 34 insertions(+), 22 deletions(-) diff --git a/event_database/cloud_functions/addresses-transferred-to-cumulative.go b/event_database/cloud_functions/addresses-transferred-to-cumulative.go index ac7123194..62543d7d1 100644 --- a/event_database/cloud_functions/addresses-transferred-to-cumulative.go +++ b/event_database/cloud_functions/addresses-transferred-to-cumulative.go @@ -35,7 +35,7 @@ var addressesToUpToYesterdayFilePath = "addresses-transferred-to-up-to-yesterday // finds all the unique addresses that have received tokens since a particular moment. func addressesTransferredToSince(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]float64 { - if _, ok := addressesToUpToYesterday["*"]; !ok { + if _, ok := addressesToUpToYesterday["*"]; !ok && loadCache { loadJsonToInterface(ctx, addressesToUpToYesterdayFilePath, &muAddressesToUpToYesterday, &addressesToUpToYesterday) } @@ -125,7 +125,7 @@ func addressesTransferredToSince(tbl *bigtable.Table, ctx context.Context, prefi // calcuates a map of recepient address to notional value received, by chain, since the start time specified. func createCumulativeAddressesOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]map[string]float64 { - if _, ok := warmCumulativeAddressesCache["*"]; !ok { + if _, ok := warmCumulativeAddressesCache["*"]; !ok && loadCache { loadJsonToInterface(ctx, warmCumulativeAddressesCacheFilePath, &muWarmCumulativeAddressesCache, &warmCumulativeAddressesCache) } diff --git a/event_database/cloud_functions/addresses-transferred-to.go b/event_database/cloud_functions/addresses-transferred-to.go index 53a888cf4..fb019c4cf 100644 --- a/event_database/cloud_functions/addresses-transferred-to.go +++ b/event_database/cloud_functions/addresses-transferred-to.go @@ -112,7 +112,7 @@ func fetchAddressRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefix // finds unique addresses tokens have been sent to, for each day since the start time passed in. func createAddressesOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]map[string]float64 { - if _, ok := warmAddressesCache["*"]; !ok { + if _, ok := warmAddressesCache["*"]; !ok && loadCache { loadJsonToInterface(ctx, warmAddressesCacheFilePath, &muWarmAddressesCache, &warmAddressesCache) } diff --git a/event_database/cloud_functions/nfts.go b/event_database/cloud_functions/nfts.go index 190dd10b3..7b3917b85 100644 --- a/event_database/cloud_functions/nfts.go +++ b/event_database/cloud_functions/nfts.go @@ -51,7 +51,7 @@ func fetchNFTRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefix str } func createNFTCountsOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, numPrevDays int, keySegments int) (map[string]map[string]int, error) { - if _, ok := warmNFTCache["2021-09-13"]; !ok { + if _, ok := warmNFTCache["2021-09-13"]; !ok && loadCache { loadJsonToInterface(ctx, warmNFTCacheFilePath, &muWarmNFTCache, &warmNFTCache) } diff --git a/event_database/cloud_functions/notional-transferred-from.go b/event_database/cloud_functions/notional-transferred-from.go index 1b0f680ad..61b00d7fa 100644 --- a/event_database/cloud_functions/notional-transferred-from.go +++ b/event_database/cloud_functions/notional-transferred-from.go @@ -24,7 +24,7 @@ var transfersFromFilePath = "notional-transferred-from.json" // finds the daily amount transferred from each chain from the specified start to the present. func createTransfersFromOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) { - if len(transfersFromCache.Daily) == 0 { + if len(transfersFromCache.Daily) == 0 && loadCache { loadJsonToInterface(ctx, transfersFromFilePath, &muTransfersFromCache, &transfersFromCache) } @@ -67,6 +67,9 @@ func createTransfersFromOfInterval(tbl *bigtable.Table, ctx context.Context, pre } } // no cache for this query, initialize the map + if transfersFromCache.Daily == nil { + transfersFromCache.Daily = map[string]map[string]float64{} + } transfersFromCache.Daily[dateStr] = map[string]float64{"*": 0} muTransfersFromCache.Unlock() @@ -119,9 +122,7 @@ func ComputeNotionalTransferredFrom(w http.ResponseWriter, r *http.Request) { return } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - + ctx := context.Background() createTransfersFromOfInterval(tbl, ctx, "", releaseDay) w.WriteHeader(http.StatusOK) diff --git a/event_database/cloud_functions/notional-transferred-to-cumulative.go b/event_database/cloud_functions/notional-transferred-to-cumulative.go index af92c027e..7b6d6e2c5 100644 --- a/event_database/cloud_functions/notional-transferred-to-cumulative.go +++ b/event_database/cloud_functions/notional-transferred-to-cumulative.go @@ -35,7 +35,7 @@ var transferredToUpToYesterdayFilePath = "notional-transferred-to-up-to-yesterda // calculates the amount of each symbol transfered to each chain. func transferredToSince(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]float64 { - if _, ok := transferredToUpToYesterday["*"]; !ok { + if _, ok := transferredToUpToYesterday["*"]; !ok && loadCache { loadJsonToInterface(ctx, transferredToUpToYesterdayFilePath, &muTransferredToUpToYesterday, &transferredToUpToYesterday) } @@ -145,7 +145,7 @@ func getDaysInRange(start, end time.Time) []string { // calcuates a running total of notional value transferred, by symbol, since the start time specified. func createCumulativeAmountsOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]map[string]float64 { - if _, ok := warmCumulativeCache["*"]; !ok { + if _, ok := warmCumulativeCache["*"]; !ok && loadCache { loadJsonToInterface(ctx, warmCumulativeCacheFilePath, &muWarmCumulativeCache, &warmCumulativeCache) } diff --git a/event_database/cloud_functions/notional-transferred-to.go b/event_database/cloud_functions/notional-transferred-to.go index 61ee2c489..90480f93d 100644 --- a/event_database/cloud_functions/notional-transferred-to.go +++ b/event_database/cloud_functions/notional-transferred-to.go @@ -135,7 +135,7 @@ func fetchTransferRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefi // finds the daily amount of each symbol transferred to each chain, from the specified start to the present. func amountsTransferredToInInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]map[string]float64 { - if _, ok := warmTransfersToCache["*"]; !ok { + if _, ok := warmTransfersToCache["*"]; !ok && loadCache { loadJsonToInterface(ctx, warmTransfersToCacheFilePath, &muWarmTransfersToCache, &warmTransfersToCache) } diff --git a/event_database/cloud_functions/notional-transferred.go b/event_database/cloud_functions/notional-transferred.go index f56de33dd..9e4701417 100644 --- a/event_database/cloud_functions/notional-transferred.go +++ b/event_database/cloud_functions/notional-transferred.go @@ -30,7 +30,7 @@ var warmTransfersCacheFilePath = "notional-transferred-cache.json" // finds the daily amount of each symbol transferred from each chain, to each chain, // from the specified start to the present. func createTransfersOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start time.Time) map[string]map[string]map[string]map[string]float64 { - if _, ok := warmTransfersCache["*"]; !ok { + if _, ok := warmTransfersCache["*"]; !ok && loadCache { loadJsonToInterface(ctx, warmTransfersCacheFilePath, &muWarmTransfersCache, &warmTransfersCache) } diff --git a/event_database/cloud_functions/notional-tvl-cumulative.go b/event_database/cloud_functions/notional-tvl-cumulative.go index 9cb0f26c3..5830033f2 100644 --- a/event_database/cloud_functions/notional-tvl-cumulative.go +++ b/event_database/cloud_functions/notional-tvl-cumulative.go @@ -41,8 +41,10 @@ func loadAndUpdateCoinGeckoPriceCache(ctx context.Context, coinIds []string, now // at cold-start, load the price cache into memory, and fetch any missing token price histories and add them to the cache if !loadedCoinGeckoPriceCache { // load the price cache - loadJsonToInterface(ctx, coinGeckoPriceCacheFilePath, &muWarmTvlCumulativeCache, &coinGeckoPriceCache) - loadedCoinGeckoPriceCache = true + if loadCache { + loadJsonToInterface(ctx, coinGeckoPriceCacheFilePath, &muWarmTvlCumulativeCache, &coinGeckoPriceCache) + loadedCoinGeckoPriceCache = true + } // find tokens missing price history missing := []string{} @@ -81,7 +83,7 @@ func loadAndUpdateCoinGeckoPriceCache(ctx context.Context, coinIds []string, now // calculates a running total of notional value transferred, by symbol, since the start time specified. func createTvlCumulativeOfInterval(tbl *bigtable.Table, ctx context.Context, start time.Time) map[string]map[string]map[string]LockedAsset { - if len(warmTvlCumulativeCache) == 0 { + if len(warmTvlCumulativeCache) == 0 && loadCache { loadJsonToInterface(ctx, warmTvlCumulativeCacheFilePath, &muWarmTvlCumulativeCache, &warmTvlCumulativeCache) } @@ -271,8 +273,7 @@ func ComputeTvlCumulative(w http.ResponseWriter, r *http.Request) { // days since launch day queryDays := int(time.Now().UTC().Sub(releaseDay).Hours() / 24) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() + ctx := context.Background() dailyTvl := map[string]map[string]map[string]LockedAsset{} diff --git a/event_database/cloud_functions/notional-tvl.go b/event_database/cloud_functions/notional-tvl.go index 502cbb4cb..3f9913f4b 100644 --- a/event_database/cloud_functions/notional-tvl.go +++ b/event_database/cloud_functions/notional-tvl.go @@ -38,7 +38,7 @@ type LockedAsset struct { // finds the daily amount of each symbol transferred to each chain, from the specified start to the present. func tvlInInterval(tbl *bigtable.Table, ctx context.Context, start time.Time) map[string]map[string]map[string]LockedAsset { - if len(warmTvlCache) == 0 { + if len(warmTvlCache) == 0 && loadCache { loadJsonToInterface(ctx, warmTvlFilePath, &muWarmTvlCache, &warmTvlCache) } @@ -303,8 +303,7 @@ func ComputeTVL(w http.ResponseWriter, r *http.Request) { // Set CORS headers for the main request. w.Header().Set("Access-Control-Allow-Origin", "*") - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() + ctx := context.Background() now := time.Now().UTC() todaysDateStr := now.Format("2006-01-02") diff --git a/event_database/cloud_functions/recent.go b/event_database/cloud_functions/recent.go index f2ad2f656..b9bc1e718 100644 --- a/event_database/cloud_functions/recent.go +++ b/event_database/cloud_functions/recent.go @@ -34,7 +34,7 @@ func getLatestOfEachEmitterAddress(tbl *bigtable.Table, ctx context.Context, pre if prefix == "" { cachePrefix = "*" } - if _, ok := warmCache[cachePrefix]; !ok { + if _, ok := warmCache[cachePrefix]; !ok && loadCache { loadJsonToInterface(ctx, warmRecentCacheFilePath, &muWarmRecentCache, &warmCache) } diff --git a/event_database/cloud_functions/shared.go b/event_database/cloud_functions/shared.go index abb791cea..97aaf6078 100644 --- a/event_database/cloud_functions/shared.go +++ b/event_database/cloud_functions/shared.go @@ -40,6 +40,8 @@ var solanaTokens = map[string]SolanaToken{} var releaseDay = time.Date(2021, 9, 13, 0, 0, 0, 0, time.UTC) +var loadCache = true + // init runs during cloud function initialization. So, this will only run during an // an instance's cold start. // https://cloud.google.com/functions/docs/bestpractices/networking#accessing_google_apis @@ -90,6 +92,12 @@ func init() { if tokenAllowlistFilePath != "" { loadJsonToInterface(context.Background(), tokenAllowlistFilePath, &sync.RWMutex{}, &tokenAllowlist) } + + loadCacheStr := os.Getenv("LOAD_CACHE") + if val, err := strconv.ParseBool(loadCacheStr); err == nil { + loadCache = val + log.Printf("loadCache set to %v\n", loadCache) + } } func timeTrack(start time.Time, name string) { diff --git a/event_database/cloud_functions/totals.go b/event_database/cloud_functions/totals.go index bffa87a37..4e7e62cc9 100644 --- a/event_database/cloud_functions/totals.go +++ b/event_database/cloud_functions/totals.go @@ -66,7 +66,7 @@ func fetchRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefix string } func createCountsOfInterval(tbl *bigtable.Table, ctx context.Context, prefix string, numPrevDays int, keySegments int) (map[string]map[string]int, error) { - if _, ok := warmTotalsCache["2021-09-13"]; !ok { + if _, ok := warmTotalsCache["2021-09-13"]; !ok && loadCache { loadJsonToInterface(ctx, warmTotalsCacheFilePath, &muWarmTotalsCache, &warmTotalsCache) } diff --git a/event_database/functions_server/.vscode/launch.json b/event_database/functions_server/.vscode/launch.json index 5a6768d07..a6b235111 100644 --- a/event_database/functions_server/.vscode/launch.json +++ b/event_database/functions_server/.vscode/launch.json @@ -20,6 +20,9 @@ "CACHE_BUCKET": "cloud-function-cache-mainnet", "TOKEN_ALLOWLIST": "token-allowlist-mainnet.json", "GOOGLE_APPLICATION_CREDENTIALS": "/home/you/path/to/your/service-account.json", + // LOAD_CACHE sets whether or not previously computed results should be loaded and used in calculations + // can be set to false to effectively rebuild the cache + // "LOAD_CACHE": "true", // CoinGecko API key if you have one. will work without - rate limit is lower. // "COINGECKO_API_KEY": "your-key-here", // SolanaBeach API key if you have one. will work without - rate limit is lower.