Fallback metrics endpoint (#921)

Add to cache without expiration.
Increase expiration to fetch data from influx to 10 minute. After 10 minute we update the data to influx is the response is success, if the response is not success we return the old data in the cache without expiration.
Add alert when return metrics endpoint using cache that are expired.

Co-authored-by: walker-16 <agpazos85@gmail.com>

api/cacheable/cacheable.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"time"
 
+	"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
 	"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
 	"go.uber.org/zap"
 )
@@ -16,6 +17,7 @@ func GetOrLoad[T any](
 	cacheClient cache.Cache,
 	expirations time.Duration,
 	key string,
+	metrics metrics.Metrics,
 	load func() (T, error),
 ) (T, error) {
 	log := logger.With(zap.String("key", key))
@@ -48,6 +50,7 @@ func GetOrLoad[T any](
 	if err != nil {
 		//If the load function fails and the cache was found and is expired, the cache value is returned anyway.
 		if foundCache {
+			metrics.IncExpiredCacheResponse(key)
 			log.Warn("load function fails but returns cached result",
 				zap.Error(err), zap.String("cacheTime", cached.Timestamp.String()))
 			return cached.Result, nil
@@ -57,7 +60,7 @@ func GetOrLoad[T any](
 
 	//Saves the result of the execution of the load function in cache.
 	newValue := CachedResult[T]{Timestamp: time.Now(), Result: result}
-	err = cacheClient.Set(ctx, key, newValue, 10*expirations)
+	err = cacheClient.Set(ctx, key, newValue, 0)
 	if err != nil {
 		log.Warn("saving the result in the cache", zap.Error(err))
 	}

api/handlers/stats/repository_test.go

@@ -1,23 +0,0 @@
-package stats
-
-import (
-	"context"
-	"testing"
-
-	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
-	"github.com/stretchr/testify/assert"
-	"go.uber.org/zap"
-)
-
-func Test_convertToDecimal(t *testing.T) {
-
-	url := "https://us-east-1-1.aws.cloud2.influxdata.com"
-	token := "FQ14tMrjuumxGGPlCIQvWfX_JDLUPJDOaTXKH_t3pHNDIvN13rbbmlG0JuuWvqo15Gw_qEjRqaeZ-BnCf0VaXA=="
-	cli := influxdb2.NewClient(url, token)
-	logger := zap.NewExample()
-	ctx := context.Background()
-	repo := NewRepository(cli, "xlabs", "wormscan-24hours-mainnet-staging", logger)
-	result, err := repo.GetSymbolWithAssets(ctx, TimeSpan30Days)
-	assert.NoError(t, err)
-	assert.NotNil(t, result)
-}

api/handlers/stats/service.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/wormhole-foundation/wormhole-explorer/api/cacheable"
+	"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
 	"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
 	"go.uber.org/zap"
 )
@@ -14,6 +15,7 @@ type Service struct {
 	repo       *Repository
 	cache      cache.Cache
 	expiration time.Duration
+	metrics    metrics.Metrics
 	logger     *zap.Logger
 }
 
@@ -22,14 +24,14 @@ const (
 )
 
 // NewService create a new Service.
-func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, logger *zap.Logger) *Service {
+func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, metrics metrics.Metrics, logger *zap.Logger) *Service {
-	return &Service{repo: repo, cache: cache, expiration: expiration, logger: logger.With(zap.String("module", "StatsService"))}
+	return &Service{repo: repo, cache: cache, expiration: expiration, metrics: metrics, logger: logger.With(zap.String("module", "StatsService"))}
 }
 
 func (s *Service) GetSymbolWithAssets(ctx context.Context, ts SymbolWithAssetsTimeSpan) ([]SymbolWithAssetDTO, error) {
 	key := topSymbolsByVolumeKey
 	key = fmt.Sprintf("%s:%s", key, ts)
-	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
+	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
 		func() ([]SymbolWithAssetDTO, error) {
 			return s.repo.GetSymbolWithAssets(ctx, ts)
 		})

api/handlers/transactions/service.go

@@ -9,6 +9,7 @@ import (
 	"github.com/wormhole-foundation/wormhole-explorer/api/cacheable"
 	"github.com/wormhole-foundation/wormhole-explorer/api/internal/errors"
 	errs "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors"
+	"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
 	"github.com/wormhole-foundation/wormhole-explorer/api/internal/pagination"
 	"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
 	"github.com/wormhole-foundation/wormhole-explorer/common/domain"
@@ -23,6 +24,7 @@ type Service struct {
 	expiration        time.Duration
 	supportedChainIDs map[vaa.ChainID]string
 	tokenProvider     *domain.TokenProvider
+	metrics           metrics.Metrics
 	logger            *zap.Logger
 }
 
@@ -35,23 +37,24 @@ const (
 )
 
 // NewService create a new Service.
-func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, logger *zap.Logger) *Service {
+func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, metrics metrics.Metrics, logger *zap.Logger) *Service {
 	supportedChainIDs := domain.GetSupportedChainIDs()
 	return &Service{repo: repo, supportedChainIDs: supportedChainIDs,
-		cache: cache, expiration: expiration, tokenProvider: tokenProvider, logger: logger.With(zap.String("module", "TransactionService"))}
+		cache: cache, expiration: expiration, tokenProvider: tokenProvider, metrics: metrics,
+		logger: logger.With(zap.String("module", "TransactionService"))}
 }
 
 // GetTransactionCount get the last transactions.
 func (s *Service) GetTransactionCount(ctx context.Context, q *TransactionCountQuery) ([]TransactionCountResult, error) {
 	key := fmt.Sprintf("%s:%s:%s:%v", lastTxsKey, q.TimeSpan, q.SampleRate, q.CumulativeSum)
-	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
+	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
 		func() ([]TransactionCountResult, error) {
 			return s.repo.GetTransactionCount(ctx, q)
 		})
 }
 
 func (s *Service) GetScorecards(ctx context.Context) (*Scorecards, error) {
-	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, scorecardsKey,
+	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, scorecardsKey, s.metrics,
 		func() (*Scorecards, error) {
 			return s.repo.GetScorecards(ctx)
 		})
@@ -62,7 +65,7 @@ func (s *Service) GetTopAssets(ctx context.Context, timeSpan *TopStatisticsTimeS
 	if timeSpan != nil {
 		key = fmt.Sprintf("%s:%s", key, *timeSpan)
 	}
-	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
+	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
 		func() ([]AssetDTO, error) {
 			return s.repo.GetTopAssets(ctx, timeSpan)
 		})
@@ -73,7 +76,7 @@ func (s *Service) GetTopChainPairs(ctx context.Context, timeSpan *TopStatisticsT
 	if timeSpan != nil {
 		key = fmt.Sprintf("%s:%s", key, *timeSpan)
 	}
-	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
+	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
 		func() ([]ChainPairDTO, error) {
 			return s.repo.GetTopChainPairs(ctx, timeSpan)
 		})
@@ -82,7 +85,7 @@ func (s *Service) GetTopChainPairs(ctx context.Context, timeSpan *TopStatisticsT
 // GetChainActivity get chain activity.
 func (s *Service) GetChainActivity(ctx context.Context, q *ChainActivityQuery) ([]ChainActivityResult, error) {
 	key := fmt.Sprintf("%s:%s:%v:%s", chainActivityKey, q.TimeSpan, q.IsNotional, strings.Join(q.GetAppIDs(), ","))
-	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
+	return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
 		func() ([]ChainActivityResult, error) {
 			return s.repo.FindChainActivity(ctx, q)
 		})

api/internal/metrics/metrics.go

@@ -0,0 +1,7 @@
+package metrics
+
+const serviceName = "wormscan-api"
+
+type Metrics interface {
+	IncExpiredCacheResponse(key string)
+}

api/internal/metrics/prometheus.go

@@ -0,0 +1,32 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// PrometheusMetrics is a Prometheus implementation of Metric interface.
+type PrometheusMetrics struct {
+	expiredCacheResponseCount *prometheus.CounterVec
+}
+
+// NewPrometheusMetrics returns a new instance of PrometheusMetrics.
+func NewPrometheusMetrics(environment string) *PrometheusMetrics {
+	vaaTxTrackerCount := promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "expired_cache_response",
+			Help: "Total expired cache response by key",
+			ConstLabels: map[string]string{
+				"environment": environment,
+				"service":     serviceName,
+			},
+		}, []string{"key"})
+
+	return &PrometheusMetrics{
+		expiredCacheResponseCount: vaaTxTrackerCount,
+	}
+}
+
+func (m *PrometheusMetrics) IncExpiredCacheResponse(key string) {
+	m.expiredCacheResponseCount.WithLabelValues(key).Inc()
+}

api/main.go

@@ -35,6 +35,7 @@ import (
 	"github.com/wormhole-foundation/wormhole-explorer/api/handlers/transactions"
 	"github.com/wormhole-foundation/wormhole-explorer/api/handlers/vaa"
 	"github.com/wormhole-foundation/wormhole-explorer/api/internal/config"
+	"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
 	"github.com/wormhole-foundation/wormhole-explorer/api/internal/tvl"
 	"github.com/wormhole-foundation/wormhole-explorer/api/middleware"
 	"github.com/wormhole-foundation/wormhole-explorer/api/response"
@@ -160,19 +161,21 @@ func main() {
 	// create token provider
 	tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork)
 
+	metrics := metrics.NewPrometheusMetrics(cfg.Environment)
+
 	// Set up services
 	rootLogger.Info("initializing services")
-	expirationTime := time.Duration(cfg.Cache.MetricExpiration) * time.Second
+	expirationTime := time.Duration(cfg.Cache.MetricExpiration) * time.Minute
 	addressService := address.NewService(addressRepo, rootLogger)
 	vaaService := vaa.NewService(vaaRepo, cache.Get, vaaParserFunc, rootLogger)
 	obsService := observations.NewService(obsRepo, rootLogger)
 	governorService := governor.NewService(governorRepo, rootLogger)
 	infrastructureService := infrastructure.NewService(infrastructureRepo, rootLogger)
 	heartbeatsService := heartbeats.NewService(heartbeatsRepo, rootLogger)
-	transactionsService := transactions.NewService(transactionsRepo, cache, expirationTime, tokenProvider, rootLogger)
+	transactionsService := transactions.NewService(transactionsRepo, cache, expirationTime, tokenProvider, metrics, rootLogger)
 	relaysService := relays.NewService(relaysRepo, rootLogger)
 	operationsService := operations.NewService(operationsRepo, rootLogger)
-	statsService := stats.NewService(statsRepo, cache, expirationTime, rootLogger)
+	statsService := stats.NewService(statsRepo, cache, expirationTime, metrics, rootLogger)
 
 	// Set up a custom error handler
 	response.SetEnableStackTrace(*cfg)