From 7f60e81b3ef8ac09bd4df499a41ff2190c8bb352 Mon Sep 17 00:00:00 2001 From: ftocal <46001274+ftocal@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:24:58 -0300 Subject: [PATCH] Fallback metrics endpoint (#921) Add to cache without expiration. Increase expiration to fetch data from influx to 10 minute. After 10 minute we update the data to influx is the response is success, if the response is not success we return the old data in the cache without expiration. Add alert when return metrics endpoint using cache that are expired. Co-authored-by: walker-16 --- api/cacheable/cacheable.go | 5 ++++- api/handlers/stats/repository_test.go | 23 ------------------- api/handlers/stats/service.go | 8 ++++--- api/handlers/transactions/service.go | 17 ++++++++------ api/internal/metrics/metrics.go | 7 ++++++ api/internal/metrics/prometheus.go | 32 +++++++++++++++++++++++++++ api/main.go | 9 +++++--- 7 files changed, 64 insertions(+), 37 deletions(-) delete mode 100644 api/handlers/stats/repository_test.go create mode 100644 api/internal/metrics/metrics.go create mode 100644 api/internal/metrics/prometheus.go diff --git a/api/cacheable/cacheable.go b/api/cacheable/cacheable.go index f7fc8944..4f6702e3 100644 --- a/api/cacheable/cacheable.go +++ b/api/cacheable/cacheable.go @@ -5,6 +5,7 @@ import ( "encoding/json" "time" + "github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics" "github.com/wormhole-foundation/wormhole-explorer/common/client/cache" "go.uber.org/zap" ) @@ -16,6 +17,7 @@ func GetOrLoad[T any]( cacheClient cache.Cache, expirations time.Duration, key string, + metrics metrics.Metrics, load func() (T, error), ) (T, error) { log := logger.With(zap.String("key", key)) @@ -48,6 +50,7 @@ func GetOrLoad[T any]( if err != nil { //If the load function fails and the cache was found and is expired, the cache value is returned anyway. if foundCache { + metrics.IncExpiredCacheResponse(key) log.Warn("load function fails but returns cached result", zap.Error(err), zap.String("cacheTime", cached.Timestamp.String())) return cached.Result, nil @@ -57,7 +60,7 @@ func GetOrLoad[T any]( //Saves the result of the execution of the load function in cache. newValue := CachedResult[T]{Timestamp: time.Now(), Result: result} - err = cacheClient.Set(ctx, key, newValue, 10*expirations) + err = cacheClient.Set(ctx, key, newValue, 0) if err != nil { log.Warn("saving the result in the cache", zap.Error(err)) } diff --git a/api/handlers/stats/repository_test.go b/api/handlers/stats/repository_test.go deleted file mode 100644 index 17fcc3c8..00000000 --- a/api/handlers/stats/repository_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package stats - -import ( - "context" - "testing" - - influxdb2 "github.com/influxdata/influxdb-client-go/v2" - "github.com/stretchr/testify/assert" - "go.uber.org/zap" -) - -func Test_convertToDecimal(t *testing.T) { - - url := "https://us-east-1-1.aws.cloud2.influxdata.com" - token := "FQ14tMrjuumxGGPlCIQvWfX_JDLUPJDOaTXKH_t3pHNDIvN13rbbmlG0JuuWvqo15Gw_qEjRqaeZ-BnCf0VaXA==" - cli := influxdb2.NewClient(url, token) - logger := zap.NewExample() - ctx := context.Background() - repo := NewRepository(cli, "xlabs", "wormscan-24hours-mainnet-staging", logger) - result, err := repo.GetSymbolWithAssets(ctx, TimeSpan30Days) - assert.NoError(t, err) - assert.NotNil(t, result) -} diff --git a/api/handlers/stats/service.go b/api/handlers/stats/service.go index b5fabde0..586edffc 100644 --- a/api/handlers/stats/service.go +++ b/api/handlers/stats/service.go @@ -6,6 +6,7 @@ import ( "time" "github.com/wormhole-foundation/wormhole-explorer/api/cacheable" + "github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics" "github.com/wormhole-foundation/wormhole-explorer/common/client/cache" "go.uber.org/zap" ) @@ -14,6 +15,7 @@ type Service struct { repo *Repository cache cache.Cache expiration time.Duration + metrics metrics.Metrics logger *zap.Logger } @@ -22,14 +24,14 @@ const ( ) // NewService create a new Service. -func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, logger *zap.Logger) *Service { - return &Service{repo: repo, cache: cache, expiration: expiration, logger: logger.With(zap.String("module", "StatsService"))} +func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, metrics metrics.Metrics, logger *zap.Logger) *Service { + return &Service{repo: repo, cache: cache, expiration: expiration, metrics: metrics, logger: logger.With(zap.String("module", "StatsService"))} } func (s *Service) GetSymbolWithAssets(ctx context.Context, ts SymbolWithAssetsTimeSpan) ([]SymbolWithAssetDTO, error) { key := topSymbolsByVolumeKey key = fmt.Sprintf("%s:%s", key, ts) - return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, + return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics, func() ([]SymbolWithAssetDTO, error) { return s.repo.GetSymbolWithAssets(ctx, ts) }) diff --git a/api/handlers/transactions/service.go b/api/handlers/transactions/service.go index 28b58c99..b08d5c0d 100644 --- a/api/handlers/transactions/service.go +++ b/api/handlers/transactions/service.go @@ -9,6 +9,7 @@ import ( "github.com/wormhole-foundation/wormhole-explorer/api/cacheable" "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors" errs "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors" + "github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics" "github.com/wormhole-foundation/wormhole-explorer/api/internal/pagination" "github.com/wormhole-foundation/wormhole-explorer/common/client/cache" "github.com/wormhole-foundation/wormhole-explorer/common/domain" @@ -23,6 +24,7 @@ type Service struct { expiration time.Duration supportedChainIDs map[vaa.ChainID]string tokenProvider *domain.TokenProvider + metrics metrics.Metrics logger *zap.Logger } @@ -35,23 +37,24 @@ const ( ) // NewService create a new Service. -func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, logger *zap.Logger) *Service { +func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, metrics metrics.Metrics, logger *zap.Logger) *Service { supportedChainIDs := domain.GetSupportedChainIDs() return &Service{repo: repo, supportedChainIDs: supportedChainIDs, - cache: cache, expiration: expiration, tokenProvider: tokenProvider, logger: logger.With(zap.String("module", "TransactionService"))} + cache: cache, expiration: expiration, tokenProvider: tokenProvider, metrics: metrics, + logger: logger.With(zap.String("module", "TransactionService"))} } // GetTransactionCount get the last transactions. func (s *Service) GetTransactionCount(ctx context.Context, q *TransactionCountQuery) ([]TransactionCountResult, error) { key := fmt.Sprintf("%s:%s:%s:%v", lastTxsKey, q.TimeSpan, q.SampleRate, q.CumulativeSum) - return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, + return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics, func() ([]TransactionCountResult, error) { return s.repo.GetTransactionCount(ctx, q) }) } func (s *Service) GetScorecards(ctx context.Context) (*Scorecards, error) { - return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, scorecardsKey, + return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, scorecardsKey, s.metrics, func() (*Scorecards, error) { return s.repo.GetScorecards(ctx) }) @@ -62,7 +65,7 @@ func (s *Service) GetTopAssets(ctx context.Context, timeSpan *TopStatisticsTimeS if timeSpan != nil { key = fmt.Sprintf("%s:%s", key, *timeSpan) } - return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, + return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics, func() ([]AssetDTO, error) { return s.repo.GetTopAssets(ctx, timeSpan) }) @@ -73,7 +76,7 @@ func (s *Service) GetTopChainPairs(ctx context.Context, timeSpan *TopStatisticsT if timeSpan != nil { key = fmt.Sprintf("%s:%s", key, *timeSpan) } - return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, + return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics, func() ([]ChainPairDTO, error) { return s.repo.GetTopChainPairs(ctx, timeSpan) }) @@ -82,7 +85,7 @@ func (s *Service) GetTopChainPairs(ctx context.Context, timeSpan *TopStatisticsT // GetChainActivity get chain activity. func (s *Service) GetChainActivity(ctx context.Context, q *ChainActivityQuery) ([]ChainActivityResult, error) { key := fmt.Sprintf("%s:%s:%v:%s", chainActivityKey, q.TimeSpan, q.IsNotional, strings.Join(q.GetAppIDs(), ",")) - return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, + return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics, func() ([]ChainActivityResult, error) { return s.repo.FindChainActivity(ctx, q) }) diff --git a/api/internal/metrics/metrics.go b/api/internal/metrics/metrics.go new file mode 100644 index 00000000..c13a361b --- /dev/null +++ b/api/internal/metrics/metrics.go @@ -0,0 +1,7 @@ +package metrics + +const serviceName = "wormscan-api" + +type Metrics interface { + IncExpiredCacheResponse(key string) +} diff --git a/api/internal/metrics/prometheus.go b/api/internal/metrics/prometheus.go new file mode 100644 index 00000000..747e5c47 --- /dev/null +++ b/api/internal/metrics/prometheus.go @@ -0,0 +1,32 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// PrometheusMetrics is a Prometheus implementation of Metric interface. +type PrometheusMetrics struct { + expiredCacheResponseCount *prometheus.CounterVec +} + +// NewPrometheusMetrics returns a new instance of PrometheusMetrics. +func NewPrometheusMetrics(environment string) *PrometheusMetrics { + vaaTxTrackerCount := promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "expired_cache_response", + Help: "Total expired cache response by key", + ConstLabels: map[string]string{ + "environment": environment, + "service": serviceName, + }, + }, []string{"key"}) + + return &PrometheusMetrics{ + expiredCacheResponseCount: vaaTxTrackerCount, + } +} + +func (m *PrometheusMetrics) IncExpiredCacheResponse(key string) { + m.expiredCacheResponseCount.WithLabelValues(key).Inc() +} diff --git a/api/main.go b/api/main.go index fd1ab450..23203ed6 100644 --- a/api/main.go +++ b/api/main.go @@ -35,6 +35,7 @@ import ( "github.com/wormhole-foundation/wormhole-explorer/api/handlers/transactions" "github.com/wormhole-foundation/wormhole-explorer/api/handlers/vaa" "github.com/wormhole-foundation/wormhole-explorer/api/internal/config" + "github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics" "github.com/wormhole-foundation/wormhole-explorer/api/internal/tvl" "github.com/wormhole-foundation/wormhole-explorer/api/middleware" "github.com/wormhole-foundation/wormhole-explorer/api/response" @@ -160,19 +161,21 @@ func main() { // create token provider tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork) + metrics := metrics.NewPrometheusMetrics(cfg.Environment) + // Set up services rootLogger.Info("initializing services") - expirationTime := time.Duration(cfg.Cache.MetricExpiration) * time.Second + expirationTime := time.Duration(cfg.Cache.MetricExpiration) * time.Minute addressService := address.NewService(addressRepo, rootLogger) vaaService := vaa.NewService(vaaRepo, cache.Get, vaaParserFunc, rootLogger) obsService := observations.NewService(obsRepo, rootLogger) governorService := governor.NewService(governorRepo, rootLogger) infrastructureService := infrastructure.NewService(infrastructureRepo, rootLogger) heartbeatsService := heartbeats.NewService(heartbeatsRepo, rootLogger) - transactionsService := transactions.NewService(transactionsRepo, cache, expirationTime, tokenProvider, rootLogger) + transactionsService := transactions.NewService(transactionsRepo, cache, expirationTime, tokenProvider, metrics, rootLogger) relaysService := relays.NewService(relaysRepo, rootLogger) operationsService := operations.NewService(operationsRepo, rootLogger) - statsService := stats.NewService(statsRepo, cache, expirationTime, rootLogger) + statsService := stats.NewService(statsRepo, cache, expirationTime, metrics, rootLogger) // Set up a custom error handler response.SetEnableStackTrace(*cfg)