Fallback metrics endpoint (#921)
Add to cache without expiration. Increase expiration to fetch data from influx to 10 minute. After 10 minute we update the data to influx is the response is success, if the response is not success we return the old data in the cache without expiration. Add alert when return metrics endpoint using cache that are expired. Co-authored-by: walker-16 <agpazos85@gmail.com>
This commit is contained in:
parent
fba3adef65
commit
7f60e81b3e
|
@ -5,6 +5,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
|
"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -16,6 +17,7 @@ func GetOrLoad[T any](
|
||||||
cacheClient cache.Cache,
|
cacheClient cache.Cache,
|
||||||
expirations time.Duration,
|
expirations time.Duration,
|
||||||
key string,
|
key string,
|
||||||
|
metrics metrics.Metrics,
|
||||||
load func() (T, error),
|
load func() (T, error),
|
||||||
) (T, error) {
|
) (T, error) {
|
||||||
log := logger.With(zap.String("key", key))
|
log := logger.With(zap.String("key", key))
|
||||||
|
@ -48,6 +50,7 @@ func GetOrLoad[T any](
|
||||||
if err != nil {
|
if err != nil {
|
||||||
//If the load function fails and the cache was found and is expired, the cache value is returned anyway.
|
//If the load function fails and the cache was found and is expired, the cache value is returned anyway.
|
||||||
if foundCache {
|
if foundCache {
|
||||||
|
metrics.IncExpiredCacheResponse(key)
|
||||||
log.Warn("load function fails but returns cached result",
|
log.Warn("load function fails but returns cached result",
|
||||||
zap.Error(err), zap.String("cacheTime", cached.Timestamp.String()))
|
zap.Error(err), zap.String("cacheTime", cached.Timestamp.String()))
|
||||||
return cached.Result, nil
|
return cached.Result, nil
|
||||||
|
@ -57,7 +60,7 @@ func GetOrLoad[T any](
|
||||||
|
|
||||||
//Saves the result of the execution of the load function in cache.
|
//Saves the result of the execution of the load function in cache.
|
||||||
newValue := CachedResult[T]{Timestamp: time.Now(), Result: result}
|
newValue := CachedResult[T]{Timestamp: time.Now(), Result: result}
|
||||||
err = cacheClient.Set(ctx, key, newValue, 10*expirations)
|
err = cacheClient.Set(ctx, key, newValue, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("saving the result in the cache", zap.Error(err))
|
log.Warn("saving the result in the cache", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,23 +0,0 @@
|
||||||
package stats
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Test_convertToDecimal(t *testing.T) {
|
|
||||||
|
|
||||||
url := "https://us-east-1-1.aws.cloud2.influxdata.com"
|
|
||||||
token := "FQ14tMrjuumxGGPlCIQvWfX_JDLUPJDOaTXKH_t3pHNDIvN13rbbmlG0JuuWvqo15Gw_qEjRqaeZ-BnCf0VaXA=="
|
|
||||||
cli := influxdb2.NewClient(url, token)
|
|
||||||
logger := zap.NewExample()
|
|
||||||
ctx := context.Background()
|
|
||||||
repo := NewRepository(cli, "xlabs", "wormscan-24hours-mainnet-staging", logger)
|
|
||||||
result, err := repo.GetSymbolWithAssets(ctx, TimeSpan30Days)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.NotNil(t, result)
|
|
||||||
}
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/cacheable"
|
"github.com/wormhole-foundation/wormhole-explorer/api/cacheable"
|
||||||
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
|
"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -14,6 +15,7 @@ type Service struct {
|
||||||
repo *Repository
|
repo *Repository
|
||||||
cache cache.Cache
|
cache cache.Cache
|
||||||
expiration time.Duration
|
expiration time.Duration
|
||||||
|
metrics metrics.Metrics
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,14 +24,14 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewService create a new Service.
|
// NewService create a new Service.
|
||||||
func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, logger *zap.Logger) *Service {
|
func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, metrics metrics.Metrics, logger *zap.Logger) *Service {
|
||||||
return &Service{repo: repo, cache: cache, expiration: expiration, logger: logger.With(zap.String("module", "StatsService"))}
|
return &Service{repo: repo, cache: cache, expiration: expiration, metrics: metrics, logger: logger.With(zap.String("module", "StatsService"))}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) GetSymbolWithAssets(ctx context.Context, ts SymbolWithAssetsTimeSpan) ([]SymbolWithAssetDTO, error) {
|
func (s *Service) GetSymbolWithAssets(ctx context.Context, ts SymbolWithAssetsTimeSpan) ([]SymbolWithAssetDTO, error) {
|
||||||
key := topSymbolsByVolumeKey
|
key := topSymbolsByVolumeKey
|
||||||
key = fmt.Sprintf("%s:%s", key, ts)
|
key = fmt.Sprintf("%s:%s", key, ts)
|
||||||
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
|
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
|
||||||
func() ([]SymbolWithAssetDTO, error) {
|
func() ([]SymbolWithAssetDTO, error) {
|
||||||
return s.repo.GetSymbolWithAssets(ctx, ts)
|
return s.repo.GetSymbolWithAssets(ctx, ts)
|
||||||
})
|
})
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/cacheable"
|
"github.com/wormhole-foundation/wormhole-explorer/api/cacheable"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/internal/errors"
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/errors"
|
||||||
errs "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors"
|
errs "github.com/wormhole-foundation/wormhole-explorer/api/internal/errors"
|
||||||
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/internal/pagination"
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/pagination"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
|
"github.com/wormhole-foundation/wormhole-explorer/common/client/cache"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
|
"github.com/wormhole-foundation/wormhole-explorer/common/domain"
|
||||||
|
@ -23,6 +24,7 @@ type Service struct {
|
||||||
expiration time.Duration
|
expiration time.Duration
|
||||||
supportedChainIDs map[vaa.ChainID]string
|
supportedChainIDs map[vaa.ChainID]string
|
||||||
tokenProvider *domain.TokenProvider
|
tokenProvider *domain.TokenProvider
|
||||||
|
metrics metrics.Metrics
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,23 +37,24 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewService create a new Service.
|
// NewService create a new Service.
|
||||||
func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, logger *zap.Logger) *Service {
|
func NewService(repo *Repository, cache cache.Cache, expiration time.Duration, tokenProvider *domain.TokenProvider, metrics metrics.Metrics, logger *zap.Logger) *Service {
|
||||||
supportedChainIDs := domain.GetSupportedChainIDs()
|
supportedChainIDs := domain.GetSupportedChainIDs()
|
||||||
return &Service{repo: repo, supportedChainIDs: supportedChainIDs,
|
return &Service{repo: repo, supportedChainIDs: supportedChainIDs,
|
||||||
cache: cache, expiration: expiration, tokenProvider: tokenProvider, logger: logger.With(zap.String("module", "TransactionService"))}
|
cache: cache, expiration: expiration, tokenProvider: tokenProvider, metrics: metrics,
|
||||||
|
logger: logger.With(zap.String("module", "TransactionService"))}
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetTransactionCount get the last transactions.
|
// GetTransactionCount get the last transactions.
|
||||||
func (s *Service) GetTransactionCount(ctx context.Context, q *TransactionCountQuery) ([]TransactionCountResult, error) {
|
func (s *Service) GetTransactionCount(ctx context.Context, q *TransactionCountQuery) ([]TransactionCountResult, error) {
|
||||||
key := fmt.Sprintf("%s:%s:%s:%v", lastTxsKey, q.TimeSpan, q.SampleRate, q.CumulativeSum)
|
key := fmt.Sprintf("%s:%s:%s:%v", lastTxsKey, q.TimeSpan, q.SampleRate, q.CumulativeSum)
|
||||||
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
|
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
|
||||||
func() ([]TransactionCountResult, error) {
|
func() ([]TransactionCountResult, error) {
|
||||||
return s.repo.GetTransactionCount(ctx, q)
|
return s.repo.GetTransactionCount(ctx, q)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) GetScorecards(ctx context.Context) (*Scorecards, error) {
|
func (s *Service) GetScorecards(ctx context.Context) (*Scorecards, error) {
|
||||||
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, scorecardsKey,
|
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, scorecardsKey, s.metrics,
|
||||||
func() (*Scorecards, error) {
|
func() (*Scorecards, error) {
|
||||||
return s.repo.GetScorecards(ctx)
|
return s.repo.GetScorecards(ctx)
|
||||||
})
|
})
|
||||||
|
@ -62,7 +65,7 @@ func (s *Service) GetTopAssets(ctx context.Context, timeSpan *TopStatisticsTimeS
|
||||||
if timeSpan != nil {
|
if timeSpan != nil {
|
||||||
key = fmt.Sprintf("%s:%s", key, *timeSpan)
|
key = fmt.Sprintf("%s:%s", key, *timeSpan)
|
||||||
}
|
}
|
||||||
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
|
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
|
||||||
func() ([]AssetDTO, error) {
|
func() ([]AssetDTO, error) {
|
||||||
return s.repo.GetTopAssets(ctx, timeSpan)
|
return s.repo.GetTopAssets(ctx, timeSpan)
|
||||||
})
|
})
|
||||||
|
@ -73,7 +76,7 @@ func (s *Service) GetTopChainPairs(ctx context.Context, timeSpan *TopStatisticsT
|
||||||
if timeSpan != nil {
|
if timeSpan != nil {
|
||||||
key = fmt.Sprintf("%s:%s", key, *timeSpan)
|
key = fmt.Sprintf("%s:%s", key, *timeSpan)
|
||||||
}
|
}
|
||||||
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
|
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
|
||||||
func() ([]ChainPairDTO, error) {
|
func() ([]ChainPairDTO, error) {
|
||||||
return s.repo.GetTopChainPairs(ctx, timeSpan)
|
return s.repo.GetTopChainPairs(ctx, timeSpan)
|
||||||
})
|
})
|
||||||
|
@ -82,7 +85,7 @@ func (s *Service) GetTopChainPairs(ctx context.Context, timeSpan *TopStatisticsT
|
||||||
// GetChainActivity get chain activity.
|
// GetChainActivity get chain activity.
|
||||||
func (s *Service) GetChainActivity(ctx context.Context, q *ChainActivityQuery) ([]ChainActivityResult, error) {
|
func (s *Service) GetChainActivity(ctx context.Context, q *ChainActivityQuery) ([]ChainActivityResult, error) {
|
||||||
key := fmt.Sprintf("%s:%s:%v:%s", chainActivityKey, q.TimeSpan, q.IsNotional, strings.Join(q.GetAppIDs(), ","))
|
key := fmt.Sprintf("%s:%s:%v:%s", chainActivityKey, q.TimeSpan, q.IsNotional, strings.Join(q.GetAppIDs(), ","))
|
||||||
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key,
|
return cacheable.GetOrLoad(ctx, s.logger, s.cache, s.expiration, key, s.metrics,
|
||||||
func() ([]ChainActivityResult, error) {
|
func() ([]ChainActivityResult, error) {
|
||||||
return s.repo.FindChainActivity(ctx, q)
|
return s.repo.FindChainActivity(ctx, q)
|
||||||
})
|
})
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
const serviceName = "wormscan-api"
|
||||||
|
|
||||||
|
type Metrics interface {
|
||||||
|
IncExpiredCacheResponse(key string)
|
||||||
|
}
|
|
@ -0,0 +1,32 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PrometheusMetrics is a Prometheus implementation of Metric interface.
|
||||||
|
type PrometheusMetrics struct {
|
||||||
|
expiredCacheResponseCount *prometheus.CounterVec
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPrometheusMetrics returns a new instance of PrometheusMetrics.
|
||||||
|
func NewPrometheusMetrics(environment string) *PrometheusMetrics {
|
||||||
|
vaaTxTrackerCount := promauto.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "expired_cache_response",
|
||||||
|
Help: "Total expired cache response by key",
|
||||||
|
ConstLabels: map[string]string{
|
||||||
|
"environment": environment,
|
||||||
|
"service": serviceName,
|
||||||
|
},
|
||||||
|
}, []string{"key"})
|
||||||
|
|
||||||
|
return &PrometheusMetrics{
|
||||||
|
expiredCacheResponseCount: vaaTxTrackerCount,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *PrometheusMetrics) IncExpiredCacheResponse(key string) {
|
||||||
|
m.expiredCacheResponseCount.WithLabelValues(key).Inc()
|
||||||
|
}
|
|
@ -35,6 +35,7 @@ import (
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/handlers/transactions"
|
"github.com/wormhole-foundation/wormhole-explorer/api/handlers/transactions"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/handlers/vaa"
|
"github.com/wormhole-foundation/wormhole-explorer/api/handlers/vaa"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/internal/config"
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/config"
|
||||||
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/metrics"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/internal/tvl"
|
"github.com/wormhole-foundation/wormhole-explorer/api/internal/tvl"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/middleware"
|
"github.com/wormhole-foundation/wormhole-explorer/api/middleware"
|
||||||
"github.com/wormhole-foundation/wormhole-explorer/api/response"
|
"github.com/wormhole-foundation/wormhole-explorer/api/response"
|
||||||
|
@ -160,19 +161,21 @@ func main() {
|
||||||
// create token provider
|
// create token provider
|
||||||
tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork)
|
tokenProvider := domain.NewTokenProvider(cfg.P2pNetwork)
|
||||||
|
|
||||||
|
metrics := metrics.NewPrometheusMetrics(cfg.Environment)
|
||||||
|
|
||||||
// Set up services
|
// Set up services
|
||||||
rootLogger.Info("initializing services")
|
rootLogger.Info("initializing services")
|
||||||
expirationTime := time.Duration(cfg.Cache.MetricExpiration) * time.Second
|
expirationTime := time.Duration(cfg.Cache.MetricExpiration) * time.Minute
|
||||||
addressService := address.NewService(addressRepo, rootLogger)
|
addressService := address.NewService(addressRepo, rootLogger)
|
||||||
vaaService := vaa.NewService(vaaRepo, cache.Get, vaaParserFunc, rootLogger)
|
vaaService := vaa.NewService(vaaRepo, cache.Get, vaaParserFunc, rootLogger)
|
||||||
obsService := observations.NewService(obsRepo, rootLogger)
|
obsService := observations.NewService(obsRepo, rootLogger)
|
||||||
governorService := governor.NewService(governorRepo, rootLogger)
|
governorService := governor.NewService(governorRepo, rootLogger)
|
||||||
infrastructureService := infrastructure.NewService(infrastructureRepo, rootLogger)
|
infrastructureService := infrastructure.NewService(infrastructureRepo, rootLogger)
|
||||||
heartbeatsService := heartbeats.NewService(heartbeatsRepo, rootLogger)
|
heartbeatsService := heartbeats.NewService(heartbeatsRepo, rootLogger)
|
||||||
transactionsService := transactions.NewService(transactionsRepo, cache, expirationTime, tokenProvider, rootLogger)
|
transactionsService := transactions.NewService(transactionsRepo, cache, expirationTime, tokenProvider, metrics, rootLogger)
|
||||||
relaysService := relays.NewService(relaysRepo, rootLogger)
|
relaysService := relays.NewService(relaysRepo, rootLogger)
|
||||||
operationsService := operations.NewService(operationsRepo, rootLogger)
|
operationsService := operations.NewService(operationsRepo, rootLogger)
|
||||||
statsService := stats.NewService(statsRepo, cache, expirationTime, rootLogger)
|
statsService := stats.NewService(statsRepo, cache, expirationTime, metrics, rootLogger)
|
||||||
|
|
||||||
// Set up a custom error handler
|
// Set up a custom error handler
|
||||||
response.SetEnableStackTrace(*cfg)
|
response.SetEnableStackTrace(*cfg)
|
||||||
|
|
Loading…
Reference in New Issue