102 lines
3.1 KiB
Go
102 lines
3.1 KiB
Go
package near
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/certusone/wormhole/node/pkg/p2p"
|
|
"github.com/certusone/wormhole/node/pkg/supervisor"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
"github.com/wormhole-foundation/wormhole/sdk/vaa"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type eventType int
|
|
|
|
const (
|
|
EVENT_FINALIZED_CACHE_MISS eventType = iota
|
|
EVENT_NEAR_MESSAGE_CONFIRMED
|
|
EVENT_NEAR_API_HTTP_ERR // NEAR API returned a status code other than 200
|
|
EVENT_NEAR_WATCHER_TOO_FAR_BEHIND
|
|
)
|
|
|
|
func (e *Watcher) runMetrics(ctx context.Context) error {
|
|
logger := supervisor.Logger(ctx)
|
|
|
|
reg := prometheus.NewRegistry()
|
|
|
|
wormholeMsgCounter := 0
|
|
var wormholeMsgTotalTime time.Duration = 0
|
|
|
|
wormholeTxAvgDuration := promauto.With(reg).NewGauge(
|
|
prometheus.GaugeOpts{
|
|
Name: "wormhole_near_tx_avg_duration",
|
|
Help: "Average duration it takes for a wormhole message to be processed in milliseconds",
|
|
})
|
|
|
|
txqueueLen := promauto.With(reg).NewGauge(
|
|
prometheus.GaugeOpts{
|
|
Name: "wormhole_near_tx_queue",
|
|
Help: "Current Near transaction processing queue length",
|
|
})
|
|
|
|
chunkqueueLen := promauto.With(reg).NewGauge(
|
|
prometheus.GaugeOpts{
|
|
Name: "wormhole_near_chunk_queue",
|
|
Help: "Current Near chunk processing queue length",
|
|
})
|
|
|
|
nearMessagesConfirmed := promauto.With(reg).NewCounter(
|
|
prometheus.CounterOpts{
|
|
Name: "wormhole_near_observations_confirmed_total",
|
|
Help: "Total number of verified Near observations found",
|
|
})
|
|
|
|
nearFinalizedCacheMisses := promauto.With(reg).NewCounter(
|
|
prometheus.CounterOpts{
|
|
Name: "wormhole_near_finalized_cache_misses",
|
|
Help: "Number of times the watcher needed to check the finalization status of a block by walking the chain forward because the finalization status was not cached.",
|
|
})
|
|
|
|
nearRpcErrorCounter := promauto.With(reg).NewCounter(
|
|
prometheus.CounterOpts{
|
|
Name: "wormhole_near_rpc_error",
|
|
Help: "NEAR RPC Error Counter",
|
|
})
|
|
|
|
metricsIntervalTimer := time.NewTicker(metricsInterval) // this is just one ms for the first iteration.
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-metricsIntervalTimer.C:
|
|
// compute and publish periodic metrics
|
|
l1 := e.transactionProcessingQueueCounter.Load()
|
|
l2 := len(e.chunkProcessingQueue)
|
|
txqueueLen.Set(float64(l1))
|
|
chunkqueueLen.Set(float64(l2))
|
|
logger.Info("metrics", zap.Int64("txqueueLen", l1), zap.Int("chunkqueueLen", l2))
|
|
|
|
case event := <-e.eventChan:
|
|
switch event {
|
|
case EVENT_FINALIZED_CACHE_MISS:
|
|
nearFinalizedCacheMisses.Inc()
|
|
case EVENT_NEAR_MESSAGE_CONFIRMED:
|
|
nearMessagesConfirmed.Inc()
|
|
case EVENT_NEAR_WATCHER_TOO_FAR_BEHIND:
|
|
logger.Error("NEAR Watcher fell behind too far", zap.String("log_msg_type", "watcher_behind"))
|
|
p2p.DefaultRegistry.AddErrorCount(vaa.ChainIDNear, 1)
|
|
case EVENT_NEAR_API_HTTP_ERR:
|
|
nearRpcErrorCounter.Inc()
|
|
}
|
|
case d := <-e.eventChanTxProcessedDuration:
|
|
wormholeMsgCounter++
|
|
wormholeMsgTotalTime += d
|
|
avgDurationMs := wormholeMsgTotalTime.Milliseconds() / int64(wormholeMsgCounter)
|
|
wormholeTxAvgDuration.Set(float64(avgDurationMs))
|
|
}
|
|
}
|
|
}
|