node/pkg/solana: retry skipped slots only once
Adds extra logging to validate hypothesis that we've found a bug in GetConfirmedBlocks, rather than a general issue. Change-Id: Ia3acfd7fa393091a3892c453433255cfbe38e4a0
This commit is contained in:
parent
b771ad0e2a
commit
e7df95839d
|
@ -207,7 +207,7 @@ func (s *SolanaWatcher) Run(ctx context.Context) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SolanaWatcher) retryFetchBlock(ctx context.Context, logger *zap.Logger, slot uint64, retry uint) {
|
func (s *SolanaWatcher) retryFetchBlock(ctx context.Context, logger *zap.Logger, slot uint64, retry uint) {
|
||||||
ok := s.fetchBlock(ctx, logger, slot)
|
ok := s.fetchBlock(ctx, logger, slot, false)
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
if retry >= maxRetries {
|
if retry >= maxRetries {
|
||||||
|
@ -229,8 +229,11 @@ func (s *SolanaWatcher) retryFetchBlock(ctx context.Context, logger *zap.Logger,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SolanaWatcher) fetchBlock(ctx context.Context, logger *zap.Logger, slot uint64) bool {
|
func (s *SolanaWatcher) fetchBlock(ctx context.Context, logger *zap.Logger, slot uint64, emptyRetry bool) (ok bool) {
|
||||||
logger.Debug("requesting block", zap.Uint64("slot", slot), zap.String("commitment", string(s.commitment)))
|
logger.Debug("requesting block",
|
||||||
|
zap.Uint64("slot", slot),
|
||||||
|
zap.String("commitment", string(s.commitment)),
|
||||||
|
zap.Bool("is_empty_retry", emptyRetry))
|
||||||
rCtx, cancel := context.WithTimeout(ctx, rpcTimeout)
|
rCtx, cancel := context.WithTimeout(ctx, rpcTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
@ -247,7 +250,17 @@ func (s *SolanaWatcher) fetchBlock(ctx context.Context, logger *zap.Logger, slot
|
||||||
var rpcErr *jsonrpc.RPCError
|
var rpcErr *jsonrpc.RPCError
|
||||||
if errors.As(err, &rpcErr) && (rpcErr.Code == -32007 /* SLOT_SKIPPED */ || rpcErr.Code == -32004 /* BLOCK_NOT_AVAILABLE */) {
|
if errors.As(err, &rpcErr) && (rpcErr.Code == -32007 /* SLOT_SKIPPED */ || rpcErr.Code == -32004 /* BLOCK_NOT_AVAILABLE */) {
|
||||||
logger.Info("empty slot", zap.Uint64("slot", slot),
|
logger.Info("empty slot", zap.Uint64("slot", slot),
|
||||||
|
zap.Int("code", rpcErr.Code),
|
||||||
zap.String("commitment", string(s.commitment)))
|
zap.String("commitment", string(s.commitment)))
|
||||||
|
|
||||||
|
// Schedule a single retry just in case the Solana node was confused about the block being missing.
|
||||||
|
if !emptyRetry {
|
||||||
|
go func() {
|
||||||
|
time.Sleep(1 * time.Minute)
|
||||||
|
s.fetchBlock(ctx, logger, slot, true)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
return true
|
||||||
} else {
|
} else {
|
||||||
logger.Error("failed to request block", zap.Error(err), zap.Uint64("slot", slot),
|
logger.Error("failed to request block", zap.Error(err), zap.Uint64("slot", slot),
|
||||||
zap.String("commitment", string(s.commitment)))
|
zap.String("commitment", string(s.commitment)))
|
||||||
|
@ -348,6 +361,12 @@ OUTER:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if emptyRetry {
|
||||||
|
logger.Error("SOLANA BUG: skipped or unavailable block retrieved on second attempt",
|
||||||
|
zap.Uint64("slot", slot),
|
||||||
|
zap.String("commitment", string(s.commitment)))
|
||||||
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue