From 24add74ebc63f07cf27b0ad0d3e4391e6fec917c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 08:45:55 +0100 Subject: [PATCH 01/32] wip --- src/grpc_subscription.rs | 5 +- src/grpc_subscription_autoreconnect.rs | 101 ++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/src/grpc_subscription.rs b/src/grpc_subscription.rs index 477d29f..e4059f4 100644 --- a/src/grpc_subscription.rs +++ b/src/grpc_subscription.rs @@ -3,12 +3,13 @@ // rpc_polling::vote_accounts_and_cluster_info_polling::poll_vote_accounts_and_cluster_info, // }; use anyhow::{bail, Context}; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; -use tokio::sync::broadcast::Sender; +use tokio::sync::broadcast::{Receiver, Sender}; use yellowstone_grpc_client::GeyserGrpcClient; +use yellowstone_grpc_proto::geyser::SubscribeRequest; use yellowstone_grpc_proto::prelude::{ subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequestFilterBlocks, SubscribeUpdateBlock, diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index 57a593f..84b6dbf 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -1,19 +1,25 @@ use async_stream::stream; use futures::{Stream, StreamExt}; -use log::{debug, info, log, trace, warn, Level}; +use log::{debug, info, log, trace, warn, Level, error}; use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::time::Duration; +use futures::channel::mpsc; +use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; use tokio::time::{sleep, timeout}; -use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult}; +use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; use yellowstone_grpc_proto::geyser::{ CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, }; use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; +use yellowstone_grpc_proto::tonic; +use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; +use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; use yellowstone_grpc_proto::tonic::Status; +use crate::grpc_subscription_autoreconnect::TheState::{Connected, FatalError, NotConnected, RecoverableConnectionError}; #[derive(Clone, Debug)] pub struct GrpcConnectionTimeouts { @@ -264,6 +270,97 @@ pub fn create_geyser_reconnecting_stream( the_stream } + +enum TheState>> { + NotConnected(Attempt), + RecoverableConnectionError(Attempt), + FatalError(Attempt), + Connected(Attempt, S), + +} + + +pub fn create_geyser_reconnecting_task( + grpc_source: GrpcSourceConfig, + subscribe_filter: SubscribeRequest, +) -> Receiver { + let (tx, rx) = tokio::sync::broadcast::channel::(1000); + + let geyser_task = tokio::spawn(async move { + let mut attempt = 1; + + let mut state = NotConnected(0); + + loop { + + state = match state { + NotConnected(_) => { + let addr = grpc_source.grpc_addr.clone(); + let token = grpc_source.grpc_x_token.clone(); + let config = grpc_source.tls_config.clone(); + let connect_timeout = grpc_source.timeouts.as_ref().map(|t| t.connect_timeout); + let request_timeout = grpc_source.timeouts.as_ref().map(|t| t.request_timeout); + let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); + let subscribe_filter = subscribe_filter.clone(); + log!(if attempt > 1 { Level::Warn } else { Level::Debug }, "Connecting attempt #{} to {}", attempt, addr); + let connect_result = GeyserGrpcClient::connect_with_timeout( + addr, token, config, + connect_timeout, + request_timeout, + false) + .await; + let mut client = connect_result?; + + debug!("Subscribe with filter {:?}", subscribe_filter); + + let subscribe_result = timeout(subscribe_timeout.unwrap_or(Duration::MAX), + client + .subscribe_once2(subscribe_filter)) + .await; + + // maybe not optimal + let subscribe_result = subscribe_result.map_err(|_| Status::unknown("unspecific subscribe timeout"))?; + + match subscribe_result { + Ok(geyser_stream) => { + Connected(geyser_stream) + } + Err(GeyserGrpcClientError::TonicError(_)) => { + warn!("! subscribe failed on {} - retrying", grpc_source); + RecoverableConnectionError(attempt) + } + Err(GeyserGrpcClientError::TonicStatus(_)) => { + warn!("! subscribe failed on {} - retrying", grpc_source); + RecoverableConnectionError(attempt) + } + // non-recoverable + Err(unrecoverable_error) => { + error!("! subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error); + FatalError(attempt) + } + } + } + RecoverableConnectionError(attempt) => { + let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); + info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); + sleep(Duration::from_secs_f32(backoff_secs)).await; + } + FatalError(_) => { + // TOOD what to do + panic!("Fatal error") + } + } + + } + + }); + + + rx +} + + + #[cfg(test)] mod tests { use super::*; From fa05a1bd5fb1531b6819c55665bc8f2663e7f441 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 10:09:32 +0100 Subject: [PATCH 02/32] add connection state --- examples/stream_blocks_single.rs | 36 +++++++++++++++++-------- src/grpc_subscription_autoreconnect.rs | 37 +++++++++++++++++++++----- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index 2d7ee51..f1d01f9 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -5,9 +5,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::grpc_subscription_autoreconnect::{ - create_geyser_reconnecting_stream, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, -}; +use geyser_grpc_connector::grpc_subscription_autoreconnect::{create_geyser_reconnecting_stream, create_geyser_reconnecting_task, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; @@ -92,16 +90,32 @@ pub async fn main() { GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone()); info!("Write Block stream.."); - let green_stream = create_geyser_reconnecting_stream( + + let (jh_geyser_task, mut green_stream) = create_geyser_reconnecting_task( green_config.clone(), - GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(), - // GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), + GeyserFilter(CommitmentConfig::confirmed()).blocks(), ); - let multiplex_stream = create_multiplexed_stream( - vec![green_stream], - BlockMiniExtractor(CommitmentConfig::confirmed()), - ); - start_example_blockmini_consumer(multiplex_stream); + + tokio::spawn(async move { + while let Some(mini) = green_stream.recv().await { + info!( + "emitted block mini #{}@{} with {} bytes from multiplexer", + mini.slot, mini.commitment_config.commitment, mini.blocksize + ); + } + }); + + + // let green_stream = create_geyser_reconnecting_stream( + // green_config.clone(), + // GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(), + // // GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), + // ); + // let multiplex_stream = create_multiplexed_stream( + // vec![green_stream], + // BlockMiniExtractor(CommitmentConfig::confirmed()), + // ); + // start_example_blockmini_consumer(multiplex_stream); // "infinite" sleep sleep(Duration::from_secs(1800)).await; diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index 84b6dbf..d3adfae 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -13,6 +13,7 @@ use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrp use yellowstone_grpc_proto::geyser::{ CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, }; +use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; use yellowstone_grpc_proto::tonic; use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; @@ -80,6 +81,8 @@ impl GrpcSourceConfig { type Attempt = u32; // wraps payload and status messages +// clone is required by broacast channel +#[derive(Clone)] pub enum Message { GeyserSubscribeUpdate(Box), // connect (attempt=1) or reconnect(attempt=2..) @@ -283,18 +286,18 @@ enum TheState>> { pub fn create_geyser_reconnecting_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, -) -> Receiver { +) -> (JoinHandle<()>, Receiver) { let (tx, rx) = tokio::sync::broadcast::channel::(1000); - let geyser_task = tokio::spawn(async move { - let mut attempt = 1; - + let jh_geyser_task = tokio::spawn(async move { let mut state = NotConnected(0); loop { state = match state { - NotConnected(_) => { + NotConnected(mut attempt) => { + attempt += 1; + let addr = grpc_source.grpc_addr.clone(); let token = grpc_source.grpc_x_token.clone(); let config = grpc_source.tls_config.clone(); @@ -309,6 +312,7 @@ pub fn create_geyser_reconnecting_task( request_timeout, false) .await; + let mut client = connect_result?; debug!("Subscribe with filter {:?}", subscribe_filter); @@ -323,7 +327,7 @@ pub fn create_geyser_reconnecting_task( match subscribe_result { Ok(geyser_stream) => { - Connected(geyser_stream) + Connected(attempt, geyser_stream) } Err(GeyserGrpcClientError::TonicError(_)) => { warn!("! subscribe failed on {} - retrying", grpc_source); @@ -344,11 +348,30 @@ pub fn create_geyser_reconnecting_task( let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); sleep(Duration::from_secs_f32(backoff_secs)).await; + NotConnected(attempt) } FatalError(_) => { // TOOD what to do panic!("Fatal error") } + Connected(attempt, mut geyser_stream) => { + match geyser_stream.next().await { + Some(Ok(update_message)) => { + trace!("> recv update message from {}", grpc_source); + (ConnectionState::Ready(attempt, geyser_stream), Message::GeyserSubscribeUpdate(Box::new(update_message))) + } + Some(Err(tonic_status)) => { + // ATM we consider all errors recoverable + warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); + (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + } + None => { + // should not arrive here, Mean the stream close. + warn!("geyser stream closed on {} - retrying", grpc_source); + (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + } + } + } } } @@ -356,7 +379,7 @@ pub fn create_geyser_reconnecting_task( }); - rx + (jh_geyser_task, rx) } From c9144ee39f03348b009ffa8fd57e00e203d01947 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 10:57:04 +0100 Subject: [PATCH 03/32] compiles --- src/grpc_subscription_autoreconnect.rs | 64 ++++++++++++++++++++------ 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index d3adfae..f66c8f3 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -4,11 +4,13 @@ use log::{debug, info, log, trace, warn, Level, error}; use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; +use std::pin::Pin; use std::time::Duration; use futures::channel::mpsc; use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; -use tokio::time::{sleep, timeout}; +use tokio::time::{sleep, timeout, Timeout}; +use tokio::time::error::Elapsed; use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; use yellowstone_grpc_proto::geyser::{ CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, @@ -18,9 +20,10 @@ use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; use yellowstone_grpc_proto::tonic; use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; +use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; use yellowstone_grpc_proto::tonic::Status; -use crate::grpc_subscription_autoreconnect::TheState::{Connected, FatalError, NotConnected, RecoverableConnectionError}; +use crate::grpc_subscription_autoreconnect::TheState::*; #[derive(Clone, Debug)] pub struct GrpcConnectionTimeouts { @@ -274,12 +277,15 @@ pub fn create_geyser_reconnecting_stream( } -enum TheState>> { +enum TheState>, F: Interceptor> { NotConnected(Attempt), + // Connected(Attempt, Box>>), + Connected(Attempt, GeyserGrpcClient), + Ready(Attempt, S), + // error states RecoverableConnectionError(Attempt), FatalError(Attempt), - Connected(Attempt, S), - + WaitReconnect(Attempt), } @@ -313,21 +319,43 @@ pub fn create_geyser_reconnecting_task( false) .await; - let mut client = connect_result?; + match connect_result { + Ok(client) => { + Connected(attempt, client) + } + Err(_) => { + todo!() + } + } + + + } + Connected(attempt, mut client) => { + let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); + let subscribe_filter = subscribe_filter.clone(); debug!("Subscribe with filter {:?}", subscribe_filter); - let subscribe_result = timeout(subscribe_timeout.unwrap_or(Duration::MAX), - client - .subscribe_once2(subscribe_filter)) + let subscribe_result_timeout = + timeout(subscribe_timeout.unwrap_or(Duration::MAX), + client.subscribe_once2(subscribe_filter)) .await; + let subscribe_result; + match subscribe_result_timeout.map_err(|_| Status::unknown("unspecific subscribe timeout")) { + Ok(fooo) => { + subscribe_result = fooo; + } + Err(_elapsed) => { + todo!() + } + } // maybe not optimal - let subscribe_result = subscribe_result.map_err(|_| Status::unknown("unspecific subscribe timeout"))?; + // let subscribe_result = subscribe_result_timeout.map_err(|_| Status::unknown("unspecific subscribe timeout")); match subscribe_result { Ok(geyser_stream) => { - Connected(attempt, geyser_stream) + Ready(attempt, geyser_stream) } Err(GeyserGrpcClientError::TonicError(_)) => { warn!("! subscribe failed on {} - retrying", grpc_source); @@ -354,21 +382,27 @@ pub fn create_geyser_reconnecting_task( // TOOD what to do panic!("Fatal error") } - Connected(attempt, mut geyser_stream) => { + TheState::WaitReconnect(attempt) => { + let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); + info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); + sleep(Duration::from_secs_f32(backoff_secs)).await; + TheState::NotConnected(attempt) + } + Ready(attempt, mut geyser_stream) => { match geyser_stream.next().await { Some(Ok(update_message)) => { trace!("> recv update message from {}", grpc_source); - (ConnectionState::Ready(attempt, geyser_stream), Message::GeyserSubscribeUpdate(Box::new(update_message))) + TheState::Ready(attempt, geyser_stream) } Some(Err(tonic_status)) => { // ATM we consider all errors recoverable warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); - (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + TheState::WaitReconnect(attempt) } None => { // should not arrive here, Mean the stream close. warn!("geyser stream closed on {} - retrying", grpc_source); - (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + TheState::WaitReconnect(attempt) } } } From bbba7b1fea50422d891ca5d8fe3aac0b4d3980ff Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 11:11:46 +0100 Subject: [PATCH 04/32] send works now --- src/grpc_subscription_autoreconnect.rs | 47 +++++++++++++++++--------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index f66c8f3..dfad979 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -7,6 +7,7 @@ use std::fmt::{Debug, Display}; use std::pin::Pin; use std::time::Duration; use futures::channel::mpsc; +use tokio::sync::broadcast::error::SendError; use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; use tokio::time::{sleep, timeout, Timeout}; @@ -293,7 +294,7 @@ pub fn create_geyser_reconnecting_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, ) -> (JoinHandle<()>, Receiver) { - let (tx, rx) = tokio::sync::broadcast::channel::(1000); + let (sender, receiver_stream) = tokio::sync::broadcast::channel::(1000); let jh_geyser_task = tokio::spawn(async move { let mut state = NotConnected(0); @@ -389,20 +390,34 @@ pub fn create_geyser_reconnecting_task( TheState::NotConnected(attempt) } Ready(attempt, mut geyser_stream) => { - match geyser_stream.next().await { - Some(Ok(update_message)) => { - trace!("> recv update message from {}", grpc_source); - TheState::Ready(attempt, geyser_stream) - } - Some(Err(tonic_status)) => { - // ATM we consider all errors recoverable - warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); - TheState::WaitReconnect(attempt) - } - None => { - // should not arrive here, Mean the stream close. - warn!("geyser stream closed on {} - retrying", grpc_source); - TheState::WaitReconnect(attempt) + 'recv_loop: loop { + match geyser_stream.next().await { + Some(Ok(update_message)) => { + trace!("> recv update message from {}", grpc_source); + match sender.send(Message::GeyserSubscribeUpdate(Box::new(update_message))) { + Ok(n_subscribers) => { + trace!("sent update message to {} subscribers (buffer={})", + n_subscribers, + sender.len()); + continue 'recv_loop; + } + Err(SendError(_)) => { + // note: error does not mean that future sends will also fail! + trace!("no subscribers for update message"); + continue 'recv_loop; + } + }; + } + Some(Err(tonic_status)) => { + // ATM we consider all errors recoverable + warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); + break 'recv_loop TheState::WaitReconnect(attempt); + } + None => { + // should not arrive here, Mean the stream close. + warn!("geyser stream closed on {} - retrying", grpc_source); + break 'recv_loop TheState::WaitReconnect(attempt); + } } } } @@ -413,7 +428,7 @@ pub fn create_geyser_reconnecting_task( }); - (jh_geyser_task, rx) + (jh_geyser_task, receiver_stream) } From e093a589a2ddf13b218344433995a0676551962e Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 11:38:45 +0100 Subject: [PATCH 05/32] check tonic error --- src/grpc_subscription_autoreconnect.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index dfad979..890aa74 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -23,7 +23,7 @@ use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; -use yellowstone_grpc_proto::tonic::Status; +use yellowstone_grpc_proto::tonic::{Code, Status}; use crate::grpc_subscription_autoreconnect::TheState::*; #[derive(Clone, Debug)] @@ -409,17 +409,16 @@ pub fn create_geyser_reconnecting_task( }; } Some(Err(tonic_status)) => { - // ATM we consider all errors recoverable + // all tonic errors are recoverable warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); break 'recv_loop TheState::WaitReconnect(attempt); } None => { - // should not arrive here, Mean the stream close. warn!("geyser stream closed on {} - retrying", grpc_source); break 'recv_loop TheState::WaitReconnect(attempt); } } - } + } // -- end loop } } @@ -479,3 +478,4 @@ mod tests { ); } } + From def0853c0ff2dbd3d13634ca8184713c5112a65c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 13:30:49 +0100 Subject: [PATCH 06/32] more cases on connect --- src/grpc_subscription_autoreconnect.rs | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index 890aa74..1dfe9f5 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -310,8 +310,6 @@ pub fn create_geyser_reconnecting_task( let config = grpc_source.tls_config.clone(); let connect_timeout = grpc_source.timeouts.as_ref().map(|t| t.connect_timeout); let request_timeout = grpc_source.timeouts.as_ref().map(|t| t.request_timeout); - let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); - let subscribe_filter = subscribe_filter.clone(); log!(if attempt > 1 { Level::Warn } else { Level::Debug }, "Connecting attempt #{} to {}", attempt, addr); let connect_result = GeyserGrpcClient::connect_with_timeout( addr, token, config, @@ -320,17 +318,32 @@ pub fn create_geyser_reconnecting_task( false) .await; - match connect_result { Ok(client) => { Connected(attempt, client) } - Err(_) => { - todo!() + Err(GeyserGrpcClientError::InvalidUri(_)) => { + FatalError(attempt) + } + Err(GeyserGrpcClientError::MetadataValueError(_)) => { + FatalError(attempt) + } + Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => { + FatalError(attempt) + } + Err(GeyserGrpcClientError::TonicError(tonic_error)) => { + warn!("! connect failed on {} - aborting: {:?}", grpc_source, tonic_error); + FatalError(attempt) + } + Err(GeyserGrpcClientError::TonicStatus(tonic_status)) => { + warn!("! connect failed on {} - retrying: {:?}", grpc_source, tonic_status); + RecoverableConnectionError(attempt) + } + Err(GeyserGrpcClientError::SubscribeSendError(send_error)) => { + warn!("! connect failed with send error on {} - retrying: {:?}", grpc_source, send_error); + RecoverableConnectionError(attempt) } } - - } Connected(attempt, mut client) => { let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); From ca55a8bba82abac2ee56468fbb28d0bb3d51401f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 13:38:15 +0100 Subject: [PATCH 07/32] handle connect problems --- src/grpc_subscription_autoreconnect.rs | 46 ++++++++++++-------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect.rs index 1dfe9f5..28cc2ed 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect.rs @@ -355,35 +355,31 @@ pub fn create_geyser_reconnecting_task( client.subscribe_once2(subscribe_filter)) .await; - let subscribe_result; - match subscribe_result_timeout.map_err(|_| Status::unknown("unspecific subscribe timeout")) { - Ok(fooo) => { - subscribe_result = fooo; + match subscribe_result_timeout { + Ok(subscribe_result) => { + match subscribe_result { + Ok(geyser_stream) => { + Ready(attempt, geyser_stream) + } + Err(GeyserGrpcClientError::TonicError(_)) => { + warn!("! subscribe failed on {} - retrying", grpc_source); + RecoverableConnectionError(attempt) + } + Err(GeyserGrpcClientError::TonicStatus(_)) => { + warn!("! subscribe failed on {} - retrying", grpc_source); + RecoverableConnectionError(attempt) + } + // non-recoverable + Err(unrecoverable_error) => { + error!("! subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error); + FatalError(attempt) + } + } } Err(_elapsed) => { - todo!() - } - } - // maybe not optimal - // let subscribe_result = subscribe_result_timeout.map_err(|_| Status::unknown("unspecific subscribe timeout")); - - match subscribe_result { - Ok(geyser_stream) => { - Ready(attempt, geyser_stream) - } - Err(GeyserGrpcClientError::TonicError(_)) => { - warn!("! subscribe failed on {} - retrying", grpc_source); + warn!("! subscribe failed with timeout on {} - retrying", grpc_source); RecoverableConnectionError(attempt) } - Err(GeyserGrpcClientError::TonicStatus(_)) => { - warn!("! subscribe failed on {} - retrying", grpc_source); - RecoverableConnectionError(attempt) - } - // non-recoverable - Err(unrecoverable_error) => { - error!("! subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error); - FatalError(attempt) - } } } RecoverableConnectionError(attempt) => { From 33cb8cbfa729624d2f811997581573d38ac217cf Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 18 Jan 2024 13:45:19 +0100 Subject: [PATCH 08/32] split impl --- examples/stream_blocks_mainnet.rs | 5 +- examples/stream_blocks_single.rs | 28 +- ...grpc_subscription_autoreconnect_streams.rs | 270 +++++++++++++++++ ... grpc_subscription_autoreconnect_tasks.rs} | 285 +++++------------- src/grpcmultiplex_fastestwins.rs | 4 +- src/lib.rs | 64 +++- 6 files changed, 428 insertions(+), 228 deletions(-) create mode 100644 src/grpc_subscription_autoreconnect_streams.rs rename src/{grpc_subscription_autoreconnect.rs => grpc_subscription_autoreconnect_tasks.rs} (56%) diff --git a/examples/stream_blocks_mainnet.rs b/examples/stream_blocks_mainnet.rs index 73440ea..d3c3326 100644 --- a/examples/stream_blocks_mainnet.rs +++ b/examples/stream_blocks_mainnet.rs @@ -21,8 +21,8 @@ use solana_sdk::signature::Signature; use solana_sdk::transaction::TransactionError; use yellowstone_grpc_proto::geyser::SubscribeUpdateBlock; -use geyser_grpc_connector::grpc_subscription_autoreconnect::{ - create_geyser_reconnecting_stream, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ + create_geyser_reconnecting_stream, GeyserFilter, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, @@ -30,6 +30,7 @@ use geyser_grpc_connector::grpcmultiplex_fastestwins::{ use tokio::time::{sleep, Duration}; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; +use geyser_grpc_connector::{GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_block_consumer( multiplex_stream: impl Stream + Send + 'static, diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index f1d01f9..ecd59bb 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -5,14 +5,19 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::grpc_subscription_autoreconnect::{create_geyser_reconnecting_stream, create_geyser_reconnecting_task, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ + create_geyser_reconnecting_stream, GeyserFilter, +}; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; use tokio::time::{sleep, Duration}; +use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; -use yellowstone_grpc_proto::prost::Message; +use yellowstone_grpc_proto::prost::Message as _; +use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{create_geyser_reconnecting_task, Message}; +use geyser_grpc_connector::{GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_blockmini_consumer( multiplex_stream: impl Stream + Send + 'static, @@ -93,19 +98,24 @@ pub async fn main() { let (jh_geyser_task, mut green_stream) = create_geyser_reconnecting_task( green_config.clone(), - GeyserFilter(CommitmentConfig::confirmed()).blocks(), + GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); tokio::spawn(async move { - while let Some(mini) = green_stream.recv().await { - info!( - "emitted block mini #{}@{} with {} bytes from multiplexer", - mini.slot, mini.commitment_config.commitment, mini.blocksize - ); + while let Ok(message) = green_stream.recv().await { + match message { + Message::GeyserSubscribeUpdate(subscriber_update) => { + // info!("got update: {:?}", subscriber_update.update_oneof.); + info!("got update!!!"); + } + Message::Connecting(attempt) => { + warn!("Connection attempt: {}", attempt); + } + } } + warn!("Stream aborted"); }); - // let green_stream = create_geyser_reconnecting_stream( // green_config.clone(), // GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(), diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs new file mode 100644 index 0000000..687184d --- /dev/null +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -0,0 +1,270 @@ +use async_stream::stream; +use futures::channel::mpsc; +use futures::{Stream, StreamExt}; +use log::{debug, error, info, log, trace, warn, Level}; +use solana_sdk::commitment_config::CommitmentConfig; +use std::collections::HashMap; +use std::fmt::{Debug, Display}; +use std::pin::Pin; +use std::time::Duration; +use tokio::sync::broadcast::error::SendError; +use tokio::sync::broadcast::Receiver; +use tokio::task::JoinHandle; +use tokio::time::error::Elapsed; +use tokio::time::{sleep, timeout, Timeout}; +use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; +use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; +use yellowstone_grpc_proto::geyser::{ + CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, +}; +use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; +use yellowstone_grpc_proto::tonic; +use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; +use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; +use yellowstone_grpc_proto::tonic::service::Interceptor; +use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; +use yellowstone_grpc_proto::tonic::{Code, Status}; +use crate::GrpcSourceConfig; + +type Attempt = u32; + +// wraps payload and status messages +// clone is required by broacast channel +#[derive(Clone)] +pub enum Message { + GeyserSubscribeUpdate(Box), + // connect (attempt=1) or reconnect(attempt=2..) + Connecting(Attempt), +} + +enum ConnectionState>> { + NotConnected(Attempt), + Connecting(Attempt, JoinHandle>), + Ready(Attempt, S), + WaitReconnect(Attempt), +} + +#[derive(Clone)] +pub struct GeyserFilter(pub CommitmentConfig); + +impl GeyserFilter { + pub fn blocks_and_txs(&self) -> SubscribeRequest { + let mut blocks_subs = HashMap::new(); + blocks_subs.insert( + "client".to_string(), + SubscribeRequestFilterBlocks { + account_include: Default::default(), + include_transactions: Some(true), + include_accounts: Some(false), + include_entries: Some(false), + }, + ); + + SubscribeRequest { + slots: HashMap::new(), + accounts: Default::default(), + transactions: HashMap::new(), + entry: Default::default(), + blocks: blocks_subs, + blocks_meta: HashMap::new(), + commitment: Some(map_commitment_level(self.0) as i32), + accounts_data_slice: Default::default(), + ping: None, + } + } + + pub fn blocks_meta(&self) -> SubscribeRequest { + let mut blocksmeta_subs = HashMap::new(); + blocksmeta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {}); + + SubscribeRequest { + slots: HashMap::new(), + accounts: Default::default(), + transactions: HashMap::new(), + entry: Default::default(), + blocks: HashMap::new(), + blocks_meta: blocksmeta_subs, + commitment: Some(map_commitment_level(self.0) as i32), + accounts_data_slice: Default::default(), + ping: None, + } + } +} + +fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel { + // solana_sdk -> yellowstone + match commitment_config.commitment { + solana_sdk::commitment_config::CommitmentLevel::Processed => { + yellowstone_grpc_proto::prelude::CommitmentLevel::Processed + } + solana_sdk::commitment_config::CommitmentLevel::Confirmed => { + yellowstone_grpc_proto::prelude::CommitmentLevel::Confirmed + } + solana_sdk::commitment_config::CommitmentLevel::Finalized => { + yellowstone_grpc_proto::prelude::CommitmentLevel::Finalized + } + _ => { + panic!( + "unsupported commitment level {}", + commitment_config.commitment + ) + } + } +} + +// Take geyser filter, connect to Geyser and return a generic stream of SubscribeUpdate +// note: stream never terminates +pub fn create_geyser_reconnecting_stream( + grpc_source: GrpcSourceConfig, + subscribe_filter: SubscribeRequest, +) -> impl Stream { + let mut state = ConnectionState::NotConnected(0); + + // in case of cancellation, we restart from here: + // thus we want to keep the progression in a state object outside the stream! makro + let the_stream = stream! { + loop { + let yield_value; + + (state, yield_value) = match state { + + ConnectionState::NotConnected(mut attempt) => { + attempt += 1; + + let connection_task = tokio::spawn({ + let addr = grpc_source.grpc_addr.clone(); + let token = grpc_source.grpc_x_token.clone(); + let config = grpc_source.tls_config.clone(); + let connect_timeout = grpc_source.timeouts.as_ref().map(|t| t.connect_timeout); + let request_timeout = grpc_source.timeouts.as_ref().map(|t| t.request_timeout); + let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); + let subscribe_filter = subscribe_filter.clone(); + log!(if attempt > 1 { Level::Warn } else { Level::Debug }, "Connecting attempt #{} to {}", attempt, addr); + async move { + + let connect_result = GeyserGrpcClient::connect_with_timeout( + addr, token, config, + connect_timeout, + request_timeout, + false) + .await; + let mut client = connect_result?; + + + debug!("Subscribe with filter {:?}", subscribe_filter); + + let subscribe_result = timeout(subscribe_timeout.unwrap_or(Duration::MAX), + client + .subscribe_once2(subscribe_filter)) + .await; + + // maybe not optimal + subscribe_result.map_err(|_| Status::unknown("unspecific subscribe timeout"))? + } + }); + + (ConnectionState::Connecting(attempt, connection_task), Message::Connecting(attempt)) + } + + ConnectionState::Connecting(attempt, connection_task) => { + let subscribe_result = connection_task.await; + + match subscribe_result { + Ok(Ok(subscribed_stream)) => (ConnectionState::Ready(attempt, subscribed_stream), Message::Connecting(attempt)), + Ok(Err(geyser_error)) => { + // ATM we consider all errors recoverable + warn!("! subscribe failed on {} - retrying: {:?}", grpc_source, geyser_error); + (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + }, + Err(geyser_grpc_task_error) => { + panic!("! task aborted - should not happen :{geyser_grpc_task_error}"); + } + } + + } + + ConnectionState::Ready(attempt, mut geyser_stream) => { + + match geyser_stream.next().await { + Some(Ok(update_message)) => { + trace!("> recv update message from {}", grpc_source); + (ConnectionState::Ready(attempt, geyser_stream), Message::GeyserSubscribeUpdate(Box::new(update_message))) + } + Some(Err(tonic_status)) => { + // ATM we consider all errors recoverable + warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); + (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + } + None => { + // should not arrive here, Mean the stream close. + warn!("geyser stream closed on {} - retrying", grpc_source); + (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + } + } + + } + + ConnectionState::WaitReconnect(attempt) => { + let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); + info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); + sleep(Duration::from_secs_f32(backoff_secs)).await; + (ConnectionState::NotConnected(attempt), Message::Connecting(attempt)) + } + + }; // -- match + + yield yield_value + } + + }; // -- stream! + + the_stream +} + +#[cfg(test)] +mod tests { + use crate::GrpcConnectionTimeouts; + use super::*; + + #[tokio::test] + async fn test_debug_no_secrets() { + let timeout_config = GrpcConnectionTimeouts { + connect_timeout: Duration::from_secs(1), + request_timeout: Duration::from_secs(2), + subscribe_timeout: Duration::from_secs(3), + }; + assert_eq!( + format!( + "{:?}", + GrpcSourceConfig::new( + "http://localhost:1234".to_string(), + Some("my-secret".to_string()), + None, + timeout_config + ) + ), + "grpc_addr http://localhost:1234" + ); + } + + #[tokio::test] + async fn test_display_no_secrets() { + let timeout_config = GrpcConnectionTimeouts { + connect_timeout: Duration::from_secs(1), + request_timeout: Duration::from_secs(2), + subscribe_timeout: Duration::from_secs(3), + }; + assert_eq!( + format!( + "{}", + GrpcSourceConfig::new( + "http://localhost:1234".to_string(), + Some("my-secret".to_string()), + None, + timeout_config + ) + ), + "grpc_addr http://localhost:1234" + ); + } +} diff --git a/src/grpc_subscription_autoreconnect.rs b/src/grpc_subscription_autoreconnect_tasks.rs similarity index 56% rename from src/grpc_subscription_autoreconnect.rs rename to src/grpc_subscription_autoreconnect_tasks.rs index 28cc2ed..91e3986 100644 --- a/src/grpc_subscription_autoreconnect.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,22 +1,23 @@ +use crate::grpc_subscription_autoreconnect_tasks::TheState::*; use async_stream::stream; +use futures::channel::mpsc; use futures::{Stream, StreamExt}; -use log::{debug, info, log, trace, warn, Level, error}; +use log::{debug, error, info, log, trace, warn, Level}; use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::pin::Pin; use std::time::Duration; -use futures::channel::mpsc; use tokio::sync::broadcast::error::SendError; use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; -use tokio::time::{sleep, timeout, Timeout}; use tokio::time::error::Elapsed; +use tokio::time::{sleep, timeout, Timeout}; use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; +use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::{ CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, }; -use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; use yellowstone_grpc_proto::tonic; use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; @@ -24,63 +25,7 @@ use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; use yellowstone_grpc_proto::tonic::{Code, Status}; -use crate::grpc_subscription_autoreconnect::TheState::*; - -#[derive(Clone, Debug)] -pub struct GrpcConnectionTimeouts { - pub connect_timeout: Duration, - pub request_timeout: Duration, - pub subscribe_timeout: Duration, -} - -#[derive(Clone)] -pub struct GrpcSourceConfig { - grpc_addr: String, - grpc_x_token: Option, - tls_config: Option, - timeouts: Option, -} - -impl Display for GrpcSourceConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "grpc_addr {}", - crate::obfuscate::url_obfuscate_api_token(&self.grpc_addr) - ) - } -} - -impl Debug for GrpcSourceConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self, f) - } -} - -impl GrpcSourceConfig { - /// Create a grpc source without tls and timeouts - pub fn new_simple(grpc_addr: String) -> Self { - Self { - grpc_addr, - grpc_x_token: None, - tls_config: None, - timeouts: None, - } - } - pub fn new( - grpc_addr: String, - grpc_x_token: Option, - tls_config: Option, - timeouts: GrpcConnectionTimeouts, - ) -> Self { - Self { - grpc_addr, - grpc_x_token, - tls_config, - timeouts: Some(timeouts), - } - } -} +use crate::GrpcSourceConfig; type Attempt = u32; @@ -168,116 +113,6 @@ fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel } } -// Take geyser filter, connect to Geyser and return a generic stream of SubscribeUpdate -// note: stream never terminates -pub fn create_geyser_reconnecting_stream( - grpc_source: GrpcSourceConfig, - subscribe_filter: SubscribeRequest, -) -> impl Stream { - let mut state = ConnectionState::NotConnected(0); - - // in case of cancellation, we restart from here: - // thus we want to keep the progression in a state object outside the stream! makro - let the_stream = stream! { - loop { - let yield_value; - - (state, yield_value) = match state { - - ConnectionState::NotConnected(mut attempt) => { - attempt += 1; - - let connection_task = tokio::spawn({ - let addr = grpc_source.grpc_addr.clone(); - let token = grpc_source.grpc_x_token.clone(); - let config = grpc_source.tls_config.clone(); - let connect_timeout = grpc_source.timeouts.as_ref().map(|t| t.connect_timeout); - let request_timeout = grpc_source.timeouts.as_ref().map(|t| t.request_timeout); - let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); - let subscribe_filter = subscribe_filter.clone(); - log!(if attempt > 1 { Level::Warn } else { Level::Debug }, "Connecting attempt #{} to {}", attempt, addr); - async move { - - let connect_result = GeyserGrpcClient::connect_with_timeout( - addr, token, config, - connect_timeout, - request_timeout, - false) - .await; - let mut client = connect_result?; - - - debug!("Subscribe with filter {:?}", subscribe_filter); - - let subscribe_result = timeout(subscribe_timeout.unwrap_or(Duration::MAX), - client - .subscribe_once2(subscribe_filter)) - .await; - - // maybe not optimal - subscribe_result.map_err(|_| Status::unknown("unspecific subscribe timeout"))? - } - }); - - (ConnectionState::Connecting(attempt, connection_task), Message::Connecting(attempt)) - } - - ConnectionState::Connecting(attempt, connection_task) => { - let subscribe_result = connection_task.await; - - match subscribe_result { - Ok(Ok(subscribed_stream)) => (ConnectionState::Ready(attempt, subscribed_stream), Message::Connecting(attempt)), - Ok(Err(geyser_error)) => { - // ATM we consider all errors recoverable - warn!("! subscribe failed on {} - retrying: {:?}", grpc_source, geyser_error); - (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) - }, - Err(geyser_grpc_task_error) => { - panic!("! task aborted - should not happen :{geyser_grpc_task_error}"); - } - } - - } - - ConnectionState::Ready(attempt, mut geyser_stream) => { - - match geyser_stream.next().await { - Some(Ok(update_message)) => { - trace!("> recv update message from {}", grpc_source); - (ConnectionState::Ready(attempt, geyser_stream), Message::GeyserSubscribeUpdate(Box::new(update_message))) - } - Some(Err(tonic_status)) => { - // ATM we consider all errors recoverable - warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); - (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) - } - None => { - // should not arrive here, Mean the stream close. - warn!("geyser stream closed on {} - retrying", grpc_source); - (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) - } - } - - } - - ConnectionState::WaitReconnect(attempt) => { - let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); - info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); - sleep(Duration::from_secs_f32(backoff_secs)).await; - (ConnectionState::NotConnected(attempt), Message::Connecting(attempt)) - } - - }; // -- match - - yield yield_value - } - - }; // -- stream! - - the_stream -} - - enum TheState>, F: Interceptor> { NotConnected(Attempt), // Connected(Attempt, Box>>), @@ -289,7 +124,6 @@ enum TheState>, F: Interceptor> WaitReconnect(Attempt), } - pub fn create_geyser_reconnecting_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, @@ -300,7 +134,6 @@ pub fn create_geyser_reconnecting_task( let mut state = NotConnected(0); loop { - state = match state { NotConnected(mut attempt) => { attempt += 1; @@ -310,57 +143,70 @@ pub fn create_geyser_reconnecting_task( let config = grpc_source.tls_config.clone(); let connect_timeout = grpc_source.timeouts.as_ref().map(|t| t.connect_timeout); let request_timeout = grpc_source.timeouts.as_ref().map(|t| t.request_timeout); - log!(if attempt > 1 { Level::Warn } else { Level::Debug }, "Connecting attempt #{} to {}", attempt, addr); + log!( + if attempt > 1 { + Level::Warn + } else { + Level::Debug + }, + "Connecting attempt #{} to {}", + attempt, + addr + ); let connect_result = GeyserGrpcClient::connect_with_timeout( - addr, token, config, + addr, + token, + config, connect_timeout, request_timeout, - false) - .await; + false, + ) + .await; match connect_result { - Ok(client) => { - Connected(attempt, client) - } - Err(GeyserGrpcClientError::InvalidUri(_)) => { - FatalError(attempt) - } - Err(GeyserGrpcClientError::MetadataValueError(_)) => { - FatalError(attempt) - } - Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => { - FatalError(attempt) - } + Ok(client) => Connected(attempt, client), + Err(GeyserGrpcClientError::InvalidUri(_)) => FatalError(attempt), + Err(GeyserGrpcClientError::MetadataValueError(_)) => FatalError(attempt), + Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => FatalError(attempt), Err(GeyserGrpcClientError::TonicError(tonic_error)) => { - warn!("! connect failed on {} - aborting: {:?}", grpc_source, tonic_error); + warn!( + "! connect failed on {} - aborting: {:?}", + grpc_source, tonic_error + ); FatalError(attempt) } Err(GeyserGrpcClientError::TonicStatus(tonic_status)) => { - warn!("! connect failed on {} - retrying: {:?}", grpc_source, tonic_status); + warn!( + "! connect failed on {} - retrying: {:?}", + grpc_source, tonic_status + ); RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::SubscribeSendError(send_error)) => { - warn!("! connect failed with send error on {} - retrying: {:?}", grpc_source, send_error); + warn!( + "! connect failed with send error on {} - retrying: {:?}", + grpc_source, send_error + ); RecoverableConnectionError(attempt) } } } Connected(attempt, mut client) => { - let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); + let subscribe_timeout = + grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); let subscribe_filter = subscribe_filter.clone(); debug!("Subscribe with filter {:?}", subscribe_filter); - let subscribe_result_timeout = - timeout(subscribe_timeout.unwrap_or(Duration::MAX), - client.subscribe_once2(subscribe_filter)) - .await; + let subscribe_result_timeout = timeout( + subscribe_timeout.unwrap_or(Duration::MAX), + client.subscribe_once2(subscribe_filter), + ) + .await; match subscribe_result_timeout { Ok(subscribe_result) => { match subscribe_result { - Ok(geyser_stream) => { - Ready(attempt, geyser_stream) - } + Ok(geyser_stream) => Ready(attempt, geyser_stream), Err(GeyserGrpcClientError::TonicError(_)) => { warn!("! subscribe failed on {} - retrying", grpc_source); RecoverableConnectionError(attempt) @@ -371,20 +217,29 @@ pub fn create_geyser_reconnecting_task( } // non-recoverable Err(unrecoverable_error) => { - error!("! subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error); + error!( + "! subscribe to {} failed with unrecoverable error: {}", + grpc_source, unrecoverable_error + ); FatalError(attempt) } } } Err(_elapsed) => { - warn!("! subscribe failed with timeout on {} - retrying", grpc_source); + warn!( + "! subscribe failed with timeout on {} - retrying", + grpc_source + ); RecoverableConnectionError(attempt) } } } RecoverableConnectionError(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); - info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); + info!( + "! waiting {} seconds, then reconnect to {}", + backoff_secs, grpc_source + ); sleep(Duration::from_secs_f32(backoff_secs)).await; NotConnected(attempt) } @@ -394,7 +249,10 @@ pub fn create_geyser_reconnecting_task( } TheState::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); - info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); + info!( + "! waiting {} seconds, then reconnect to {}", + backoff_secs, grpc_source + ); sleep(Duration::from_secs_f32(backoff_secs)).await; TheState::NotConnected(attempt) } @@ -403,11 +261,15 @@ pub fn create_geyser_reconnecting_task( match geyser_stream.next().await { Some(Ok(update_message)) => { trace!("> recv update message from {}", grpc_source); - match sender.send(Message::GeyserSubscribeUpdate(Box::new(update_message))) { + match sender + .send(Message::GeyserSubscribeUpdate(Box::new(update_message))) + { Ok(n_subscribers) => { - trace!("sent update message to {} subscribers (buffer={})", + trace!( + "sent update message to {} subscribers (buffer={})", n_subscribers, - sender.len()); + sender.len() + ); continue 'recv_loop; } Err(SendError(_)) => { @@ -422,7 +284,7 @@ pub fn create_geyser_reconnecting_task( warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); break 'recv_loop TheState::WaitReconnect(attempt); } - None => { + None => { warn!("geyser stream closed on {} - retrying", grpc_source); break 'recv_loop TheState::WaitReconnect(attempt); } @@ -430,19 +292,15 @@ pub fn create_geyser_reconnecting_task( } // -- end loop } } - } - }); - (jh_geyser_task, receiver_stream) } - - #[cfg(test)] mod tests { + use crate::GrpcConnectionTimeouts; use super::*; #[tokio::test] @@ -487,4 +345,3 @@ mod tests { ); } } - diff --git a/src/grpcmultiplex_fastestwins.rs b/src/grpcmultiplex_fastestwins.rs index 81d58c4..b8ef7cb 100644 --- a/src/grpcmultiplex_fastestwins.rs +++ b/src/grpcmultiplex_fastestwins.rs @@ -1,5 +1,5 @@ -use crate::grpc_subscription_autoreconnect::Message; -use crate::grpc_subscription_autoreconnect::Message::GeyserSubscribeUpdate; +use crate::grpc_subscription_autoreconnect_streams::Message; +use crate::grpc_subscription_autoreconnect_streams::Message::GeyserSubscribeUpdate; use async_stream::stream; use futures::{Stream, StreamExt}; use log::{info, warn}; diff --git a/src/lib.rs b/src/lib.rs index 91794cd..41516f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,66 @@ +use std::fmt::{Debug, Display}; +use std::time::Duration; +use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; + pub mod grpc_subscription; -pub mod grpc_subscription_autoreconnect; +pub mod grpc_subscription_autoreconnect_streams; +pub mod grpc_subscription_autoreconnect_tasks; pub mod grpcmultiplex_fastestwins; mod obfuscate; + + +#[derive(Clone, Debug)] +pub struct GrpcConnectionTimeouts { + pub connect_timeout: Duration, + pub request_timeout: Duration, + pub subscribe_timeout: Duration, +} + +#[derive(Clone)] +pub struct GrpcSourceConfig { + grpc_addr: String, + grpc_x_token: Option, + tls_config: Option, + timeouts: Option, +} + +impl Display for GrpcSourceConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "grpc_addr {}", + crate::obfuscate::url_obfuscate_api_token(&self.grpc_addr) + ) + } +} + +impl Debug for GrpcSourceConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self, f) + } +} + +impl GrpcSourceConfig { + /// Create a grpc source without tls and timeouts + pub fn new_simple(grpc_addr: String) -> Self { + Self { + grpc_addr, + grpc_x_token: None, + tls_config: None, + timeouts: None, + } + } + pub fn new( + grpc_addr: String, + grpc_x_token: Option, + tls_config: Option, + timeouts: GrpcConnectionTimeouts, + ) -> Self { + Self { + grpc_addr, + grpc_x_token, + tls_config, + timeouts: Some(timeouts), + } + } +} From 21f2ef3b7cdc0ae4975b17838e57390340648dbd Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 08:25:05 +0100 Subject: [PATCH 09/32] wip --- src/grpc_subscription_autoreconnect_tasks.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 91e3986..9b96e00 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::pin::Pin; use std::time::Duration; +use anyhow::bail; use tokio::sync::broadcast::error::SendError; use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; @@ -115,15 +116,16 @@ fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel enum TheState>, F: Interceptor> { NotConnected(Attempt), - // Connected(Attempt, Box>>), Connected(Attempt, GeyserGrpcClient), Ready(Attempt, S), // error states RecoverableConnectionError(Attempt), + // non-recoverable error FatalError(Attempt), WaitReconnect(Attempt), } +/// return handler will exit on fatal error pub fn create_geyser_reconnecting_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, @@ -245,7 +247,8 @@ pub fn create_geyser_reconnecting_task( } FatalError(_) => { // TOOD what to do - panic!("Fatal error") + error!("! fatal error grpc connection - aborting"); + bail!("! fatal error grpc connection - aborting"); } TheState::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); @@ -261,6 +264,8 @@ pub fn create_geyser_reconnecting_task( match geyser_stream.next().await { Some(Ok(update_message)) => { trace!("> recv update message from {}", grpc_source); + // TODO consider extract this + // backpressure - should'n we block here? match sender .send(Message::GeyserSubscribeUpdate(Box::new(update_message))) { From dc53a50e57236b23a765635b5612b66a22fc3160 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 08:43:47 +0100 Subject: [PATCH 10/32] switch to mpsc --- src/grpc_subscription_autoreconnect_tasks.rs | 69 ++++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 9b96e00..c9c955f 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,19 +1,18 @@ use crate::grpc_subscription_autoreconnect_tasks::TheState::*; -use async_stream::stream; -use futures::channel::mpsc; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; +use std::future::Future; use std::pin::Pin; use std::time::Duration; use anyhow::bail; -use tokio::sync::broadcast::error::SendError; -use tokio::sync::broadcast::Receiver; +use tokio::sync::mpsc::error::{SendError, SendTimeoutError}; +use tokio::sync::mpsc::Receiver; use tokio::task::JoinHandle; use tokio::time::error::Elapsed; -use tokio::time::{sleep, timeout, Timeout}; +use tokio::time::{Instant, sleep, timeout, Timeout}; use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::{ @@ -130,10 +129,12 @@ pub fn create_geyser_reconnecting_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, ) -> (JoinHandle<()>, Receiver) { - let (sender, receiver_stream) = tokio::sync::broadcast::channel::(1000); + // read this for argument: http://www.randomhacks.net/2019/03/08/should-rust-channels-panic-on-send/ + let (sender, receiver_stream) = tokio::sync::mpsc::channel::(1); let jh_geyser_task = tokio::spawn(async move { let mut state = NotConnected(0); + let mut messages_forwared = 0; loop { state = match state { @@ -266,23 +267,51 @@ pub fn create_geyser_reconnecting_task( trace!("> recv update message from {}", grpc_source); // TODO consider extract this // backpressure - should'n we block here? - match sender - .send(Message::GeyserSubscribeUpdate(Box::new(update_message))) - { - Ok(n_subscribers) => { - trace!( - "sent update message to {} subscribers (buffer={})", - n_subscribers, - sender.len() - ); + // TODO extract timeout param; TODO respect startup + // emit warning if message not received + let warning_threshold = if messages_forwared < 1 { Duration::from_millis(5000) } else { Duration::from_millis(500) }; + let started_at = Instant::now(); + match sender.send_timeout(Message::GeyserSubscribeUpdate(Box::new(update_message)), warning_threshold).await { + Ok(()) => { + messages_forwared += 1; + trace!("sent update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32()); continue 'recv_loop; } - Err(SendError(_)) => { - // note: error does not mean that future sends will also fail! - trace!("no subscribers for update message"); - continue 'recv_loop; + Err(SendTimeoutError::Timeout(_)) => { + warn!("downstream receiver did not pick put message for {}ms - keep waiting", warning_threshold.as_millis()); + + match sender.send(Message::GeyserSubscribeUpdate(Box::new(update_message))).await { + Ok(()) => { + messages_forwared += 1; + trace!("sent delayed update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32()); + } + Err(_send_error ) => { + warn!("downstream receiver closed, message is lost - aborting"); + break 'recv_loop TheState::FatalError(attempt); + } + } + } - }; + Err(SendTimeoutError::Closed(_)) => { + warn!("downstream receiver closed - aborting"); + break 'recv_loop TheState::FatalError(attempt); + } + } + // { + // Ok(n_subscribers) => { + // trace!( + // "sent update message to {} subscribers (buffer={})", + // n_subscribers, + // sender.len() + // ); + // continue 'recv_loop; + // } + // Err(SendError(_)) => { + // // note: error does not mean that future sends will also fail! + // trace!("no subscribers for update message"); + // continue 'recv_loop; + // } + // }; } Some(Err(tonic_status)) => { // all tonic errors are recoverable From cf0c83b0c59c678eacf60f6d3b8ffb0b841fe46c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 08:50:08 +0100 Subject: [PATCH 11/32] compile --- examples/stream_blocks_mainnet.rs | 4 +- examples/stream_blocks_single.rs | 10 +- ...grpc_subscription_autoreconnect_streams.rs | 68 ------------ src/grpc_subscription_autoreconnect_tasks.rs | 103 ++++-------------- src/lib.rs | 72 ++++++++++++ 5 files changed, 99 insertions(+), 158 deletions(-) diff --git a/examples/stream_blocks_mainnet.rs b/examples/stream_blocks_mainnet.rs index d3c3326..47861be 100644 --- a/examples/stream_blocks_mainnet.rs +++ b/examples/stream_blocks_mainnet.rs @@ -22,7 +22,7 @@ use solana_sdk::transaction::TransactionError; use yellowstone_grpc_proto::geyser::SubscribeUpdateBlock; use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, GeyserFilter, + create_geyser_reconnecting_stream, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, @@ -30,7 +30,7 @@ use geyser_grpc_connector::grpcmultiplex_fastestwins::{ use tokio::time::{sleep, Duration}; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; -use geyser_grpc_connector::{GrpcConnectionTimeouts, GrpcSourceConfig}; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_block_consumer( multiplex_stream: impl Stream + Send + 'static, diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index ecd59bb..2655598 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -6,7 +6,7 @@ use std::env; use std::pin::pin; use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, GeyserFilter, + create_geyser_reconnecting_stream, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, @@ -16,8 +16,8 @@ use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; -use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{create_geyser_reconnecting_task, Message}; -use geyser_grpc_connector::{GrpcConnectionTimeouts, GrpcSourceConfig}; +use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{create_geyser_autoconnection_task, Message}; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_blockmini_consumer( multiplex_stream: impl Stream + Send + 'static, @@ -96,13 +96,13 @@ pub async fn main() { info!("Write Block stream.."); - let (jh_geyser_task, mut green_stream) = create_geyser_reconnecting_task( + let (jh_geyser_task, mut green_stream) = create_geyser_autoconnection_task( green_config.clone(), GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); tokio::spawn(async move { - while let Ok(message) = green_stream.recv().await { + while let Some(message) = green_stream.recv().await { match message { Message::GeyserSubscribeUpdate(subscriber_update) => { // info!("got update: {:?}", subscriber_update.update_oneof.); diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs index 687184d..1200647 100644 --- a/src/grpc_subscription_autoreconnect_streams.rs +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -44,74 +44,6 @@ enum ConnectionState>> { WaitReconnect(Attempt), } -#[derive(Clone)] -pub struct GeyserFilter(pub CommitmentConfig); - -impl GeyserFilter { - pub fn blocks_and_txs(&self) -> SubscribeRequest { - let mut blocks_subs = HashMap::new(); - blocks_subs.insert( - "client".to_string(), - SubscribeRequestFilterBlocks { - account_include: Default::default(), - include_transactions: Some(true), - include_accounts: Some(false), - include_entries: Some(false), - }, - ); - - SubscribeRequest { - slots: HashMap::new(), - accounts: Default::default(), - transactions: HashMap::new(), - entry: Default::default(), - blocks: blocks_subs, - blocks_meta: HashMap::new(), - commitment: Some(map_commitment_level(self.0) as i32), - accounts_data_slice: Default::default(), - ping: None, - } - } - - pub fn blocks_meta(&self) -> SubscribeRequest { - let mut blocksmeta_subs = HashMap::new(); - blocksmeta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {}); - - SubscribeRequest { - slots: HashMap::new(), - accounts: Default::default(), - transactions: HashMap::new(), - entry: Default::default(), - blocks: HashMap::new(), - blocks_meta: blocksmeta_subs, - commitment: Some(map_commitment_level(self.0) as i32), - accounts_data_slice: Default::default(), - ping: None, - } - } -} - -fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel { - // solana_sdk -> yellowstone - match commitment_config.commitment { - solana_sdk::commitment_config::CommitmentLevel::Processed => { - yellowstone_grpc_proto::prelude::CommitmentLevel::Processed - } - solana_sdk::commitment_config::CommitmentLevel::Confirmed => { - yellowstone_grpc_proto::prelude::CommitmentLevel::Confirmed - } - solana_sdk::commitment_config::CommitmentLevel::Finalized => { - yellowstone_grpc_proto::prelude::CommitmentLevel::Finalized - } - _ => { - panic!( - "unsupported commitment level {}", - commitment_config.commitment - ) - } - } -} - // Take geyser filter, connect to Geyser and return a generic stream of SubscribeUpdate // note: stream never terminates pub fn create_geyser_reconnecting_stream( diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index c9c955f..3fdfc93 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,4 +1,4 @@ -use crate::grpc_subscription_autoreconnect_tasks::TheState::*; +use crate::grpc_subscription_autoreconnect_tasks::State::*; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; use solana_sdk::commitment_config::CommitmentConfig; @@ -38,6 +38,11 @@ pub enum Message { Connecting(Attempt), } +#[derive(Debug, Clone)] +pub enum AutoconnectionError { + AbortedFatalError, +} + enum ConnectionState>> { NotConnected(Attempt), Connecting(Attempt, JoinHandle>), @@ -45,75 +50,7 @@ enum ConnectionState>> { WaitReconnect(Attempt), } -#[derive(Clone)] -pub struct GeyserFilter(pub CommitmentConfig); - -impl GeyserFilter { - pub fn blocks_and_txs(&self) -> SubscribeRequest { - let mut blocks_subs = HashMap::new(); - blocks_subs.insert( - "client".to_string(), - SubscribeRequestFilterBlocks { - account_include: Default::default(), - include_transactions: Some(true), - include_accounts: Some(false), - include_entries: Some(false), - }, - ); - - SubscribeRequest { - slots: HashMap::new(), - accounts: Default::default(), - transactions: HashMap::new(), - entry: Default::default(), - blocks: blocks_subs, - blocks_meta: HashMap::new(), - commitment: Some(map_commitment_level(self.0) as i32), - accounts_data_slice: Default::default(), - ping: None, - } - } - - pub fn blocks_meta(&self) -> SubscribeRequest { - let mut blocksmeta_subs = HashMap::new(); - blocksmeta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {}); - - SubscribeRequest { - slots: HashMap::new(), - accounts: Default::default(), - transactions: HashMap::new(), - entry: Default::default(), - blocks: HashMap::new(), - blocks_meta: blocksmeta_subs, - commitment: Some(map_commitment_level(self.0) as i32), - accounts_data_slice: Default::default(), - ping: None, - } - } -} - -fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel { - // solana_sdk -> yellowstone - match commitment_config.commitment { - solana_sdk::commitment_config::CommitmentLevel::Processed => { - yellowstone_grpc_proto::prelude::CommitmentLevel::Processed - } - solana_sdk::commitment_config::CommitmentLevel::Confirmed => { - yellowstone_grpc_proto::prelude::CommitmentLevel::Confirmed - } - solana_sdk::commitment_config::CommitmentLevel::Finalized => { - yellowstone_grpc_proto::prelude::CommitmentLevel::Finalized - } - _ => { - panic!( - "unsupported commitment level {}", - commitment_config.commitment - ) - } - } -} - -enum TheState>, F: Interceptor> { +enum State>, F: Interceptor> { NotConnected(Attempt), Connected(Attempt, GeyserGrpcClient), Ready(Attempt, S), @@ -125,10 +62,10 @@ enum TheState>, F: Interceptor> } /// return handler will exit on fatal error -pub fn create_geyser_reconnecting_task( +pub fn create_geyser_autoconnection_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, -) -> (JoinHandle<()>, Receiver) { +) -> (JoinHandle>, Receiver) { // read this for argument: http://www.randomhacks.net/2019/03/08/should-rust-channels-panic-on-send/ let (sender, receiver_stream) = tokio::sync::mpsc::channel::(1); @@ -249,16 +186,16 @@ pub fn create_geyser_reconnecting_task( FatalError(_) => { // TOOD what to do error!("! fatal error grpc connection - aborting"); - bail!("! fatal error grpc connection - aborting"); + return Err(AutoconnectionError::AbortedFatalError); } - TheState::WaitReconnect(attempt) => { + State::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( "! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source ); sleep(Duration::from_secs_f32(backoff_secs)).await; - TheState::NotConnected(attempt) + State::NotConnected(attempt) } Ready(attempt, mut geyser_stream) => { 'recv_loop: loop { @@ -274,27 +211,27 @@ pub fn create_geyser_reconnecting_task( match sender.send_timeout(Message::GeyserSubscribeUpdate(Box::new(update_message)), warning_threshold).await { Ok(()) => { messages_forwared += 1; - trace!("sent update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32()); + trace!("sent update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32() * 1000.0); continue 'recv_loop; } - Err(SendTimeoutError::Timeout(_)) => { + Err(SendTimeoutError::Timeout(the_message)) => { warn!("downstream receiver did not pick put message for {}ms - keep waiting", warning_threshold.as_millis()); - match sender.send(Message::GeyserSubscribeUpdate(Box::new(update_message))).await { + match sender.send(the_message).await { Ok(()) => { messages_forwared += 1; - trace!("sent delayed update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32()); + trace!("sent delayed update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32() * 1000.0); } Err(_send_error ) => { warn!("downstream receiver closed, message is lost - aborting"); - break 'recv_loop TheState::FatalError(attempt); + break 'recv_loop State::FatalError(attempt); } } } Err(SendTimeoutError::Closed(_)) => { warn!("downstream receiver closed - aborting"); - break 'recv_loop TheState::FatalError(attempt); + break 'recv_loop State::FatalError(attempt); } } // { @@ -316,11 +253,11 @@ pub fn create_geyser_reconnecting_task( Some(Err(tonic_status)) => { // all tonic errors are recoverable warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); - break 'recv_loop TheState::WaitReconnect(attempt); + break 'recv_loop State::WaitReconnect(attempt); } None => { warn!("geyser stream closed on {} - retrying", grpc_source); - break 'recv_loop TheState::WaitReconnect(attempt); + break 'recv_loop State::WaitReconnect(attempt); } } } // -- end loop diff --git a/src/lib.rs b/src/lib.rs index 41516f3..dee574b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,8 @@ +use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::time::Duration; +use solana_sdk::commitment_config::CommitmentConfig; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta}; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; pub mod grpc_subscription; @@ -64,3 +67,72 @@ impl GrpcSourceConfig { } } } + + +#[derive(Clone)] +pub struct GeyserFilter(pub CommitmentConfig); + +impl GeyserFilter { + pub fn blocks_and_txs(&self) -> SubscribeRequest { + let mut blocks_subs = HashMap::new(); + blocks_subs.insert( + "client".to_string(), + SubscribeRequestFilterBlocks { + account_include: Default::default(), + include_transactions: Some(true), + include_accounts: Some(false), + include_entries: Some(false), + }, + ); + + SubscribeRequest { + slots: HashMap::new(), + accounts: Default::default(), + transactions: HashMap::new(), + entry: Default::default(), + blocks: blocks_subs, + blocks_meta: HashMap::new(), + commitment: Some(map_commitment_level(self.0) as i32), + accounts_data_slice: Default::default(), + ping: None, + } + } + + pub fn blocks_meta(&self) -> SubscribeRequest { + let mut blocksmeta_subs = HashMap::new(); + blocksmeta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {}); + + SubscribeRequest { + slots: HashMap::new(), + accounts: Default::default(), + transactions: HashMap::new(), + entry: Default::default(), + blocks: HashMap::new(), + blocks_meta: blocksmeta_subs, + commitment: Some(map_commitment_level(self.0) as i32), + accounts_data_slice: Default::default(), + ping: None, + } + } +} + +fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel { + // solana_sdk -> yellowstone + match commitment_config.commitment { + solana_sdk::commitment_config::CommitmentLevel::Processed => { + yellowstone_grpc_proto::prelude::CommitmentLevel::Processed + } + solana_sdk::commitment_config::CommitmentLevel::Confirmed => { + yellowstone_grpc_proto::prelude::CommitmentLevel::Confirmed + } + solana_sdk::commitment_config::CommitmentLevel::Finalized => { + yellowstone_grpc_proto::prelude::CommitmentLevel::Finalized + } + _ => { + panic!( + "unsupported commitment level {}", + commitment_config.commitment + ) + } + } +} From 9dec39dfebbc5af6147a1e03de7042f9e1ac0eb6 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 09:04:03 +0100 Subject: [PATCH 12/32] test/example for autoconnect --- examples/stream_blocks_autoconnect.rs | 145 +++++++++++++++++++ examples/stream_blocks_single.rs | 11 +- src/grpc_subscription_autoreconnect_tasks.rs | 20 ++- 3 files changed, 164 insertions(+), 12 deletions(-) create mode 100644 examples/stream_blocks_autoconnect.rs diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs new file mode 100644 index 0000000..c7eb42d --- /dev/null +++ b/examples/stream_blocks_autoconnect.rs @@ -0,0 +1,145 @@ +use futures::{Stream, StreamExt}; +use log::info; +use solana_sdk::clock::Slot; +use solana_sdk::commitment_config::CommitmentConfig; +use std::env; +use std::pin::pin; + +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ + create_geyser_reconnecting_stream, +}; +use geyser_grpc_connector::grpcmultiplex_fastestwins::{ + create_multiplexed_stream, FromYellowstoneExtractor, +}; +use tokio::time::{sleep, Duration}; +use tracing::warn; +use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; +use yellowstone_grpc_proto::geyser::SubscribeUpdate; +use yellowstone_grpc_proto::prost::Message as _; +use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{create_geyser_autoconnection_task, Message}; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; + +fn start_example_blockmini_consumer( + multiplex_stream: impl Stream + Send + 'static, +) { + tokio::spawn(async move { + let mut blockmeta_stream = pin!(multiplex_stream); + while let Some(mini) = blockmeta_stream.next().await { + info!( + "emitted block mini #{}@{} with {} bytes from multiplexer", + mini.slot, mini.commitment_config.commitment, mini.blocksize + ); + } + }); +} + +pub struct BlockMini { + pub blocksize: usize, + pub slot: Slot, + pub commitment_config: CommitmentConfig, +} + +struct BlockMiniExtractor(CommitmentConfig); + +impl FromYellowstoneExtractor for BlockMiniExtractor { + type Target = BlockMini; + fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(Slot, Self::Target)> { + match update.update_oneof { + Some(UpdateOneof::Block(update_block_message)) => { + let blocksize = update_block_message.encoded_len(); + let slot = update_block_message.slot; + let mini = BlockMini { + blocksize, + slot, + commitment_config: self.0, + }; + Some((slot, mini)) + } + Some(UpdateOneof::BlockMeta(update_blockmeta_message)) => { + let blocksize = update_blockmeta_message.encoded_len(); + let slot = update_blockmeta_message.slot; + let mini = BlockMini { + blocksize, + slot, + commitment_config: self.0, + }; + Some((slot, mini)) + } + _ => None, + } + } +} + +enum TestCases { + Basic, + SlowReceiver, +} + + +#[tokio::main] +pub async fn main() { + // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace + tracing_subscriber::fmt::init(); + // console_subscriber::init(); + + let test_case = TestCases::SlowReceiver; + + let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green"); + let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok(); + + info!( + "Using grpc source on {} ({})", + grpc_addr_green, + grpc_x_token_green.is_some() + ); + + let timeouts = GrpcConnectionTimeouts { + connect_timeout: Duration::from_secs(5), + request_timeout: Duration::from_secs(5), + subscribe_timeout: Duration::from_secs(5), + }; + + let green_config = + GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone()); + + info!("Write Block stream.."); + + let (jh_geyser_task, mut green_stream) = create_geyser_autoconnection_task( + green_config.clone(), + GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), + ); + + tokio::spawn(async move { + + if let TestCases::SlowReceiver = test_case { + sleep(Duration::from_secs(5)).await; + } + + while let Some(message) = green_stream.recv().await { + match message { + Message::GeyserSubscribeUpdate(subscriber_update) => { + // info!("got update: {:?}", subscriber_update.update_oneof.); + info!("got update!!!"); + } + Message::Connecting(attempt) => { + warn!("Connection attempt: {}", attempt); + } + } + } + warn!("Stream aborted"); + }); + + // let green_stream = create_geyser_reconnecting_stream( + // green_config.clone(), + // GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(), + // // GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), + // ); + // let multiplex_stream = create_multiplexed_stream( + // vec![green_stream], + // BlockMiniExtractor(CommitmentConfig::confirmed()), + // ); + // start_example_blockmini_consumer(multiplex_stream); + + // "infinite" sleep + sleep(Duration::from_secs(1800)).await; +} diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index 2655598..e490c7c 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -5,9 +5,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, -}; +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{create_geyser_reconnecting_stream, Message}; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; @@ -16,7 +14,6 @@ use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; -use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{create_geyser_autoconnection_task, Message}; use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_blockmini_consumer( @@ -96,13 +93,15 @@ pub async fn main() { info!("Write Block stream.."); - let (jh_geyser_task, mut green_stream) = create_geyser_autoconnection_task( + let green_stream= create_geyser_reconnecting_stream( green_config.clone(), GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); + tokio::spawn(async move { - while let Some(message) = green_stream.recv().await { + let mut green_stream = pin!(green_stream); + while let Some(message) = green_stream.next().await { match message { Message::GeyserSubscribeUpdate(subscriber_update) => { // info!("got update: {:?}", subscriber_update.update_oneof.); diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 3fdfc93..288c469 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -71,7 +71,7 @@ pub fn create_geyser_autoconnection_task( let jh_geyser_task = tokio::spawn(async move { let mut state = NotConnected(0); - let mut messages_forwared = 0; + let mut messages_forwarded = 0; loop { state = match state { @@ -206,12 +206,19 @@ pub fn create_geyser_autoconnection_task( // backpressure - should'n we block here? // TODO extract timeout param; TODO respect startup // emit warning if message not received - let warning_threshold = if messages_forwared < 1 { Duration::from_millis(5000) } else { Duration::from_millis(500) }; + // note: first send never blocks + let warning_threshold = if messages_forwarded == 1 { Duration::from_millis(3000) } else { Duration::from_millis(500) }; let started_at = Instant::now(); match sender.send_timeout(Message::GeyserSubscribeUpdate(Box::new(update_message)), warning_threshold).await { Ok(()) => { - messages_forwared += 1; - trace!("sent update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32() * 1000.0); + messages_forwarded += 1; + if messages_forwarded == 1 { + // note: first send never blocks - do not print time as this is a lie + trace!("queued first update message"); + } else { + trace!("queued update message {} in {:.02}ms", + messages_forwarded, started_at.elapsed().as_secs_f32() * 1000.0); + } continue 'recv_loop; } Err(SendTimeoutError::Timeout(the_message)) => { @@ -219,8 +226,9 @@ pub fn create_geyser_autoconnection_task( match sender.send(the_message).await { Ok(()) => { - messages_forwared += 1; - trace!("sent delayed update message to channel in {:.02}ms", started_at.elapsed().as_secs_f32() * 1000.0); + messages_forwarded += 1; + trace!("queued delayed update message {} in {:.02}ms", + messages_forwarded, started_at.elapsed().as_secs_f32() * 1000.0); } Err(_send_error ) => { warn!("downstream receiver closed, message is lost - aborting"); From b9875cbb6c042f6730cf63016762a966a1c36cff Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 09:04:25 +0100 Subject: [PATCH 13/32] cleanup --- examples/stream_blocks_autoconnect.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index c7eb42d..8839d3e 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -129,17 +129,6 @@ pub async fn main() { warn!("Stream aborted"); }); - // let green_stream = create_geyser_reconnecting_stream( - // green_config.clone(), - // GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(), - // // GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), - // ); - // let multiplex_stream = create_multiplexed_stream( - // vec![green_stream], - // BlockMiniExtractor(CommitmentConfig::confirmed()), - // ); - // start_example_blockmini_consumer(multiplex_stream); - // "infinite" sleep sleep(Duration::from_secs(1800)).await; } From 0200e9d46924572fe67d4f44b0c920042faea872 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 09:13:37 +0100 Subject: [PATCH 14/32] map fatal errors --- examples/stream_blocks_autoconnect.rs | 11 ++- src/grpc_subscription_autoreconnect_tasks.rs | 79 +++++++++++++------- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index 8839d3e..99884fb 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -73,6 +73,7 @@ impl FromYellowstoneExtractor for BlockMiniExtractor { enum TestCases { Basic, SlowReceiver, + CloseAfterReceiving, } @@ -82,7 +83,7 @@ pub async fn main() { tracing_subscriber::fmt::init(); // console_subscriber::init(); - let test_case = TestCases::SlowReceiver; + let test_case = TestCases::CloseAfterReceiving; let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green"); let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok(); @@ -116,10 +117,18 @@ pub async fn main() { } while let Some(message) = green_stream.recv().await { + + + match message { Message::GeyserSubscribeUpdate(subscriber_update) => { // info!("got update: {:?}", subscriber_update.update_oneof.); info!("got update!!!"); + + if let TestCases::CloseAfterReceiving = test_case { + info!("(testcase) closing stream after receiving"); + return; + } } Message::Connecting(attempt) => { warn!("Connection attempt: {}", attempt); diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 288c469..a8a776d 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,4 +1,3 @@ -use crate::grpc_subscription_autoreconnect_tasks::State::*; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; use solana_sdk::commitment_config::CommitmentConfig; @@ -50,6 +49,15 @@ enum ConnectionState>> { WaitReconnect(Attempt), } +enum FatalErrorReason { + DownstreamChannelClosed, + ConfigurationError, + NetworkError, + SubscribeError, + // everything else + Misc, +} + enum State>, F: Interceptor> { NotConnected(Attempt), Connected(Attempt, GeyserGrpcClient), @@ -57,7 +65,7 @@ enum State>, F: Interceptor> { // error states RecoverableConnectionError(Attempt), // non-recoverable error - FatalError(Attempt), + FatalError(Attempt, FatalErrorReason), WaitReconnect(Attempt), } @@ -70,12 +78,12 @@ pub fn create_geyser_autoconnection_task( let (sender, receiver_stream) = tokio::sync::mpsc::channel::(1); let jh_geyser_task = tokio::spawn(async move { - let mut state = NotConnected(0); + let mut state = State::NotConnected(0); let mut messages_forwarded = 0; loop { state = match state { - NotConnected(mut attempt) => { + State::NotConnected(mut attempt) => { attempt += 1; let addr = grpc_source.grpc_addr.clone(); @@ -104,34 +112,34 @@ pub fn create_geyser_autoconnection_task( .await; match connect_result { - Ok(client) => Connected(attempt, client), - Err(GeyserGrpcClientError::InvalidUri(_)) => FatalError(attempt), - Err(GeyserGrpcClientError::MetadataValueError(_)) => FatalError(attempt), - Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => FatalError(attempt), + Ok(client) => State::Connected(attempt, client), + Err(GeyserGrpcClientError::InvalidUri(_)) => State::FatalError(attempt, FatalErrorReason::ConfigurationError), + Err(GeyserGrpcClientError::MetadataValueError(_)) => State::FatalError(attempt, FatalErrorReason::ConfigurationError), + Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => State::FatalError(attempt, FatalErrorReason::ConfigurationError), Err(GeyserGrpcClientError::TonicError(tonic_error)) => { warn!( "! connect failed on {} - aborting: {:?}", grpc_source, tonic_error ); - FatalError(attempt) + State::FatalError(attempt, FatalErrorReason::NetworkError) } Err(GeyserGrpcClientError::TonicStatus(tonic_status)) => { warn!( "! connect failed on {} - retrying: {:?}", grpc_source, tonic_status ); - RecoverableConnectionError(attempt) + State::RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::SubscribeSendError(send_error)) => { warn!( "! connect failed with send error on {} - retrying: {:?}", grpc_source, send_error ); - RecoverableConnectionError(attempt) + State::RecoverableConnectionError(attempt) } } } - Connected(attempt, mut client) => { + State::Connected(attempt, mut client) => { let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); let subscribe_filter = subscribe_filter.clone(); @@ -146,14 +154,14 @@ pub fn create_geyser_autoconnection_task( match subscribe_result_timeout { Ok(subscribe_result) => { match subscribe_result { - Ok(geyser_stream) => Ready(attempt, geyser_stream), + Ok(geyser_stream) => State::Ready(attempt, geyser_stream), Err(GeyserGrpcClientError::TonicError(_)) => { warn!("! subscribe failed on {} - retrying", grpc_source); - RecoverableConnectionError(attempt) + State::RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::TonicStatus(_)) => { warn!("! subscribe failed on {} - retrying", grpc_source); - RecoverableConnectionError(attempt) + State::RecoverableConnectionError(attempt) } // non-recoverable Err(unrecoverable_error) => { @@ -161,7 +169,7 @@ pub fn create_geyser_autoconnection_task( "! subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error ); - FatalError(attempt) + State::FatalError(attempt, FatalErrorReason::SubscribeError) } } } @@ -170,23 +178,42 @@ pub fn create_geyser_autoconnection_task( "! subscribe failed with timeout on {} - retrying", grpc_source ); - RecoverableConnectionError(attempt) + State::RecoverableConnectionError(attempt) } } } - RecoverableConnectionError(attempt) => { + State::RecoverableConnectionError(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( "! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source ); sleep(Duration::from_secs_f32(backoff_secs)).await; - NotConnected(attempt) + State::NotConnected(attempt) } - FatalError(_) => { - // TOOD what to do - error!("! fatal error grpc connection - aborting"); - return Err(AutoconnectionError::AbortedFatalError); + State::FatalError(_attempt, reason) => { + match reason { + FatalErrorReason::DownstreamChannelClosed => { + warn!("! downstream closed - aborting"); + return Err(AutoconnectionError::AbortedFatalError); + } + FatalErrorReason::ConfigurationError => { + warn!("! fatal configuration error - aborting"); + return Err(AutoconnectionError::AbortedFatalError); + } + FatalErrorReason::NetworkError => { + warn!("! fatal network error - aborting"); + return Err(AutoconnectionError::AbortedFatalError); + } + FatalErrorReason::SubscribeError => { + warn!("! fatal grpc subscribe error - aborting"); + return Err(AutoconnectionError::AbortedFatalError); + } + FatalErrorReason::Misc => { + error!("! fatal misc error grpc connection - aborting"); + return Err(AutoconnectionError::AbortedFatalError); + } + } } State::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); @@ -197,7 +224,7 @@ pub fn create_geyser_autoconnection_task( sleep(Duration::from_secs_f32(backoff_secs)).await; State::NotConnected(attempt) } - Ready(attempt, mut geyser_stream) => { + State::Ready(attempt, mut geyser_stream) => { 'recv_loop: loop { match geyser_stream.next().await { Some(Ok(update_message)) => { @@ -232,14 +259,14 @@ pub fn create_geyser_autoconnection_task( } Err(_send_error ) => { warn!("downstream receiver closed, message is lost - aborting"); - break 'recv_loop State::FatalError(attempt); + break 'recv_loop State::FatalError(attempt, FatalErrorReason::DownstreamChannelClosed); } } } Err(SendTimeoutError::Closed(_)) => { warn!("downstream receiver closed - aborting"); - break 'recv_loop State::FatalError(attempt); + break 'recv_loop State::FatalError(attempt, FatalErrorReason::DownstreamChannelClosed); } } // { From 11b24bc537d2e8c0a8fd297e5415372009c634f0 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 09:23:55 +0100 Subject: [PATCH 15/32] testcase lagging --- examples/stream_blocks_autoconnect.rs | 29 +++++++++++++++----- src/grpc_subscription_autoreconnect_tasks.rs | 8 +++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index 99884fb..0b019b6 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -72,9 +72,12 @@ impl FromYellowstoneExtractor for BlockMiniExtractor { enum TestCases { Basic, - SlowReceiver, + SlowReceiverStartup, + TemporaryLaggingReceiver, CloseAfterReceiving, + AbortTaskFromOutside, } +const TEST_CASE: TestCases = TestCases::TemporaryLaggingReceiver; #[tokio::main] @@ -83,7 +86,6 @@ pub async fn main() { tracing_subscriber::fmt::init(); // console_subscriber::init(); - let test_case = TestCases::CloseAfterReceiving; let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green"); let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok(); @@ -112,20 +114,25 @@ pub async fn main() { tokio::spawn(async move { - if let TestCases::SlowReceiver = test_case { + if let TestCases::SlowReceiverStartup = TEST_CASE { sleep(Duration::from_secs(5)).await; } + let mut message_count = 0; while let Some(message) = green_stream.recv().await { - - - + if let TestCases::AbortTaskFromOutside = TEST_CASE { + if message_count > 5 { + info!("(testcase) aborting task from outside"); + jh_geyser_task.abort(); + } + } match message { Message::GeyserSubscribeUpdate(subscriber_update) => { + message_count += 1; // info!("got update: {:?}", subscriber_update.update_oneof.); info!("got update!!!"); - if let TestCases::CloseAfterReceiving = test_case { + if let TestCases::CloseAfterReceiving = TEST_CASE { info!("(testcase) closing stream after receiving"); return; } @@ -134,6 +141,14 @@ pub async fn main() { warn!("Connection attempt: {}", attempt); } } + + if let TestCases::TemporaryLaggingReceiver = TEST_CASE { + if message_count % 3 == 1 { + info!("(testcase) lagging a bit"); + sleep(Duration::from_millis(1500)).await; + } + } + } warn!("Stream aborted"); }); diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index a8a776d..d85db6e 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -69,7 +69,13 @@ enum State>, F: Interceptor> { WaitReconnect(Attempt), } -/// return handler will exit on fatal error +/// connect to grpc source performing autoconect if required, +/// returns mpsc channel; task will abort on fatal error +/// +/// implementation hints: +/// * no panic/unwrap +/// * do not use "?" +/// * do not "return" unless you really want to abort the task pub fn create_geyser_autoconnection_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, From 80d5c979f3770cd0f09144943c327517b02d9780 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 11:33:37 +0100 Subject: [PATCH 16/32] channel plugger mspc->broadcast --- src/channel_plugger.rs | 152 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/channel_plugger.rs diff --git a/src/channel_plugger.rs b/src/channel_plugger.rs new file mode 100644 index 0000000..12dad03 --- /dev/null +++ b/src/channel_plugger.rs @@ -0,0 +1,152 @@ +use log::{debug, info, warn}; +use std::time::Duration; +use tokio::sync::broadcast::error::RecvError; +use tokio::sync::mpsc::error::SendTimeoutError; +use tokio::time::{sleep, timeout}; + + +pub fn spawn_plugger_mpcs_to_broadcast( + mut upstream: tokio::sync::mpsc::Receiver, + downstream: tokio::sync::broadcast::Sender, +) { + // abort forwarder by closing the sender + let _donothing = tokio::spawn(async move { + while let Some(value) = upstream.recv().await { + match downstream.send(value) { + Ok(n_subscribers) => { + debug!("forwarded to {} subscribers", n_subscribers); + } + Err(_dropped_msg) => { + // decide to continue if no subscribers + debug!("no subscribers - dropping payload and continue"); + } + } + } + debug!("no more messages from producer - shutting down connector"); + }); +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn connect_broadcast_to_mpsc() { + solana_logger::setup_with_default("debug"); + + let (tx1, rx1) = tokio::sync::mpsc::channel::(1); + let (tx2, rx2) = tokio::sync::broadcast::channel::(2); + drop(rx2); + + let jh_producer = tokio::spawn(async move { + for i in 1..=10 { + info!("producer sending {}", i); + if let Err(SendTimeoutError::Timeout(message)) = + tx1.send_timeout(i, Duration::from_millis(200)).await + { + info!("producer send was blocked"); + tx1.send(message).await.unwrap(); + } + sleep(Duration::from_millis(500)).await; + } + }); + + // downstream receiver A connected to broadcast + let mut channel_a = tx2.subscribe(); + tokio::spawn(async move { + loop { + match channel_a.recv().await { + Ok(msg) => { + info!("A: {:?} (len={})", msg, channel_a.len()); + } + Err(RecvError::Lagged(n_missed)) => { + warn!("channel A lagged {} messages", n_missed); + } + Err(RecvError::Closed) => { + info!("channel A closed (by forwarder)"); + break; + } + } + } + }); + + // downstream receiver B connected to broadcast + let mut channel_b = tx2.subscribe(); + tokio::spawn(async move { + loop { + match channel_b.recv().await { + Ok(msg) => { + info!("B: {:?} (len={})", msg, channel_b.len()); + // slow receiver + sleep(Duration::from_millis(1000)).await; + } + Err(RecvError::Lagged(n_missed)) => { + warn!("channel B lagged {} messages", n_missed); + } + Err(RecvError::Closed) => { + info!("channel B closed (by forwarder)"); + break; + } + } + } + }); + + // connect them + spawn_plugger_mpcs_to_broadcast(rx1, tx2); + + // wait forever + info!("Started tasks .. waiting for producer to finish"); + // should take 5 secs + assert!( + timeout(Duration::from_secs(10), jh_producer).await.is_ok(), + "timeout" + ); + info!("producer done - wait a bit longer ..."); + sleep(Duration::from_secs(3)).await; + info!("done."); + + // note how messages pile up for slow receiver B + } + + #[tokio::test] + async fn connect_broadcast_to_mpsc_nosubscribers() { + solana_logger::setup_with_default("debug"); + + let (tx1, rx1) = tokio::sync::mpsc::channel::(1); + let (tx2, rx2) = tokio::sync::broadcast::channel::(2); + + let jh_producer = tokio::spawn(async move { + for i in 1..=10 { + info!("producer sending {}", i); + if let Err(SendTimeoutError::Timeout(message)) = + tx1.send_timeout(i, Duration::from_millis(200)).await + { + info!("producer send was blocked"); + tx1.send(message).await.unwrap(); + } + sleep(Duration::from_millis(500)).await; + } + }); + + // connect them + spawn_plugger_mpcs_to_broadcast(rx1, tx2); + + sleep(Duration::from_secs(3)).await; + info!("dropping subscriber"); + drop(rx2); + + // wait forever + info!("Started tasks .. waiting for producer to finish"); + // should take 5 secs + assert!( + timeout(Duration::from_secs(10), jh_producer).await.is_ok(), + "timeout" + ); + info!("producer done - wait a bit longer ..."); + sleep(Duration::from_secs(3)).await; + info!("done."); + + // note how messages pile up for slow receiver B + } +} From afefb1847bed10044faa5413dbc10423913ba379 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 11:34:45 +0100 Subject: [PATCH 17/32] cleanup --- Cargo.lock | 1 + Cargo.toml | 1 + examples/stream_blocks_autoconnect.rs | 17 ++- examples/stream_blocks_mainnet.rs | 6 +- examples/stream_blocks_single.rs | 9 +- ...grpc_subscription_autoreconnect_streams.rs | 4 +- src/grpc_subscription_autoreconnect_tasks.rs | 116 ++++++++++-------- src/lib.rs | 10 +- 8 files changed, 92 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 898c6f9..c1705cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,6 +1250,7 @@ dependencies = [ "itertools 0.10.5", "log", "merge-streams", + "solana-logger", "solana-sdk", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 9f88a6f..ec0e897 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,3 +33,4 @@ bincode = "1.3.3" [dev-dependencies] tracing-subscriber = "0.3.16" +solana-logger = "1" diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index 0b019b6..6b8d2a0 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -5,19 +5,19 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; +use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{ + create_geyser_autoconnection_task, Message, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; use tokio::time::{sleep, Duration}; use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; -use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{create_geyser_autoconnection_task, Message}; -use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_blockmini_consumer( multiplex_stream: impl Stream + Send + 'static, @@ -70,6 +70,7 @@ impl FromYellowstoneExtractor for BlockMiniExtractor { } } +#[warn(dead_code)] enum TestCases { Basic, SlowReceiverStartup, @@ -79,14 +80,12 @@ enum TestCases { } const TEST_CASE: TestCases = TestCases::TemporaryLaggingReceiver; - #[tokio::main] pub async fn main() { // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace tracing_subscriber::fmt::init(); // console_subscriber::init(); - let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green"); let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok(); @@ -107,19 +106,18 @@ pub async fn main() { info!("Write Block stream.."); - let (jh_geyser_task, mut green_stream) = create_geyser_autoconnection_task( + let (jh_geyser_task, mut message_channel) = create_geyser_autoconnection_task( green_config.clone(), GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); tokio::spawn(async move { - if let TestCases::SlowReceiverStartup = TEST_CASE { sleep(Duration::from_secs(5)).await; } let mut message_count = 0; - while let Some(message) = green_stream.recv().await { + while let Some(message) = message_channel.recv().await { if let TestCases::AbortTaskFromOutside = TEST_CASE { if message_count > 5 { info!("(testcase) aborting task from outside"); @@ -148,7 +146,6 @@ pub async fn main() { sleep(Duration::from_millis(1500)).await; } } - } warn!("Stream aborted"); }); diff --git a/examples/stream_blocks_mainnet.rs b/examples/stream_blocks_mainnet.rs index 47861be..a8e0836 100644 --- a/examples/stream_blocks_mainnet.rs +++ b/examples/stream_blocks_mainnet.rs @@ -21,16 +21,14 @@ use solana_sdk::signature::Signature; use solana_sdk::transaction::TransactionError; use yellowstone_grpc_proto::geyser::SubscribeUpdateBlock; -use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, -}; +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; use tokio::time::{sleep, Duration}; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; -use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_block_consumer( multiplex_stream: impl Stream + Send + 'static, diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index e490c7c..f5db555 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -5,16 +5,18 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{create_geyser_reconnecting_stream, Message}; +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ + create_geyser_reconnecting_stream, Message, +}; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; use tokio::time::{sleep, Duration}; use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; -use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; fn start_example_blockmini_consumer( multiplex_stream: impl Stream + Send + 'static, @@ -93,12 +95,11 @@ pub async fn main() { info!("Write Block stream.."); - let green_stream= create_geyser_reconnecting_stream( + let green_stream = create_geyser_reconnecting_stream( green_config.clone(), GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); - tokio::spawn(async move { let mut green_stream = pin!(green_stream); while let Some(message) = green_stream.next().await { diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs index 1200647..544389a 100644 --- a/src/grpc_subscription_autoreconnect_streams.rs +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -1,3 +1,4 @@ +use crate::GrpcSourceConfig; use async_stream::stream; use futures::channel::mpsc; use futures::{Stream, StreamExt}; @@ -24,7 +25,6 @@ use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; use yellowstone_grpc_proto::tonic::{Code, Status}; -use crate::GrpcSourceConfig; type Attempt = u32; @@ -155,8 +155,8 @@ pub fn create_geyser_reconnecting_stream( #[cfg(test)] mod tests { - use crate::GrpcConnectionTimeouts; use super::*; + use crate::GrpcConnectionTimeouts; #[tokio::test] async fn test_debug_no_secrets() { diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index d85db6e..b21411e 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,3 +1,5 @@ +use crate::GrpcSourceConfig; +use anyhow::bail; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; use solana_sdk::commitment_config::CommitmentConfig; @@ -6,12 +8,11 @@ use std::fmt::{Debug, Display}; use std::future::Future; use std::pin::Pin; use std::time::Duration; -use anyhow::bail; use tokio::sync::mpsc::error::{SendError, SendTimeoutError}; use tokio::sync::mpsc::Receiver; -use tokio::task::JoinHandle; +use tokio::task::{AbortHandle, JoinHandle}; use tokio::time::error::Elapsed; -use tokio::time::{Instant, sleep, timeout, Timeout}; +use tokio::time::{sleep, timeout, Instant, Timeout}; use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::{ @@ -24,7 +25,6 @@ use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; use yellowstone_grpc_proto::tonic::{Code, Status}; -use crate::GrpcSourceConfig; type Attempt = u32; @@ -37,11 +37,6 @@ pub enum Message { Connecting(Attempt), } -#[derive(Debug, Clone)] -pub enum AutoconnectionError { - AbortedFatalError, -} - enum ConnectionState>> { NotConnected(Attempt), Connecting(Attempt, JoinHandle>), @@ -79,7 +74,7 @@ enum State>, F: Interceptor> { pub fn create_geyser_autoconnection_task( grpc_source: GrpcSourceConfig, subscribe_filter: SubscribeRequest, -) -> (JoinHandle>, Receiver) { +) -> (AbortHandle, Receiver) { // read this for argument: http://www.randomhacks.net/2019/03/08/should-rust-channels-panic-on-send/ let (sender, receiver_stream) = tokio::sync::mpsc::channel::(1); @@ -119,9 +114,15 @@ pub fn create_geyser_autoconnection_task( match connect_result { Ok(client) => State::Connected(attempt, client), - Err(GeyserGrpcClientError::InvalidUri(_)) => State::FatalError(attempt, FatalErrorReason::ConfigurationError), - Err(GeyserGrpcClientError::MetadataValueError(_)) => State::FatalError(attempt, FatalErrorReason::ConfigurationError), - Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => State::FatalError(attempt, FatalErrorReason::ConfigurationError), + Err(GeyserGrpcClientError::InvalidUri(_)) => { + State::FatalError(attempt, FatalErrorReason::ConfigurationError) + } + Err(GeyserGrpcClientError::MetadataValueError(_)) => { + State::FatalError(attempt, FatalErrorReason::ConfigurationError) + } + Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => { + State::FatalError(attempt, FatalErrorReason::ConfigurationError) + } Err(GeyserGrpcClientError::TonicError(tonic_error)) => { warn!( "! connect failed on {} - aborting: {:?}", @@ -197,30 +198,28 @@ pub fn create_geyser_autoconnection_task( sleep(Duration::from_secs_f32(backoff_secs)).await; State::NotConnected(attempt) } - State::FatalError(_attempt, reason) => { - match reason { - FatalErrorReason::DownstreamChannelClosed => { - warn!("! downstream closed - aborting"); - return Err(AutoconnectionError::AbortedFatalError); - } - FatalErrorReason::ConfigurationError => { - warn!("! fatal configuration error - aborting"); - return Err(AutoconnectionError::AbortedFatalError); - } - FatalErrorReason::NetworkError => { - warn!("! fatal network error - aborting"); - return Err(AutoconnectionError::AbortedFatalError); - } - FatalErrorReason::SubscribeError => { - warn!("! fatal grpc subscribe error - aborting"); - return Err(AutoconnectionError::AbortedFatalError); - } - FatalErrorReason::Misc => { - error!("! fatal misc error grpc connection - aborting"); - return Err(AutoconnectionError::AbortedFatalError); - } + State::FatalError(_attempt, reason) => match reason { + FatalErrorReason::DownstreamChannelClosed => { + warn!("! downstream closed - aborting"); + return; } - } + FatalErrorReason::ConfigurationError => { + warn!("! fatal configuration error - aborting"); + return; + } + FatalErrorReason::NetworkError => { + warn!("! fatal network error - aborting"); + return; + } + FatalErrorReason::SubscribeError => { + warn!("! fatal grpc subscribe error - aborting"); + return; + } + FatalErrorReason::Misc => { + error!("! fatal misc error grpc connection - aborting"); + return; + } + }, State::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( @@ -240,17 +239,30 @@ pub fn create_geyser_autoconnection_task( // TODO extract timeout param; TODO respect startup // emit warning if message not received // note: first send never blocks - let warning_threshold = if messages_forwarded == 1 { Duration::from_millis(3000) } else { Duration::from_millis(500) }; + let warning_threshold = if messages_forwarded == 1 { + Duration::from_millis(3000) + } else { + Duration::from_millis(500) + }; let started_at = Instant::now(); - match sender.send_timeout(Message::GeyserSubscribeUpdate(Box::new(update_message)), warning_threshold).await { + match sender + .send_timeout( + Message::GeyserSubscribeUpdate(Box::new(update_message)), + warning_threshold, + ) + .await + { Ok(()) => { messages_forwarded += 1; if messages_forwarded == 1 { // note: first send never blocks - do not print time as this is a lie trace!("queued first update message"); } else { - trace!("queued update message {} in {:.02}ms", - messages_forwarded, started_at.elapsed().as_secs_f32() * 1000.0); + trace!( + "queued update message {} in {:.02}ms", + messages_forwarded, + started_at.elapsed().as_secs_f32() * 1000.0 + ); } continue 'recv_loop; } @@ -260,19 +272,27 @@ pub fn create_geyser_autoconnection_task( match sender.send(the_message).await { Ok(()) => { messages_forwarded += 1; - trace!("queued delayed update message {} in {:.02}ms", - messages_forwarded, started_at.elapsed().as_secs_f32() * 1000.0); + trace!( + "queued delayed update message {} in {:.02}ms", + messages_forwarded, + started_at.elapsed().as_secs_f32() * 1000.0 + ); } - Err(_send_error ) => { + Err(_send_error) => { warn!("downstream receiver closed, message is lost - aborting"); - break 'recv_loop State::FatalError(attempt, FatalErrorReason::DownstreamChannelClosed); + break 'recv_loop State::FatalError( + attempt, + FatalErrorReason::DownstreamChannelClosed, + ); } } - } Err(SendTimeoutError::Closed(_)) => { warn!("downstream receiver closed - aborting"); - break 'recv_loop State::FatalError(attempt, FatalErrorReason::DownstreamChannelClosed); + break 'recv_loop State::FatalError( + attempt, + FatalErrorReason::DownstreamChannelClosed, + ); } } // { @@ -307,13 +327,13 @@ pub fn create_geyser_autoconnection_task( } }); - (jh_geyser_task, receiver_stream) + (jh_geyser_task.abort_handle(), receiver_stream) } #[cfg(test)] mod tests { - use crate::GrpcConnectionTimeouts; use super::*; + use crate::GrpcConnectionTimeouts; #[tokio::test] async fn test_debug_no_secrets() { diff --git a/src/lib.rs b/src/lib.rs index dee574b..358d941 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,17 +1,20 @@ +use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::time::Duration; -use solana_sdk::commitment_config::CommitmentConfig; -use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta}; +use yellowstone_grpc_proto::geyser::{ + CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, + SubscribeRequestFilterBlocksMeta, +}; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; +mod channel_plugger; pub mod grpc_subscription; pub mod grpc_subscription_autoreconnect_streams; pub mod grpc_subscription_autoreconnect_tasks; pub mod grpcmultiplex_fastestwins; mod obfuscate; - #[derive(Clone, Debug)] pub struct GrpcConnectionTimeouts { pub connect_timeout: Duration, @@ -68,7 +71,6 @@ impl GrpcSourceConfig { } } - #[derive(Clone)] pub struct GeyserFilter(pub CommitmentConfig); From 46f56872e5687487e3926c77b28a20aeb2a19e23 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 11:46:06 +0100 Subject: [PATCH 18/32] introduce T --- src/channel_plugger.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/channel_plugger.rs b/src/channel_plugger.rs index 12dad03..76c316b 100644 --- a/src/channel_plugger.rs +++ b/src/channel_plugger.rs @@ -5,9 +5,9 @@ use tokio::sync::mpsc::error::SendTimeoutError; use tokio::time::{sleep, timeout}; -pub fn spawn_plugger_mpcs_to_broadcast( - mut upstream: tokio::sync::mpsc::Receiver, - downstream: tokio::sync::broadcast::Sender, +pub fn spawn_plugger_mpcs_to_broadcast( + mut upstream: tokio::sync::mpsc::Receiver, + downstream: tokio::sync::broadcast::Sender, ) { // abort forwarder by closing the sender let _donothing = tokio::spawn(async move { From 87a4bbc4848b635d805e1901cde4ad69ad580237 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 19 Jan 2024 11:46:19 +0100 Subject: [PATCH 19/32] use channel_plugger in test --- examples/stream_blocks_autoconnect.rs | 6 +++++- src/lib.rs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index 6b8d2a0..bb0be04 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -18,6 +18,7 @@ use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; +use geyser_grpc_connector::channel_plugger::spawn_plugger_mpcs_to_broadcast; fn start_example_blockmini_consumer( multiplex_stream: impl Stream + Send + 'static, @@ -106,10 +107,13 @@ pub async fn main() { info!("Write Block stream.."); + let (broadcast_tx, broadcast_rx) = tokio::sync::broadcast::channel(100); let (jh_geyser_task, mut message_channel) = create_geyser_autoconnection_task( green_config.clone(), GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); + spawn_plugger_mpcs_to_broadcast(message_channel, broadcast_tx); + let mut message_channel = broadcast_rx; tokio::spawn(async move { if let TestCases::SlowReceiverStartup = TEST_CASE { @@ -117,7 +121,7 @@ pub async fn main() { } let mut message_count = 0; - while let Some(message) = message_channel.recv().await { + while let Ok(message) = message_channel.recv().await { if let TestCases::AbortTaskFromOutside = TEST_CASE { if message_count > 5 { info!("(testcase) aborting task from outside"); diff --git a/src/lib.rs b/src/lib.rs index 358d941..9e66b62 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,11 +8,11 @@ use yellowstone_grpc_proto::geyser::{ }; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; -mod channel_plugger; pub mod grpc_subscription; pub mod grpc_subscription_autoreconnect_streams; pub mod grpc_subscription_autoreconnect_tasks; pub mod grpcmultiplex_fastestwins; +pub mod channel_plugger; mod obfuscate; #[derive(Clone, Debug)] From 6e15250b96637eea66c2054a7a0576bcfd3ee1b0 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 22 Jan 2024 10:55:03 +0100 Subject: [PATCH 20/32] minor cleanup --- examples/stream_blocks_autoconnect.rs | 18 +++++++++--------- src/grpc_subscription_autoreconnect_tasks.rs | 15 --------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index bb0be04..4e21eb3 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -5,6 +5,9 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; +use geyser_grpc_connector::channel_plugger::{ + spawn_broadcast_channel_plug, spawn_plugger_mpcs_to_broadcast, +}; use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{ create_geyser_autoconnection_task, Message, @@ -18,7 +21,6 @@ use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; -use geyser_grpc_connector::channel_plugger::spawn_plugger_mpcs_to_broadcast; fn start_example_blockmini_consumer( multiplex_stream: impl Stream + Send + 'static, @@ -79,7 +81,7 @@ enum TestCases { CloseAfterReceiving, AbortTaskFromOutside, } -const TEST_CASE: TestCases = TestCases::TemporaryLaggingReceiver; +const TEST_CASE: TestCases = TestCases::Basic; #[tokio::main] pub async fn main() { @@ -107,13 +109,12 @@ pub async fn main() { info!("Write Block stream.."); - let (broadcast_tx, broadcast_rx) = tokio::sync::broadcast::channel(100); - let (jh_geyser_task, mut message_channel) = create_geyser_autoconnection_task( + let (jh_geyser_task, message_channel) = create_geyser_autoconnection_task( green_config.clone(), GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), ); - spawn_plugger_mpcs_to_broadcast(message_channel, broadcast_tx); - let mut message_channel = broadcast_rx; + let mut message_channel = + spawn_broadcast_channel_plug(tokio::sync::broadcast::channel(8), message_channel); tokio::spawn(async move { if let TestCases::SlowReceiverStartup = TEST_CASE { @@ -131,8 +132,7 @@ pub async fn main() { match message { Message::GeyserSubscribeUpdate(subscriber_update) => { message_count += 1; - // info!("got update: {:?}", subscriber_update.update_oneof.); - info!("got update!!!"); + info!("got update - {} bytes", subscriber_update.encoded_len()); if let TestCases::CloseAfterReceiving = TEST_CASE { info!("(testcase) closing stream after receiving"); @@ -155,5 +155,5 @@ pub async fn main() { }); // "infinite" sleep - sleep(Duration::from_secs(1800)).await; + sleep(Duration::from_secs(2000)).await; } diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index b21411e..52c61a5 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -295,21 +295,6 @@ pub fn create_geyser_autoconnection_task( ); } } - // { - // Ok(n_subscribers) => { - // trace!( - // "sent update message to {} subscribers (buffer={})", - // n_subscribers, - // sender.len() - // ); - // continue 'recv_loop; - // } - // Err(SendError(_)) => { - // // note: error does not mean that future sends will also fail! - // trace!("no subscribers for update message"); - // continue 'recv_loop; - // } - // }; } Some(Err(tonic_status)) => { // all tonic errors are recoverable From b3808da95e98a36475e3bec6e72ea32e692ff4fe Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 22 Jan 2024 10:56:07 +0100 Subject: [PATCH 21/32] example --- src/channel_plugger.rs | 23 +++++++++++++++++++++-- src/lib.rs | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/channel_plugger.rs b/src/channel_plugger.rs index 76c316b..83baf7f 100644 --- a/src/channel_plugger.rs +++ b/src/channel_plugger.rs @@ -4,13 +4,26 @@ use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc::error::SendTimeoutError; use tokio::time::{sleep, timeout}; +/// usage: see plug_pattern test +pub fn spawn_broadcast_channel_plug( + downstream_broadcast: ( + tokio::sync::broadcast::Sender, + tokio::sync::broadcast::Receiver, + ), + upstream: tokio::sync::mpsc::Receiver, +) -> tokio::sync::broadcast::Receiver { + spawn_plugger_mpcs_to_broadcast(upstream, downstream_broadcast.0); + downstream_broadcast.1 +} +/// note: backpressure will NOT get propagated to upstream pub fn spawn_plugger_mpcs_to_broadcast( mut upstream: tokio::sync::mpsc::Receiver, downstream: tokio::sync::broadcast::Sender, + // TODO allow multiple downstreams + fanout ) { // abort forwarder by closing the sender - let _donothing = tokio::spawn(async move { + let _private_handler = tokio::spawn(async move { while let Some(value) = upstream.recv().await { match downstream.send(value) { Ok(n_subscribers) => { @@ -26,11 +39,17 @@ pub fn spawn_plugger_mpcs_to_broadcast( }); } - #[cfg(test)] mod tests { use super::*; + #[tokio::test] + async fn plug_pattern() { + let (jh_task, message_channel) = tokio::sync::mpsc::channel::(1); + let broadcast_rx = + spawn_broadcast_channel_plug(tokio::sync::broadcast::channel(8), message_channel); + } + #[tokio::test] async fn connect_broadcast_to_mpsc() { solana_logger::setup_with_default("debug"); diff --git a/src/lib.rs b/src/lib.rs index 9e66b62..5de9710 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,11 +8,11 @@ use yellowstone_grpc_proto::geyser::{ }; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; +pub mod channel_plugger; pub mod grpc_subscription; pub mod grpc_subscription_autoreconnect_streams; pub mod grpc_subscription_autoreconnect_tasks; pub mod grpcmultiplex_fastestwins; -pub mod channel_plugger; mod obfuscate; #[derive(Clone, Debug)] From 1d0b7890c3f422679cdd018d1c5c783424f85d69 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 22 Jan 2024 18:26:26 +0100 Subject: [PATCH 22/32] move Message type --- examples/stream_blocks_autoconnect.rs | 4 ++-- ...grpc_subscription_autoreconnect_streams.rs | 13 +----------- src/grpc_subscription_autoreconnect_tasks.rs | 15 +++----------- src/grpcmultiplex_fastestwins.rs | 4 ++-- src/lib.rs | 20 +++++++++++++------ 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index 4e21eb3..c6edbdb 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -10,12 +10,12 @@ use geyser_grpc_connector::channel_plugger::{ }; use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{ - create_geyser_autoconnection_task, Message, + create_geyser_autoconnection_task, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; -use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message}; use tokio::time::{sleep, Duration}; use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs index 544389a..840451b 100644 --- a/src/grpc_subscription_autoreconnect_streams.rs +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -1,4 +1,4 @@ -use crate::GrpcSourceConfig; +use crate::{Attempt, GrpcSourceConfig, Message}; use async_stream::stream; use futures::channel::mpsc; use futures::{Stream, StreamExt}; @@ -26,17 +26,6 @@ use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; use yellowstone_grpc_proto::tonic::{Code, Status}; -type Attempt = u32; - -// wraps payload and status messages -// clone is required by broacast channel -#[derive(Clone)] -pub enum Message { - GeyserSubscribeUpdate(Box), - // connect (attempt=1) or reconnect(attempt=2..) - Connecting(Attempt), -} - enum ConnectionState>> { NotConnected(Attempt), Connecting(Attempt, JoinHandle>), diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 52c61a5..e8a4fc2 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,4 +1,4 @@ -use crate::GrpcSourceConfig; +use crate::{GrpcSourceConfig, Message}; use anyhow::bail; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; @@ -28,15 +28,6 @@ use yellowstone_grpc_proto::tonic::{Code, Status}; type Attempt = u32; -// wraps payload and status messages -// clone is required by broacast channel -#[derive(Clone)] -pub enum Message { - GeyserSubscribeUpdate(Box), - // connect (attempt=1) or reconnect(attempt=2..) - Connecting(Attempt), -} - enum ConnectionState>> { NotConnected(Attempt), Connecting(Attempt, JoinHandle>), @@ -76,7 +67,7 @@ pub fn create_geyser_autoconnection_task( subscribe_filter: SubscribeRequest, ) -> (AbortHandle, Receiver) { // read this for argument: http://www.randomhacks.net/2019/03/08/should-rust-channels-panic-on-send/ - let (sender, receiver_stream) = tokio::sync::mpsc::channel::(1); + let (sender, receiver_channel) = tokio::sync::mpsc::channel::(1); let jh_geyser_task = tokio::spawn(async move { let mut state = State::NotConnected(0); @@ -312,7 +303,7 @@ pub fn create_geyser_autoconnection_task( } }); - (jh_geyser_task.abort_handle(), receiver_stream) + (jh_geyser_task.abort_handle(), receiver_channel) } #[cfg(test)] diff --git a/src/grpcmultiplex_fastestwins.rs b/src/grpcmultiplex_fastestwins.rs index b8ef7cb..b0f36f0 100644 --- a/src/grpcmultiplex_fastestwins.rs +++ b/src/grpcmultiplex_fastestwins.rs @@ -1,11 +1,11 @@ -use crate::grpc_subscription_autoreconnect_streams::Message; -use crate::grpc_subscription_autoreconnect_streams::Message::GeyserSubscribeUpdate; +use crate::Message; use async_stream::stream; use futures::{Stream, StreamExt}; use log::{info, warn}; use merge_streams::MergeStreams; use solana_sdk::clock::Slot; use yellowstone_grpc_proto::geyser::SubscribeUpdate; +use crate::Message::GeyserSubscribeUpdate; pub trait FromYellowstoneExtractor { // Target is something like ProducedBlock diff --git a/src/lib.rs b/src/lib.rs index 5de9710..a31d646 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,10 +2,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::time::Duration; -use yellowstone_grpc_proto::geyser::{ - CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, - SubscribeRequestFilterBlocksMeta, -}; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta, SubscribeUpdate}; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; pub mod channel_plugger; @@ -15,6 +12,17 @@ pub mod grpc_subscription_autoreconnect_tasks; pub mod grpcmultiplex_fastestwins; mod obfuscate; +type Attempt = u32; + +// wraps payload and status messages +// clone is required by broacast channel +#[derive(Clone)] +pub enum Message { + GeyserSubscribeUpdate(Box), + // connect (attempt=1) or reconnect(attempt=2..) + Connecting(Attempt), +} + #[derive(Clone, Debug)] pub struct GrpcConnectionTimeouts { pub connect_timeout: Duration, @@ -24,8 +32,8 @@ pub struct GrpcConnectionTimeouts { #[derive(Clone)] pub struct GrpcSourceConfig { - grpc_addr: String, - grpc_x_token: Option, + pub grpc_addr: String, + pub grpc_x_token: Option, tls_config: Option, timeouts: Option, } From 550b6072f7a68bd897169a58be9067658f7b9d4c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 22 Jan 2024 18:26:45 +0100 Subject: [PATCH 23/32] update v1.12.0+solana.1.17.15 --- Cargo.toml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ec0e897..93b0e5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "geyser-grpc-connector" -version = "0.7.2+yellowstone.1.11" +version = "0.50.1+yellowstone.1.12" edition = "2021" description = "Multiplexing and Reconnection on Yellowstone gRPC Geyser client streaming" @@ -9,13 +9,12 @@ authors = ["GroovieGermanikus "] repository = "https://github.com/blockworks-foundation/geyser-grpc-connector" [dependencies] -# v1.11.0+solana.1.16.17 -yellowstone-grpc-proto = "1.11.0" -# v1.12.0+solana.1.16.17 -yellowstone-grpc-client = "1.12.0" +yellowstone-grpc-client = { version = "1.13.0+solana.1.17.15", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" } +yellowstone-grpc-proto = { version = "1.12.0+solana.1.17.15", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" } + # required for CommitmentConfig -solana-sdk = "~1.16.17" +solana-sdk = "~1.17.15" url = "2.5.0" async-stream = "0.3.5" From 542f11607f79a2160cbe13cc93c33d3e4e3501c5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 22 Jan 2024 18:34:03 +0100 Subject: [PATCH 24/32] Cargo.lock --- Cargo.lock | 190 ++++++++++++++++++++++++++++------------------------- 1 file changed, 102 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1705cd..26d072e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "72832d73be48bac96a5d7944568f305d829ed55b0ce3b483647089dfaf6cf704" dependencies = [ "cfg-if", "getrandom 0.2.11", @@ -249,12 +249,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "array-bytes" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ad284aeb45c13f2fb4f084de4a420ebf447423bdf9386c0540ce33cb3ef4b8c" - [[package]] name = "arrayref" version = "0.3.7" @@ -444,6 +438,9 @@ name = "bitflags" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +dependencies = [ + "serde", +] [[package]] name = "bitmaps" @@ -1239,7 +1236,7 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" -version = "0.7.2+yellowstone.1.11" +version = "0.50.1+yellowstone.1.12" dependencies = [ "anyhow", "async-stream", @@ -1326,7 +1323,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.4", ] [[package]] @@ -1677,6 +1674,18 @@ dependencies = [ "libsecp256k1-core", ] +[[package]] +name = "light-poseidon" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c9a85a9752c549ceb7578064b4ed891179d20acd85f27318573b64d2d7ee7ee" +dependencies = [ + "ark-bn254", + "ark-ff", + "num-bigint 0.4.4", + "thiserror", +] + [[package]] name = "linux-raw-sys" version = "0.4.12" @@ -2242,6 +2251,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "qualifier_attr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + [[package]] name = "quote" version = "1.0.33" @@ -2780,9 +2800,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121e55656c2094950f374247e1303dd09517f1ed49c91bf60bf114760b286eb4" +checksum = "22ea4bedfcc8686ae6d01a3d8288f5b9746cd00ec63f0ce9a6415849d35add50" dependencies = [ "Inflector", "base64 0.21.5", @@ -2793,42 +2813,21 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "solana-address-lookup-table-program", "solana-config-program", "solana-sdk", "spl-token", "spl-token-2022", + "spl-token-group-interface", "spl-token-metadata-interface", "thiserror", "zstd", ] -[[package]] -name = "solana-address-lookup-table-program" -version = "1.16.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ccb31f7f14d5876acd9ec38f5bf6097bfb4b350141d81c7ff2bf684db3ca815" -dependencies = [ - "bincode", - "bytemuck", - "log", - "num-derive 0.3.3", - "num-traits", - "rustc_version", - "serde", - "solana-frozen-abi", - "solana-frozen-abi-macro", - "solana-program", - "solana-program-runtime", - "solana-sdk", - "thiserror", -] - [[package]] name = "solana-config-program" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94dc0f4463daf1c6155f20eac948ea4ced705e5f5520546aef4e11e746a6d95d" +checksum = "8de23cd0dd8673f4590e90bfa47ff19eb629f4b7dc15a3fb173a62d932801d07" dependencies = [ "bincode", "chrono", @@ -2840,11 +2839,11 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266bf0311bb403d31206aa2904b8741f57c7f5e27580b6810ad5e22fc7c3282" +checksum = "4090f2ac64149ce1fbabd5277f41e278edc1f38121927fe8f6355e67ead3e199" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.4", "blake3", "block-buffer 0.10.4", "bs58", @@ -2853,13 +2852,10 @@ dependencies = [ "cc", "either", "generic-array", - "getrandom 0.1.16", "im", "lazy_static", "log", "memmap2", - "once_cell", - "rand_core 0.6.4", "rustc_version", "serde", "serde_bytes", @@ -2873,9 +2869,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dfe18c5155015dcb494c6de84a03b725fcf90ec2006a047769018b94c2cf0de" +checksum = "765bcdc1ecc31ea5d3d7ddb680ffa6645809c122b4ffdc223b161850e6ba352b" dependencies = [ "proc-macro2", "quote", @@ -2885,9 +2881,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f76fe25c2d06dcf621befd1e8d5655143e8a059c7e20fcb71736bc80ed779d6" +checksum = "9c7f3cad088bc5f00569cb5b4c3aaba8d935f8f7cc25c91cc0c55a8a7de2b137" dependencies = [ "env_logger", "lazy_static", @@ -2896,9 +2892,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db165b8a7f5d840abef011c78a18ffe63cad9192d676b07d94f469b6b5dc6cf6" +checksum = "2de5041d16120852c0deea047c024e1fad8819e49041491f0cca6c91c243fd5d" dependencies = [ "log", "solana-sdk", @@ -2906,9 +2902,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa01731bb3952904962d49a1ea1205db54e93f3a56f4006d32e02a7c85d60546" +checksum = "2fd6f25f0076b6eb873f7e2a85e53191ac2affe6782131be1a2867d057307e20" dependencies = [ "crossbeam-channel", "gethostname", @@ -2916,22 +2912,22 @@ dependencies = [ "log", "reqwest", "solana-sdk", + "thiserror", ] [[package]] name = "solana-program" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bb16998986492de307eef503ce47e84503d35baa92dc60832b22476948b1c16" +checksum = "c1141d1dffbe68852128f7bbcc3c43a5d2cb715ecffeeb64eb81bb93cbaf80bb" dependencies = [ "ark-bn254", "ark-ec", "ark-ff", "ark-serialize", - "array-bytes", "base64 0.21.5", "bincode", - "bitflags 1.3.2", + "bitflags 2.4.1", "blake3", "borsh 0.10.3", "borsh 0.9.3", @@ -2948,14 +2944,14 @@ dependencies = [ "lazy_static", "libc", "libsecp256k1", + "light-poseidon", "log", "memoffset", "num-bigint 0.4.4", "num-derive 0.3.3", "num-traits", "parking_lot", - "rand 0.7.3", - "rand_chacha 0.2.2", + "rand 0.8.5", "rustc_version", "rustversion", "serde", @@ -2975,9 +2971,9 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "036d6ecf67a3a7c6dc74d4f7fa6ab321e7ce8feccb7c9dff8384a41d0a12345b" +checksum = "942de577a2865cec28fc174575c9bd6cf7af815832af67fe40ca856075550998" dependencies = [ "base64 0.21.5", "bincode", @@ -2989,7 +2985,7 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "percentage", - "rand 0.7.3", + "rand 0.8.5", "rustc_version", "serde", "solana-frozen-abi", @@ -3003,14 +2999,14 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4106cda3d10833ba957dbd25fb841b50aeca7480ccf8f54859294716f54bcd4b" +checksum = "278a95acb99377dd4585599fdbec23d0a6fcb94ec78285283723fdd365fe885e" dependencies = [ "assert_matches", "base64 0.21.5", "bincode", - "bitflags 1.3.2", + "bitflags 2.4.1", "borsh 0.10.3", "bs58", "bytemuck", @@ -3033,8 +3029,9 @@ dependencies = [ "num_enum 0.6.1", "pbkdf2 0.11.0", "qstring", + "qualifier_attr", "rand 0.7.3", - "rand_chacha 0.2.2", + "rand 0.8.5", "rustc_version", "rustversion", "serde", @@ -3056,9 +3053,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e560806a3859717eb2220b26e2cd68bb757b63affa3e79c3f1d8d853b5ee78f" +checksum = "92dbaf563210f61828800f2a3d8c188fa2afede91920d364982e280318db2eb5" dependencies = [ "bs58", "proc-macro2", @@ -3068,10 +3065,16 @@ dependencies = [ ] [[package]] -name = "solana-transaction-status" -version = "1.16.17" +name = "solana-security-txt" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236dd4e43b8a7402bce250228e04c0c68d9493a3e19c71b377ccc7c4390fd969" +checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" + +[[package]] +name = "solana-transaction-status" +version = "1.17.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2031070cba17802f7108b53f6db01b82cdfb0360b0a8b9d51c584f2e9dd9e4" dependencies = [ "Inflector", "base64 0.21.5", @@ -3084,7 +3087,6 @@ dependencies = [ "serde_derive", "serde_json", "solana-account-decoder", - "solana-address-lookup-table-program", "solana-sdk", "spl-associated-token-account", "spl-memo", @@ -3095,9 +3097,9 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.16.17" +version = "1.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278c08e13bc04b6940997602909052524a375154b00cf0bfa934359a3bb7e6f0" +checksum = "ef26fb44734aa940e6648bbbeead677edc68c7e1ec09128e5f16a8924c389a38" dependencies = [ "aes-gcm-siv", "base64 0.21.5", @@ -3124,9 +3126,9 @@ dependencies = [ [[package]] name = "solana_rbpf" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17d4ba1e58947346e360fabde0697029d36ba83c42f669199b16a8931313cf29" +checksum = "3d457cc2ba742c120492a64b7fa60e22c575e891f6b55039f4d736568fb112a3" dependencies = [ "byteorder", "combine", @@ -3149,9 +3151,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "spl-associated-token-account" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385e31c29981488f2820b2022d8e731aae3b02e6e18e2fd854e4c9a94dc44fc3" +checksum = "992d9c64c2564cc8f63a4b508bf3ebcdf2254b0429b13cd1d31adb6162432a5f" dependencies = [ "assert_matches", "borsh 0.10.3", @@ -3247,9 +3249,9 @@ dependencies = [ [[package]] name = "spl-tlv-account-resolution" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062e148d3eab7b165582757453632ffeef490c02c86a48bfdb4988f63eefb3b9" +checksum = "615d381f48ddd2bb3c57c7f7fb207591a2a05054639b18a62e785117dd7a8683" dependencies = [ "bytemuck", "solana-program", @@ -3276,9 +3278,9 @@ dependencies = [ [[package]] name = "spl-token-2022" -version = "0.9.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4abf34a65ba420584a0c35f3903f8d727d1f13ababbdc3f714c6b065a686e86" +checksum = "d697fac19fd74ff472dfcc13f0b442dd71403178ce1de7b5d16f83a33561c059" dependencies = [ "arrayref", "bytemuck", @@ -3286,16 +3288,31 @@ dependencies = [ "num-traits", "num_enum 0.7.1", "solana-program", + "solana-security-txt", "solana-zk-token-sdk", "spl-memo", "spl-pod", "spl-token", + "spl-token-group-interface", "spl-token-metadata-interface", "spl-transfer-hook-interface", "spl-type-length-value", "thiserror", ] +[[package]] +name = "spl-token-group-interface" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b889509d49fa74a4a033ca5dae6c2307e9e918122d97e58562f5c4ffa795c75d" +dependencies = [ + "bytemuck", + "solana-program", + "spl-discriminator", + "spl-pod", + "spl-program-error", +] + [[package]] name = "spl-token-metadata-interface" version = "0.2.0" @@ -3312,9 +3329,9 @@ dependencies = [ [[package]] name = "spl-transfer-hook-interface" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051d31803f873cabe71aec3c1b849f35248beae5d19a347d93a5c9cccc5d5a9b" +checksum = "7aabdb7c471566f6ddcee724beb8618449ea24b399e58d464d6b5bc7db550259" dependencies = [ "arrayref", "bytemuck", @@ -3607,7 +3624,6 @@ dependencies = [ "axum", "base64 0.21.5", "bytes", - "flate2", "h2", "http", "http-body", @@ -4148,9 +4164,8 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "1.12.0+solana.1.16.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e58204f372a7e82d15d72bdf99334029c4e9cdc15bd2e9a5c33b598d9f1eb8b6" +version = "1.13.0+solana.1.17.15" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.12.0+solana.1.17.15#c7b72cc8781c2dc48e4a7c94e411f95df495cf2f" dependencies = [ "bytes", "futures", @@ -4163,9 +4178,8 @@ dependencies = [ [[package]] name = "yellowstone-grpc-proto" -version = "1.11.0+solana.1.16.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00d751c6ef3093ec90ab1e16c6a504b5bea99aca6c688c429fed4cc56782f57e" +version = "1.12.0+solana.1.17.15" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.12.0+solana.1.17.15#c7b72cc8781c2dc48e4a7c94e411f95df495cf2f" dependencies = [ "anyhow", "bincode", From 1b9926fc1b019f152f423c26837e8d2627cd4318 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 23 Jan 2024 14:48:47 +0100 Subject: [PATCH 25/32] slots --- src/lib.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index a31d646..dd3f2ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::time::Duration; -use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta, SubscribeUpdate}; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate}; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; pub mod channel_plugger; @@ -124,6 +124,26 @@ impl GeyserFilter { ping: None, } } + + pub fn slots(&self) -> SubscribeRequest { + let mut slots_subs = HashMap::new(); + slots_subs.insert("client".to_string(), + SubscribeRequestFilterSlots { + filter_by_commitment: Some(true), + }); + + SubscribeRequest { + slots: slots_subs, + accounts: Default::default(), + transactions: HashMap::new(), + entry: Default::default(), + blocks: HashMap::new(), + blocks_meta: HashMap::new(), + commitment: Some(map_commitment_level(self.0) as i32), + accounts_data_slice: Default::default(), + ping: None, + } + } } fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel { From f22a0f289ee6a93c9233d35831bd612fb5ea83e5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 23 Jan 2024 17:50:10 +0100 Subject: [PATCH 26/32] remove exclamation mark from logs --- examples/stream_blocks_single.rs | 50 ++++++++++++++++--- src/channel_plugger.rs | 1 + ...grpc_subscription_autoreconnect_streams.rs | 8 +-- src/grpc_subscription_autoreconnect_tasks.rs | 16 +++--- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index f5db555..fd1ecb2 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -6,12 +6,12 @@ use std::env; use std::pin::pin; use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, Message, + create_geyser_reconnecting_stream, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; -use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig}; +use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message}; use tokio::time::{sleep, Duration}; use tracing::warn; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; @@ -90,14 +90,19 @@ pub async fn main() { subscribe_timeout: Duration::from_secs(5), }; - let green_config = + let config = GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone()); info!("Write Block stream.."); let green_stream = create_geyser_reconnecting_stream( - green_config.clone(), - GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), + config.clone(), + GeyserFilter(CommitmentConfig::finalized()).blocks_and_txs(), + ); + + let blue_stream = create_geyser_reconnecting_stream( + config.clone(), + GeyserFilter(CommitmentConfig::processed()).blocks_and_txs(), ); tokio::spawn(async move { @@ -105,8 +110,28 @@ pub async fn main() { while let Some(message) = green_stream.next().await { match message { Message::GeyserSubscribeUpdate(subscriber_update) => { - // info!("got update: {:?}", subscriber_update.update_oneof.); - info!("got update!!!"); + let mapped = map_block_update(*subscriber_update); + if let Some(slot) = mapped { + info!("got update (green)!!! slot: {}", slot); + } + } + Message::Connecting(attempt) => { + warn!("Connection attempt: {}", attempt); + } + } + } + warn!("Stream aborted"); + }); + + tokio::spawn(async move { + let mut blue_stream = pin!(blue_stream); + while let Some(message) = blue_stream.next().await { + match message { + Message::GeyserSubscribeUpdate(subscriber_update) => { + let mapped = map_block_update(*subscriber_update); + if let Some(slot) = mapped { + info!("got update (blue)!!! slot: {}", slot); + } } Message::Connecting(attempt) => { warn!("Connection attempt: {}", attempt); @@ -130,3 +155,14 @@ pub async fn main() { // "infinite" sleep sleep(Duration::from_secs(1800)).await; } + + +fn map_block_update(update: SubscribeUpdate) -> Option { + match update.update_oneof { + Some(UpdateOneof::Block(update_block_message)) => { + let slot = update_block_message.slot; + Some(slot) + } + _ => None, + } +} diff --git a/src/channel_plugger.rs b/src/channel_plugger.rs index 83baf7f..b3b3ee8 100644 --- a/src/channel_plugger.rs +++ b/src/channel_plugger.rs @@ -3,6 +3,7 @@ use std::time::Duration; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc::error::SendTimeoutError; use tokio::time::{sleep, timeout}; +use crate::grpcmultiplex_fastestwins::FromYellowstoneExtractor; /// usage: see plug_pattern test pub fn spawn_broadcast_channel_plug( diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs index 840451b..dfea64f 100644 --- a/src/grpc_subscription_autoreconnect_streams.rs +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -94,11 +94,11 @@ pub fn create_geyser_reconnecting_stream( Ok(Ok(subscribed_stream)) => (ConnectionState::Ready(attempt, subscribed_stream), Message::Connecting(attempt)), Ok(Err(geyser_error)) => { // ATM we consider all errors recoverable - warn!("! subscribe failed on {} - retrying: {:?}", grpc_source, geyser_error); + warn!("subscribe failed on {} - retrying: {:?}", grpc_source, geyser_error); (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) }, Err(geyser_grpc_task_error) => { - panic!("! task aborted - should not happen :{geyser_grpc_task_error}"); + panic!("task aborted - should not happen :{geyser_grpc_task_error}"); } } @@ -113,7 +113,7 @@ pub fn create_geyser_reconnecting_stream( } Some(Err(tonic_status)) => { // ATM we consider all errors recoverable - warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); + warn!("error on {} - retrying: {:?}", grpc_source, tonic_status); (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) } None => { @@ -127,7 +127,7 @@ pub fn create_geyser_reconnecting_stream( ConnectionState::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); - info!("! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); + info!("waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source); sleep(Duration::from_secs_f32(backoff_secs)).await; (ConnectionState::NotConnected(attempt), Message::Connecting(attempt)) } diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index e8a4fc2..a0c29fe 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -154,11 +154,11 @@ pub fn create_geyser_autoconnection_task( match subscribe_result { Ok(geyser_stream) => State::Ready(attempt, geyser_stream), Err(GeyserGrpcClientError::TonicError(_)) => { - warn!("! subscribe failed on {} - retrying", grpc_source); + warn!("subscribe failed on {} - retrying", grpc_source); State::RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::TonicStatus(_)) => { - warn!("! subscribe failed on {} - retrying", grpc_source); + warn!("subscribe failed on {} - retrying", grpc_source); State::RecoverableConnectionError(attempt) } // non-recoverable @@ -191,23 +191,23 @@ pub fn create_geyser_autoconnection_task( } State::FatalError(_attempt, reason) => match reason { FatalErrorReason::DownstreamChannelClosed => { - warn!("! downstream closed - aborting"); + warn!("downstream closed - aborting"); return; } FatalErrorReason::ConfigurationError => { - warn!("! fatal configuration error - aborting"); + warn!("fatal configuration error - aborting"); return; } FatalErrorReason::NetworkError => { - warn!("! fatal network error - aborting"); + warn!("fatal network error - aborting"); return; } FatalErrorReason::SubscribeError => { - warn!("! fatal grpc subscribe error - aborting"); + warn!("fatal grpc subscribe error - aborting"); return; } FatalErrorReason::Misc => { - error!("! fatal misc error grpc connection - aborting"); + error!("fatal misc error grpc connection - aborting"); return; } }, @@ -289,7 +289,7 @@ pub fn create_geyser_autoconnection_task( } Some(Err(tonic_status)) => { // all tonic errors are recoverable - warn!("! error on {} - retrying: {:?}", grpc_source, tonic_status); + warn!("error on {} - retrying: {:?}", grpc_source, tonic_status); break 'recv_loop State::WaitReconnect(attempt); } None => { From ec43c2972e157dbc0f3df23b267daa5061f35259 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 26 Jan 2024 19:16:21 +0100 Subject: [PATCH 27/32] handle timeout --- src/grpc_subscription_autoreconnect_tasks.rs | 126 +++++++++---------- 1 file changed, 59 insertions(+), 67 deletions(-) diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index a0c29fe..66d6d61 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,37 +1,29 @@ use crate::{GrpcSourceConfig, Message}; -use anyhow::bail; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; -use solana_sdk::commitment_config::CommitmentConfig; -use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::future::Future; -use std::pin::Pin; use std::time::Duration; -use tokio::sync::mpsc::error::{SendError, SendTimeoutError}; +use tokio::sync::mpsc::error::SendTimeoutError; use tokio::sync::mpsc::Receiver; -use tokio::task::{AbortHandle, JoinHandle}; +use tokio::task::AbortHandle; +use tokio::time::{sleep, timeout, Instant}; use tokio::time::error::Elapsed; -use tokio::time::{sleep, timeout, Instant, Timeout}; -use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; -use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; -use yellowstone_grpc_proto::geyser::{ - CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, -}; -use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; -use yellowstone_grpc_proto::tonic; -use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; -use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; +use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError}; +use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate}; use yellowstone_grpc_proto::tonic::service::Interceptor; -use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; -use yellowstone_grpc_proto::tonic::{Code, Status}; +use yellowstone_grpc_proto::tonic::Status; type Attempt = u32; -enum ConnectionState>> { +enum ConnectionState>, F: Interceptor> { NotConnected(Attempt), - Connecting(Attempt, JoinHandle>), + Connected(Attempt, GeyserGrpcClient), Ready(Attempt, S), + // error states + RecoverableConnectionError(Attempt), + // non-recoverable error + FatalError(Attempt, FatalErrorReason), WaitReconnect(Attempt), } @@ -44,17 +36,6 @@ enum FatalErrorReason { Misc, } -enum State>, F: Interceptor> { - NotConnected(Attempt), - Connected(Attempt, GeyserGrpcClient), - Ready(Attempt, S), - // error states - RecoverableConnectionError(Attempt), - // non-recoverable error - FatalError(Attempt, FatalErrorReason), - WaitReconnect(Attempt), -} - /// connect to grpc source performing autoconect if required, /// returns mpsc channel; task will abort on fatal error /// @@ -70,12 +51,12 @@ pub fn create_geyser_autoconnection_task( let (sender, receiver_channel) = tokio::sync::mpsc::channel::(1); let jh_geyser_task = tokio::spawn(async move { - let mut state = State::NotConnected(0); + let mut state = ConnectionState::NotConnected(0); let mut messages_forwarded = 0; loop { state = match state { - State::NotConnected(mut attempt) => { + ConnectionState::NotConnected(mut attempt) => { attempt += 1; let addr = grpc_source.grpc_addr.clone(); @@ -104,40 +85,47 @@ pub fn create_geyser_autoconnection_task( .await; match connect_result { - Ok(client) => State::Connected(attempt, client), - Err(GeyserGrpcClientError::InvalidUri(_)) => { - State::FatalError(attempt, FatalErrorReason::ConfigurationError) - } + Ok(client) => ConnectionState::Connected(attempt, client), + Err(GeyserGrpcClientError::InvalidUri(_)) => ConnectionState::FatalError( + attempt, + FatalErrorReason::ConfigurationError, + ), Err(GeyserGrpcClientError::MetadataValueError(_)) => { - State::FatalError(attempt, FatalErrorReason::ConfigurationError) + ConnectionState::FatalError( + attempt, + FatalErrorReason::ConfigurationError, + ) } Err(GeyserGrpcClientError::InvalidXTokenLength(_)) => { - State::FatalError(attempt, FatalErrorReason::ConfigurationError) + ConnectionState::FatalError( + attempt, + FatalErrorReason::ConfigurationError, + ) } Err(GeyserGrpcClientError::TonicError(tonic_error)) => { warn!( "! connect failed on {} - aborting: {:?}", grpc_source, tonic_error ); - State::FatalError(attempt, FatalErrorReason::NetworkError) + ConnectionState::FatalError(attempt, FatalErrorReason::NetworkError) } Err(GeyserGrpcClientError::TonicStatus(tonic_status)) => { warn!( "! connect failed on {} - retrying: {:?}", grpc_source, tonic_status ); - State::RecoverableConnectionError(attempt) + ConnectionState::RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::SubscribeSendError(send_error)) => { warn!( "! connect failed with send error on {} - retrying: {:?}", grpc_source, send_error ); - State::RecoverableConnectionError(attempt) + ConnectionState::RecoverableConnectionError(attempt) } } } - State::Connected(attempt, mut client) => { + ConnectionState::Connected(attempt, mut client) => { let subscribe_timeout = grpc_source.timeouts.as_ref().map(|t| t.subscribe_timeout); let subscribe_filter = subscribe_filter.clone(); @@ -152,14 +140,14 @@ pub fn create_geyser_autoconnection_task( match subscribe_result_timeout { Ok(subscribe_result) => { match subscribe_result { - Ok(geyser_stream) => State::Ready(attempt, geyser_stream), + Ok(geyser_stream) => ConnectionState::Ready(attempt, geyser_stream), Err(GeyserGrpcClientError::TonicError(_)) => { warn!("subscribe failed on {} - retrying", grpc_source); - State::RecoverableConnectionError(attempt) + ConnectionState::RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::TonicStatus(_)) => { warn!("subscribe failed on {} - retrying", grpc_source); - State::RecoverableConnectionError(attempt) + ConnectionState::RecoverableConnectionError(attempt) } // non-recoverable Err(unrecoverable_error) => { @@ -167,7 +155,10 @@ pub fn create_geyser_autoconnection_task( "! subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error ); - State::FatalError(attempt, FatalErrorReason::SubscribeError) + ConnectionState::FatalError( + attempt, + FatalErrorReason::SubscribeError, + ) } } } @@ -176,20 +167,20 @@ pub fn create_geyser_autoconnection_task( "! subscribe failed with timeout on {} - retrying", grpc_source ); - State::RecoverableConnectionError(attempt) + ConnectionState::RecoverableConnectionError(attempt) } } } - State::RecoverableConnectionError(attempt) => { + ConnectionState::RecoverableConnectionError(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( "! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source ); sleep(Duration::from_secs_f32(backoff_secs)).await; - State::NotConnected(attempt) + ConnectionState::NotConnected(attempt) } - State::FatalError(_attempt, reason) => match reason { + ConnectionState::FatalError(_attempt, reason) => match reason { FatalErrorReason::DownstreamChannelClosed => { warn!("downstream closed - aborting"); return; @@ -211,25 +202,22 @@ pub fn create_geyser_autoconnection_task( return; } }, - State::WaitReconnect(attempt) => { + ConnectionState::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( "! waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source ); sleep(Duration::from_secs_f32(backoff_secs)).await; - State::NotConnected(attempt) + ConnectionState::NotConnected(attempt) } - State::Ready(attempt, mut geyser_stream) => { + ConnectionState::Ready(attempt, mut geyser_stream) => { + let receive_timeout = grpc_source.timeouts.as_ref().map(|t| t.receive_timeout); 'recv_loop: loop { - match geyser_stream.next().await { - Some(Ok(update_message)) => { + match timeout(receive_timeout.unwrap_or(Duration::MAX), geyser_stream.next()).await { + Ok(Some(Ok(update_message))) => { trace!("> recv update message from {}", grpc_source); - // TODO consider extract this - // backpressure - should'n we block here? - // TODO extract timeout param; TODO respect startup - // emit warning if message not received - // note: first send never blocks + // note: first send never blocks as the mpsc channel has capacity 1 let warning_threshold = if messages_forwarded == 1 { Duration::from_millis(3000) } else { @@ -271,7 +259,7 @@ pub fn create_geyser_autoconnection_task( } Err(_send_error) => { warn!("downstream receiver closed, message is lost - aborting"); - break 'recv_loop State::FatalError( + break 'recv_loop ConnectionState::FatalError( attempt, FatalErrorReason::DownstreamChannelClosed, ); @@ -280,21 +268,25 @@ pub fn create_geyser_autoconnection_task( } Err(SendTimeoutError::Closed(_)) => { warn!("downstream receiver closed - aborting"); - break 'recv_loop State::FatalError( + break 'recv_loop ConnectionState::FatalError( attempt, FatalErrorReason::DownstreamChannelClosed, ); } } } - Some(Err(tonic_status)) => { + Ok(Some(Err(tonic_status))) => { // all tonic errors are recoverable warn!("error on {} - retrying: {:?}", grpc_source, tonic_status); - break 'recv_loop State::WaitReconnect(attempt); + break 'recv_loop ConnectionState::WaitReconnect(attempt); } - None => { + Ok(None) => { warn!("geyser stream closed on {} - retrying", grpc_source); - break 'recv_loop State::WaitReconnect(attempt); + break 'recv_loop ConnectionState::WaitReconnect(attempt); + } + Err(_elapsed) => { + warn!("timeout on {} - retrying", grpc_source); + break 'recv_loop ConnectionState::WaitReconnect(attempt); } } } // -- end loop From db034040e86dd9c6606769b6f7752d69166ca57b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 26 Jan 2024 19:19:05 +0100 Subject: [PATCH 28/32] handle timeout for stream version --- ...grpc_subscription_autoreconnect_streams.rs | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs index dfea64f..7d403a6 100644 --- a/src/grpc_subscription_autoreconnect_streams.rs +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -1,30 +1,15 @@ use crate::{Attempt, GrpcSourceConfig, Message}; use async_stream::stream; -use futures::channel::mpsc; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; -use solana_sdk::commitment_config::CommitmentConfig; -use std::collections::HashMap; use std::fmt::{Debug, Display}; -use std::pin::Pin; use std::time::Duration; -use tokio::sync::broadcast::error::SendError; -use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; -use tokio::time::error::Elapsed; -use tokio::time::{sleep, timeout, Timeout}; -use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError, GeyserGrpcClientResult}; -use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; -use yellowstone_grpc_proto::geyser::{ - CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeUpdate, -}; -use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; -use yellowstone_grpc_proto::tonic; -use yellowstone_grpc_proto::tonic::codegen::http::uri::InvalidUri; -use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue; +use tokio::time::{sleep, timeout}; +use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult}; +use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate}; use yellowstone_grpc_proto::tonic::service::Interceptor; -use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; -use yellowstone_grpc_proto::tonic::{Code, Status}; +use yellowstone_grpc_proto::tonic::Status; enum ConnectionState>> { NotConnected(Attempt), @@ -105,22 +90,27 @@ pub fn create_geyser_reconnecting_stream( } ConnectionState::Ready(attempt, mut geyser_stream) => { - - match geyser_stream.next().await { - Some(Ok(update_message)) => { + let receive_timeout = grpc_source.timeouts.as_ref().map(|t| t.receive_timeout); + match timeout(receive_timeout.unwrap_or(Duration::MAX), geyser_stream.next()).await { + Ok(Some(Ok(update_message))) => { trace!("> recv update message from {}", grpc_source); (ConnectionState::Ready(attempt, geyser_stream), Message::GeyserSubscribeUpdate(Box::new(update_message))) } - Some(Err(tonic_status)) => { + Ok(Some(Err(tonic_status))) => { // ATM we consider all errors recoverable warn!("error on {} - retrying: {:?}", grpc_source, tonic_status); (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) } - None => { + Ok(None) => { // should not arrive here, Mean the stream close. warn!("geyser stream closed on {} - retrying", grpc_source); (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) } + Err(_elapsed) => { + // timeout + warn!("geyser stream timeout on {} - retrying", grpc_source); + (ConnectionState::WaitReconnect(attempt), Message::Connecting(attempt)) + } } } From 68221ce0ccef46da65a3f5dfd7d1c61926155145 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 26 Jan 2024 19:25:39 +0100 Subject: [PATCH 29/32] clippy+fmt --- examples/stream_blocks_autoconnect.rs | 30 ++++--------------- examples/stream_blocks_mainnet.rs | 1 + examples/stream_blocks_single.rs | 9 ++---- src/channel_plugger.rs | 7 ++--- src/grpc_subscription.rs | 9 ++---- ...grpc_subscription_autoreconnect_streams.rs | 6 ++-- src/grpc_subscription_autoreconnect_tasks.rs | 12 +++++--- src/grpcmultiplex_fastestwins.rs | 2 +- src/lib.rs | 16 ++++++---- 9 files changed, 37 insertions(+), 55 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index c6edbdb..dcc4e7b 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -5,16 +5,9 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::channel_plugger::{ - spawn_broadcast_channel_plug, spawn_plugger_mpcs_to_broadcast, -}; -use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; -use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{ - create_geyser_autoconnection_task, -}; -use geyser_grpc_connector::grpcmultiplex_fastestwins::{ - create_multiplexed_stream, FromYellowstoneExtractor, -}; +use geyser_grpc_connector::channel_plugger::spawn_broadcast_channel_plug; +use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task; +use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor; use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message}; use tokio::time::{sleep, Duration}; use tracing::warn; @@ -22,20 +15,6 @@ use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; use yellowstone_grpc_proto::prost::Message as _; -fn start_example_blockmini_consumer( - multiplex_stream: impl Stream + Send + 'static, -) { - tokio::spawn(async move { - let mut blockmeta_stream = pin!(multiplex_stream); - while let Some(mini) = blockmeta_stream.next().await { - info!( - "emitted block mini #{}@{} with {} bytes from multiplexer", - mini.slot, mini.commitment_config.commitment, mini.blocksize - ); - } - }); -} - pub struct BlockMini { pub blocksize: usize, pub slot: Slot, @@ -73,7 +52,7 @@ impl FromYellowstoneExtractor for BlockMiniExtractor { } } -#[warn(dead_code)] +#[allow(dead_code)] enum TestCases { Basic, SlowReceiverStartup, @@ -102,6 +81,7 @@ pub async fn main() { connect_timeout: Duration::from_secs(5), request_timeout: Duration::from_secs(5), subscribe_timeout: Duration::from_secs(5), + receive_timeout: Duration::from_secs(5), }; let green_config = diff --git a/examples/stream_blocks_mainnet.rs b/examples/stream_blocks_mainnet.rs index a8e0836..e2c2731 100644 --- a/examples/stream_blocks_mainnet.rs +++ b/examples/stream_blocks_mainnet.rs @@ -129,6 +129,7 @@ pub async fn main() { connect_timeout: Duration::from_secs(5), request_timeout: Duration::from_secs(5), subscribe_timeout: Duration::from_secs(5), + receive_timeout: Duration::from_secs(5), }; let green_config = diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index fd1ecb2..9b1e85c 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -5,9 +5,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::env; use std::pin::pin; -use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::{ - create_geyser_reconnecting_stream, -}; +use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; @@ -88,10 +86,10 @@ pub async fn main() { connect_timeout: Duration::from_secs(5), request_timeout: Duration::from_secs(5), subscribe_timeout: Duration::from_secs(5), + receive_timeout: Duration::from_secs(5), }; - let config = - GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone()); + let config = GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone()); info!("Write Block stream.."); @@ -156,7 +154,6 @@ pub async fn main() { sleep(Duration::from_secs(1800)).await; } - fn map_block_update(update: SubscribeUpdate) -> Option { match update.update_oneof { Some(UpdateOneof::Block(update_block_message)) => { diff --git a/src/channel_plugger.rs b/src/channel_plugger.rs index b3b3ee8..9ba1628 100644 --- a/src/channel_plugger.rs +++ b/src/channel_plugger.rs @@ -2,8 +2,6 @@ use log::{debug, info, warn}; use std::time::Duration; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc::error::SendTimeoutError; -use tokio::time::{sleep, timeout}; -use crate::grpcmultiplex_fastestwins::FromYellowstoneExtractor; /// usage: see plug_pattern test pub fn spawn_broadcast_channel_plug( @@ -43,11 +41,12 @@ pub fn spawn_plugger_mpcs_to_broadcast( #[cfg(test)] mod tests { use super::*; + use tokio::time::{sleep, timeout}; #[tokio::test] async fn plug_pattern() { - let (jh_task, message_channel) = tokio::sync::mpsc::channel::(1); - let broadcast_rx = + let (_jh_task, message_channel) = tokio::sync::mpsc::channel::(1); + let _broadcast_rx = spawn_broadcast_channel_plug(tokio::sync::broadcast::channel(8), message_channel); } diff --git a/src/grpc_subscription.rs b/src/grpc_subscription.rs index e4059f4..0d0d1ce 100644 --- a/src/grpc_subscription.rs +++ b/src/grpc_subscription.rs @@ -1,15 +1,10 @@ -// use crate::{ -// endpoint_stremers::EndpointStreaming, -// rpc_polling::vote_accounts_and_cluster_info_polling::poll_vote_accounts_and_cluster_info, -// }; use anyhow::{bail, Context}; -use futures::{Stream, StreamExt}; +use futures::StreamExt; use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; -use tokio::sync::broadcast::{Receiver, Sender}; +use tokio::sync::broadcast::Sender; use yellowstone_grpc_client::GeyserGrpcClient; -use yellowstone_grpc_proto::geyser::SubscribeRequest; use yellowstone_grpc_proto::prelude::{ subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequestFilterBlocks, SubscribeUpdateBlock, diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs index 7d403a6..e070292 100644 --- a/src/grpc_subscription_autoreconnect_streams.rs +++ b/src/grpc_subscription_autoreconnect_streams.rs @@ -1,14 +1,12 @@ use crate::{Attempt, GrpcSourceConfig, Message}; use async_stream::stream; use futures::{Stream, StreamExt}; -use log::{debug, error, info, log, trace, warn, Level}; -use std::fmt::{Debug, Display}; +use log::{debug, info, log, trace, warn, Level}; use std::time::Duration; use tokio::task::JoinHandle; use tokio::time::{sleep, timeout}; use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult}; use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate}; -use yellowstone_grpc_proto::tonic::service::Interceptor; use yellowstone_grpc_proto::tonic::Status; enum ConnectionState>> { @@ -143,6 +141,7 @@ mod tests { connect_timeout: Duration::from_secs(1), request_timeout: Duration::from_secs(2), subscribe_timeout: Duration::from_secs(3), + receive_timeout: Duration::from_secs(3), }; assert_eq!( format!( @@ -164,6 +163,7 @@ mod tests { connect_timeout: Duration::from_secs(1), request_timeout: Duration::from_secs(2), subscribe_timeout: Duration::from_secs(3), + receive_timeout: Duration::from_secs(3), }; assert_eq!( format!( diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 66d6d61..fbc6354 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -1,14 +1,11 @@ use crate::{GrpcSourceConfig, Message}; use futures::{Stream, StreamExt}; use log::{debug, error, info, log, trace, warn, Level}; -use std::fmt::{Debug, Display}; -use std::future::Future; use std::time::Duration; use tokio::sync::mpsc::error::SendTimeoutError; use tokio::sync::mpsc::Receiver; use tokio::task::AbortHandle; use tokio::time::{sleep, timeout, Instant}; -use tokio::time::error::Elapsed; use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError}; use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate}; use yellowstone_grpc_proto::tonic::service::Interceptor; @@ -214,7 +211,12 @@ pub fn create_geyser_autoconnection_task( ConnectionState::Ready(attempt, mut geyser_stream) => { let receive_timeout = grpc_source.timeouts.as_ref().map(|t| t.receive_timeout); 'recv_loop: loop { - match timeout(receive_timeout.unwrap_or(Duration::MAX), geyser_stream.next()).await { + match timeout( + receive_timeout.unwrap_or(Duration::MAX), + geyser_stream.next(), + ) + .await + { Ok(Some(Ok(update_message))) => { trace!("> recv update message from {}", grpc_source); // note: first send never blocks as the mpsc channel has capacity 1 @@ -309,6 +311,7 @@ mod tests { connect_timeout: Duration::from_secs(1), request_timeout: Duration::from_secs(2), subscribe_timeout: Duration::from_secs(3), + receive_timeout: Duration::from_secs(3), }; assert_eq!( format!( @@ -330,6 +333,7 @@ mod tests { connect_timeout: Duration::from_secs(1), request_timeout: Duration::from_secs(2), subscribe_timeout: Duration::from_secs(3), + receive_timeout: Duration::from_secs(3), }; assert_eq!( format!( diff --git a/src/grpcmultiplex_fastestwins.rs b/src/grpcmultiplex_fastestwins.rs index b0f36f0..78dbde1 100644 --- a/src/grpcmultiplex_fastestwins.rs +++ b/src/grpcmultiplex_fastestwins.rs @@ -1,11 +1,11 @@ use crate::Message; +use crate::Message::GeyserSubscribeUpdate; use async_stream::stream; use futures::{Stream, StreamExt}; use log::{info, warn}; use merge_streams::MergeStreams; use solana_sdk::clock::Slot; use yellowstone_grpc_proto::geyser::SubscribeUpdate; -use crate::Message::GeyserSubscribeUpdate; pub trait FromYellowstoneExtractor { // Target is something like ProducedBlock diff --git a/src/lib.rs b/src/lib.rs index dd3f2ad..2a493a9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,10 @@ use solana_sdk::commitment_config::CommitmentConfig; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::time::Duration; -use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate}; +use yellowstone_grpc_proto::geyser::{ + CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, + SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate, +}; use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; pub mod channel_plugger; @@ -28,6 +31,7 @@ pub struct GrpcConnectionTimeouts { pub connect_timeout: Duration, pub request_timeout: Duration, pub subscribe_timeout: Duration, + pub receive_timeout: Duration, } #[derive(Clone)] @@ -127,10 +131,12 @@ impl GeyserFilter { pub fn slots(&self) -> SubscribeRequest { let mut slots_subs = HashMap::new(); - slots_subs.insert("client".to_string(), - SubscribeRequestFilterSlots { - filter_by_commitment: Some(true), - }); + slots_subs.insert( + "client".to_string(), + SubscribeRequestFilterSlots { + filter_by_commitment: Some(true), + }, + ); SubscribeRequest { slots: slots_subs, From 1ddd4bd24a599568dc3d64009df0085030e9c7ae Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 26 Jan 2024 19:25:57 +0100 Subject: [PATCH 30/32] clippy+fmt --- examples/stream_blocks_autoconnect.rs | 3 +-- examples/stream_blocks_single.rs | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs index dcc4e7b..3e24cb0 100644 --- a/examples/stream_blocks_autoconnect.rs +++ b/examples/stream_blocks_autoconnect.rs @@ -1,9 +1,8 @@ -use futures::{Stream, StreamExt}; +use futures::StreamExt; use log::info; use solana_sdk::clock::Slot; use solana_sdk::commitment_config::CommitmentConfig; use std::env; -use std::pin::pin; use geyser_grpc_connector::channel_plugger::spawn_broadcast_channel_plug; use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task; diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index 9b1e85c..0c29fc6 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -6,9 +6,7 @@ use std::env; use std::pin::pin; use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream; -use geyser_grpc_connector::grpcmultiplex_fastestwins::{ - create_multiplexed_stream, FromYellowstoneExtractor, -}; +use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor; use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message}; use tokio::time::{sleep, Duration}; use tracing::warn; From 5c568a260235a6368523ad4bcbd0c4fed0169568 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 26 Jan 2024 19:32:37 +0100 Subject: [PATCH 31/32] remove grpc_subscription.rs --- src/grpc_subscription.rs | 78 -------------------- src/grpc_subscription_autoreconnect_tasks.rs | 6 +- src/lib.rs | 1 - 3 files changed, 3 insertions(+), 82 deletions(-) delete mode 100644 src/grpc_subscription.rs diff --git a/src/grpc_subscription.rs b/src/grpc_subscription.rs deleted file mode 100644 index 0d0d1ce..0000000 --- a/src/grpc_subscription.rs +++ /dev/null @@ -1,78 +0,0 @@ -use anyhow::{bail, Context}; -use futures::StreamExt; - -use solana_sdk::commitment_config::CommitmentConfig; -use std::collections::HashMap; -use tokio::sync::broadcast::Sender; -use yellowstone_grpc_client::GeyserGrpcClient; -use yellowstone_grpc_proto::prelude::{ - subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequestFilterBlocks, - SubscribeUpdateBlock, -}; - -pub fn create_block_processing_task( - grpc_addr: String, - grpc_x_token: Option, - block_sx: Sender, - commitment_level: CommitmentLevel, -) -> tokio::task::JoinHandle> { - let mut blocks_subs = HashMap::new(); - blocks_subs.insert( - "client".to_string(), - SubscribeRequestFilterBlocks { - account_include: Default::default(), - include_transactions: Some(true), - include_accounts: Some(false), - include_entries: Some(false), - }, - ); - - let _commitment_config = match commitment_level { - CommitmentLevel::Confirmed => CommitmentConfig::confirmed(), - CommitmentLevel::Finalized => CommitmentConfig::finalized(), - CommitmentLevel::Processed => CommitmentConfig::processed(), - }; - - tokio::spawn(async move { - // connect to grpc - let mut client = GeyserGrpcClient::connect(grpc_addr, grpc_x_token, None)?; - let mut stream = client - .subscribe_once( - HashMap::new(), - Default::default(), - HashMap::new(), - Default::default(), - blocks_subs, - Default::default(), - Some(commitment_level), - Default::default(), - None, - ) - .await?; - - while let Some(message) = stream.next().await { - let message = message?; - - let Some(update) = message.update_oneof else { - continue; - }; - - match update { - UpdateOneof::Block(block) => { - // let block = map_produced_block(block, commitment_config); - - block_sx - .send(block) - .context("Grpc failed to send a block")?; - } - UpdateOneof::Ping(_) => { - log::trace!("GRPC Ping"); - } - u => { - bail!("Unexpected update: {u:?}"); - } - }; - } - bail!("geyser slot stream ended"); - }) -} diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index fbc6354..176fcd1 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -291,10 +291,10 @@ pub fn create_geyser_autoconnection_task( break 'recv_loop ConnectionState::WaitReconnect(attempt); } } - } // -- end loop + } // -- END receive loop } - } - } + } // -- END match + } // -- endless state loop }); (jh_geyser_task.abort_handle(), receiver_channel) diff --git a/src/lib.rs b/src/lib.rs index 2a493a9..0cf09c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,6 @@ use yellowstone_grpc_proto::geyser::{ use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig; pub mod channel_plugger; -pub mod grpc_subscription; pub mod grpc_subscription_autoreconnect_streams; pub mod grpc_subscription_autoreconnect_tasks; pub mod grpcmultiplex_fastestwins; From 048ad2be28ae3c9740692bc70884f3a11f01bfc4 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 31 Jan 2024 18:53:07 +0100 Subject: [PATCH 32/32] clippy+fmt --- Cargo.lock | 2 +- Cargo.toml | 2 +- examples/stream_blocks_single.rs | 11 ----------- rust-toolchain.toml | 2 +- src/grpc_subscription_autoreconnect_tasks.rs | 20 +++++++------------- 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26d072e..df3c9b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,7 +1236,7 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" -version = "0.50.1+yellowstone.1.12" +version = "0.10.1+yellowstone.1.12" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 93b0e5b..f4a69f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "geyser-grpc-connector" -version = "0.50.1+yellowstone.1.12" +version = "0.10.1+yellowstone.1.12" edition = "2021" description = "Multiplexing and Reconnection on Yellowstone gRPC Geyser client streaming" diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs index 0c29fc6..4abc002 100644 --- a/examples/stream_blocks_single.rs +++ b/examples/stream_blocks_single.rs @@ -137,17 +137,6 @@ pub async fn main() { warn!("Stream aborted"); }); - // let green_stream = create_geyser_reconnecting_stream( - // green_config.clone(), - // GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(), - // // GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(), - // ); - // let multiplex_stream = create_multiplexed_stream( - // vec![green_stream], - // BlockMiniExtractor(CommitmentConfig::confirmed()), - // ); - // start_example_blockmini_consumer(multiplex_stream); - // "infinite" sleep sleep(Duration::from_secs(1800)).await; } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f400973..8142c30 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.70" +channel = "1.73.0" diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs index 176fcd1..55f6a4b 100644 --- a/src/grpc_subscription_autoreconnect_tasks.rs +++ b/src/grpc_subscription_autoreconnect_tasks.rs @@ -29,8 +29,6 @@ enum FatalErrorReason { ConfigurationError, NetworkError, SubscribeError, - // everything else - Misc, } /// connect to grpc source performing autoconect if required, @@ -101,21 +99,21 @@ pub fn create_geyser_autoconnection_task( } Err(GeyserGrpcClientError::TonicError(tonic_error)) => { warn!( - "! connect failed on {} - aborting: {:?}", + "connect failed on {} - aborting: {:?}", grpc_source, tonic_error ); ConnectionState::FatalError(attempt, FatalErrorReason::NetworkError) } Err(GeyserGrpcClientError::TonicStatus(tonic_status)) => { warn!( - "! connect failed on {} - retrying: {:?}", + "connect failed on {} - retrying: {:?}", grpc_source, tonic_status ); ConnectionState::RecoverableConnectionError(attempt) } Err(GeyserGrpcClientError::SubscribeSendError(send_error)) => { warn!( - "! connect failed with send error on {} - retrying: {:?}", + "connect failed with send error on {} - retrying: {:?}", grpc_source, send_error ); ConnectionState::RecoverableConnectionError(attempt) @@ -149,7 +147,7 @@ pub fn create_geyser_autoconnection_task( // non-recoverable Err(unrecoverable_error) => { error!( - "! subscribe to {} failed with unrecoverable error: {}", + "subscribe to {} failed with unrecoverable error: {}", grpc_source, unrecoverable_error ); ConnectionState::FatalError( @@ -161,7 +159,7 @@ pub fn create_geyser_autoconnection_task( } Err(_elapsed) => { warn!( - "! subscribe failed with timeout on {} - retrying", + "subscribe failed with timeout on {} - retrying", grpc_source ); ConnectionState::RecoverableConnectionError(attempt) @@ -171,7 +169,7 @@ pub fn create_geyser_autoconnection_task( ConnectionState::RecoverableConnectionError(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( - "! waiting {} seconds, then reconnect to {}", + "waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source ); sleep(Duration::from_secs_f32(backoff_secs)).await; @@ -194,15 +192,11 @@ pub fn create_geyser_autoconnection_task( warn!("fatal grpc subscribe error - aborting"); return; } - FatalErrorReason::Misc => { - error!("fatal misc error grpc connection - aborting"); - return; - } }, ConnectionState::WaitReconnect(attempt) => { let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0); info!( - "! waiting {} seconds, then reconnect to {}", + "waiting {} seconds, then reconnect to {}", backoff_secs, grpc_source ); sleep(Duration::from_secs_f32(backoff_secs)).await;