use crate::endpoint_stremers::EndpointStreaming; use crate::grpc::grpc_accounts_streaming::create_grpc_account_streaming; use crate::grpc_multiplex::{ create_grpc_multiplex_blocks_subscription, create_grpc_multiplex_processed_slots_subscription, }; use anyhow::Context; use futures::StreamExt; use geyser_grpc_connector::yellowstone_grpc_util::{ connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig, }; use geyser_grpc_connector::GrpcSourceConfig; use itertools::Itertools; use log::trace; use solana_client::nonblocking::rpc_client::RpcClient; use solana_lite_rpc_core::structures::account_data::AccountNotificationMessage; use solana_lite_rpc_core::structures::account_filter::AccountFilters; use solana_lite_rpc_core::{ structures::produced_block::{ProducedBlock, TransactionInfo}, AnyhowJoinHandle, }; use solana_sdk::program_utils::limited_deserialize; use solana_sdk::vote::instruction::VoteInstruction; use solana_sdk::{ borsh0_10::try_from_slice_unchecked, commitment_config::CommitmentConfig, compute_budget::{self, ComputeBudgetInstruction}, hash::Hash, instruction::CompiledInstruction, message::{ v0::{self, MessageAddressTableLookup}, MessageHeader, VersionedMessage, }, pubkey::Pubkey, signature::Signature, transaction::TransactionError, }; use solana_transaction_status::{Reward, RewardType}; use std::cell::OnceCell; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tokio::sync::{broadcast, Notify}; use tracing::trace_span; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequestFilterBlocks}; use crate::rpc_polling::vote_accounts_and_cluster_info_polling::{ poll_cluster_info, poll_vote_accounts, }; use solana_lite_rpc_core::solana_utils::hash_from_str; use solana_lite_rpc_core::structures::produced_block::ProducedBlockInner; use yellowstone_grpc_proto::prelude::SubscribeUpdateBlock; /// grpc version of ProducedBlock mapping pub fn from_grpc_block_update( block: SubscribeUpdateBlock, commitment_config: CommitmentConfig, ) -> ProducedBlock { let num_transactions = block.transactions.len(); let _span = trace_span!("from_grpc_block_update", ?block.slot, ?num_transactions).entered(); let txs: Vec = block .transactions .into_iter() .filter_map(|tx| { let meta = tx.meta?; let transaction = tx.transaction?; let message = transaction.message?; let header = message.header?; let signature = { let sig_bytes: [u8; 64] = tx.signature.try_into().expect("must map to signature"); Signature::from(sig_bytes) }; let err = meta.err.map(|x| { bincode::deserialize::(&x.err) .expect("TransactionError should be deserialized") }); let compute_units_consumed = meta.compute_units_consumed; let account_keys: Vec = message .account_keys .into_iter() .map(|key_bytes| { let slice: &[u8] = key_bytes.as_slice(); Pubkey::try_from(slice).expect("must map to pubkey") }) .collect(); let message = VersionedMessage::V0(v0::Message { header: MessageHeader { num_required_signatures: header.num_required_signatures as u8, num_readonly_signed_accounts: header.num_readonly_signed_accounts as u8, num_readonly_unsigned_accounts: header.num_readonly_unsigned_accounts as u8, }, account_keys: account_keys.clone(), recent_blockhash: Hash::new(&message.recent_blockhash), instructions: message .instructions .into_iter() .map(|ix| CompiledInstruction { program_id_index: ix.program_id_index as u8, accounts: ix.accounts, data: ix.data, }) .collect(), address_table_lookups: message .address_table_lookups .into_iter() .map(|table| { let slice: &[u8] = table.account_key.as_slice(); let account_key = Pubkey::try_from(slice).expect("must map to pubkey"); MessageAddressTableLookup { account_key, writable_indexes: table.writable_indexes, readonly_indexes: table.readonly_indexes, } }) .collect(), }); let (cu_requested, prioritization_fees) = map_compute_budget_instructions(&message); let is_vote_transaction = message.instructions().iter().any(|i| { i.program_id(message.static_account_keys()) .eq(&solana_sdk::vote::program::id()) && limited_deserialize::(&i.data) .map(|vi| vi.is_simple_vote()) .unwrap_or(false) }); let readable_accounts = account_keys .iter() .enumerate() .filter(|(index, _)| !message.is_maybe_writable(*index)) .map(|(_, pk)| *pk) .collect(); let writable_accounts = account_keys .iter() .enumerate() .filter(|(index, _)| message.is_maybe_writable(*index)) .map(|(_, pk)| *pk) .collect(); let address_lookup_tables = message .address_table_lookups() .map(|x| x.to_vec()) .unwrap_or_default(); Some(TransactionInfo { signature, is_vote: is_vote_transaction, err, cu_requested, prioritization_fees, cu_consumed: compute_units_consumed, recent_blockhash: *message.recent_blockhash(), message, readable_accounts, writable_accounts, address_lookup_tables, }) }) .collect(); let rewards = block.rewards.map(|rewards| { rewards .rewards .into_iter() .map(|reward| Reward { pubkey: reward.pubkey.to_owned(), lamports: reward.lamports, post_balance: reward.post_balance, reward_type: match reward.reward_type() { yellowstone_grpc_proto::prelude::RewardType::Unspecified => None, yellowstone_grpc_proto::prelude::RewardType::Fee => Some(RewardType::Fee), yellowstone_grpc_proto::prelude::RewardType::Rent => Some(RewardType::Rent), yellowstone_grpc_proto::prelude::RewardType::Staking => { Some(RewardType::Staking) } yellowstone_grpc_proto::prelude::RewardType::Voting => Some(RewardType::Voting), }, commission: None, }) .collect_vec() }); let leader_id = if let Some(rewards) = &rewards { rewards .iter() .find(|reward| Some(RewardType::Fee) == reward.reward_type) .map(|leader_reward| leader_reward.pubkey.clone()) } else { None }; let inner = ProducedBlockInner { transactions: txs, block_height: block .block_height .map(|block_height| block_height.block_height) .unwrap(), block_time: block.block_time.map(|time| time.timestamp).unwrap() as u64, blockhash: hash_from_str(&block.blockhash).expect("valid blockhash"), previous_blockhash: hash_from_str(&block.parent_blockhash).expect("valid blockhash"), leader_id, parent_slot: block.parent_slot, slot: block.slot, rewards, }; ProducedBlock::new(inner, commitment_config) } fn map_compute_budget_instructions(message: &VersionedMessage) -> (Option, Option) { let cu_requested_cell: OnceCell = OnceCell::new(); let legacy_cu_requested_cell: OnceCell = OnceCell::new(); let prioritization_fees_cell: OnceCell = OnceCell::new(); let legacy_prio_fees_cell: OnceCell = OnceCell::new(); for compute_budget_ins in message.instructions().iter().filter(|instruction| { instruction .program_id(message.static_account_keys()) .eq(&compute_budget::id()) }) { if let Ok(budget_ins) = try_from_slice_unchecked::(compute_budget_ins.data.as_slice()) { match budget_ins { // aka cu requested ComputeBudgetInstruction::SetComputeUnitLimit(limit) => { cu_requested_cell .set(limit) .expect("cu_limit must be set only once"); } // aka prio fees ComputeBudgetInstruction::SetComputeUnitPrice(price) => { prioritization_fees_cell .set(price) .expect("prioritization_fees must be set only once"); } // legacy ComputeBudgetInstruction::RequestUnitsDeprecated { units, additional_fee, } => { let _ = legacy_cu_requested_cell.set(units); if additional_fee > 0 { let _ = legacy_prio_fees_cell.set(((units * 1000) / additional_fee) as u64); }; } _ => { trace!("skip compute budget instruction"); } } } } let cu_requested = cu_requested_cell .get() .or(legacy_cu_requested_cell.get()) .cloned(); let prioritization_fees = prioritization_fees_cell .get() .or(legacy_prio_fees_cell.get()) .cloned(); (cu_requested, prioritization_fees) } pub fn create_block_processing_task( grpc_addr: String, grpc_x_token: Option, block_sx: tokio::sync::mpsc::Sender, commitment_level: CommitmentLevel, mut exit_notify: broadcast::Receiver<()>, ) -> AnyhowJoinHandle { tokio::spawn(async move { 'main_loop: loop { let mut blocks_subs = HashMap::new(); blocks_subs.insert( "block_client".to_string(), SubscribeRequestFilterBlocks { account_include: Default::default(), include_transactions: Some(true), include_accounts: Some(false), include_entries: Some(false), }, ); // connect to grpc let mut client = connect_with_timeout_with_buffers( grpc_addr.clone(), grpc_x_token.clone(), None, Some(Duration::from_secs(10)), Some(Duration::from_secs(10)), GeyserGrpcClientBufferConfig { buffer_size: Some(65536), conn_window: Some(5242880), stream_window: Some(4194304), }, ) .await?; let mut stream = tokio::select! { res = client .subscribe_once( HashMap::new(), Default::default(), HashMap::new(), Default::default(), blocks_subs, Default::default(), Some(commitment_level), Default::default(), None, ) => { res? }, _ = exit_notify.recv() => { break; } }; loop { tokio::select! { message = stream.next() => { let Some(Ok(message)) = message else { break; }; let Some(update) = message.update_oneof else { continue; }; match update { UpdateOneof::Block(block) => { log::trace!( "received block, hash: {} slot: {}", block.blockhash, block.slot ); block_sx .send(block) .await .context("Problem sending on block channel")?; } UpdateOneof::Ping(_) => { log::trace!("GRPC Ping"); } _ => { log::trace!("unknown GRPC notification"); } }; }, _ = exit_notify.recv() => { break 'main_loop; } } } drop(stream); drop(client); log::error!("Grpc block subscription broken (resubscribing)"); tokio::time::sleep(std::time::Duration::from_secs(1)).await; } Ok(()) }) } pub fn create_grpc_subscription( rpc_client: Arc, grpc_sources: Vec, accounts_filter: AccountFilters, ) -> anyhow::Result<(EndpointStreaming, Vec)> { let (cluster_info_sx, cluster_info_notifier) = tokio::sync::broadcast::channel(10); let (va_sx, vote_account_notifier) = tokio::sync::broadcast::channel(10); // processed slot is required to keep up with leader schedule let (slot_multiplex_channel, jh_multiplex_slotstream) = create_grpc_multiplex_processed_slots_subscription(grpc_sources.clone()); let (block_multiplex_channel, blockmeta_channel, jh_multiplex_blockstream) = create_grpc_multiplex_blocks_subscription(grpc_sources.clone()); let cluster_info_polling = poll_cluster_info(rpc_client.clone(), cluster_info_sx); let vote_accounts_polling = poll_vote_accounts(rpc_client.clone(), va_sx); // accounts if !accounts_filter.is_empty() { let (account_sender, accounts_stream) = tokio::sync::broadcast::channel::(1024); let account_jh = create_grpc_account_streaming( grpc_sources, accounts_filter, account_sender, Arc::new(Notify::new()), ); let streamers = EndpointStreaming { blocks_notifier: block_multiplex_channel, blockinfo_notifier: blockmeta_channel, slot_notifier: slot_multiplex_channel, cluster_info_notifier, vote_account_notifier, processed_account_stream: Some(accounts_stream), }; let endpoint_tasks = vec![ jh_multiplex_slotstream, jh_multiplex_blockstream, cluster_info_polling, vote_accounts_polling, account_jh, ]; Ok((streamers, endpoint_tasks)) } else { let streamers = EndpointStreaming { blocks_notifier: block_multiplex_channel, blockinfo_notifier: blockmeta_channel, slot_notifier: slot_multiplex_channel, cluster_info_notifier, vote_account_notifier, processed_account_stream: None, }; let endpoint_tasks = vec![ jh_multiplex_slotstream, jh_multiplex_blockstream, cluster_info_polling, vote_accounts_polling, ]; Ok((streamers, endpoint_tasks)) } }