diff --git a/Cargo.lock b/Cargo.lock index 9b8e95a20d..8b71dc3fd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4113,6 +4113,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "soketto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74e48087dbeed4833785c2f3352b59140095dc192dce966a3bfc155020a439f" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "futures 0.3.17", + "httparse", + "log 0.4.14", + "rand 0.8.3", + "sha-1 0.9.6", +] + [[package]] name = "solana-account-decoder" version = "1.8.0" @@ -4476,6 +4491,8 @@ dependencies = [ "itertools 0.10.1", "jsonrpc-core", "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-pubsub", "log 0.4.14", "lru", "matches", @@ -5318,6 +5335,7 @@ dependencies = [ "bincode", "bs58 0.4.0", "crossbeam-channel", + "dashmap", "itertools 0.10.1", "jsonrpc-core", "jsonrpc-core-client", @@ -5332,6 +5350,7 @@ dependencies = [ "serde_derive", "serde_json", "serial_test", + "soketto", "solana-account-decoder", "solana-client", "solana-entry", @@ -5353,7 +5372,9 @@ dependencies = [ "solana-version", "solana-vote-program", "spl-token", + "stream-cancel", "symlink", + "thiserror", "tokio", "tokio-util", ] @@ -5910,6 +5931,17 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +[[package]] +name = "stream-cancel" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0a9eb2715209fb8cc0d942fcdff45674bfc9f0090a0d897e85a22955ad159b" +dependencies = [ + "futures-core", + "pin-project 1.0.7", + "tokio", +] + [[package]] name = "strsim" version = "0.8.0" @@ -6428,6 +6460,7 @@ checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" dependencies = [ "bytes 1.0.1", "futures-core", + "futures-io", "futures-sink", "log 0.4.14", "pin-project-lite", diff --git a/account-decoder/src/lib.rs b/account-decoder/src/lib.rs index 31cf56e374..c2e2b3d466 100644 --- a/account-decoder/src/lib.rs +++ b/account-decoder/src/lib.rs @@ -49,7 +49,7 @@ pub enum UiAccountData { Binary(String, UiAccountEncoding), } -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] pub enum UiAccountEncoding { Binary, // Legacy. Retained for RPC backwards compatibility @@ -179,7 +179,7 @@ impl Default for UiFeeCalculator { } } -#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UiDataSliceConfig { pub offset: usize, diff --git a/client/src/rpc_filter.rs b/client/src/rpc_filter.rs index 500f74f9c0..eeb54e5dcf 100644 --- a/client/src/rpc_filter.rs +++ b/client/src/rpc_filter.rs @@ -1,6 +1,6 @@ use thiserror::Error; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum RpcFilterType { DataSize(u64), @@ -40,19 +40,19 @@ pub enum RpcFilterError { Base58DataTooLarge, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum MemcmpEncoding { Binary, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase", untagged)] pub enum MemcmpEncodedBytes { Binary(String), } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct Memcmp { /// Data offset to begin match pub offset: usize, diff --git a/core/Cargo.toml b/core/Cargo.toml index 95477b3bc6..5cd14fec1c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -64,6 +64,8 @@ trees = "0.4.2" [dev-dependencies] jsonrpc-core = "18.0.0" jsonrpc-core-client = { version = "18.0.0", features = ["ipc", "ws"] } +jsonrpc-derive = "18.0.0" +jsonrpc-pubsub = "18.0.0" matches = "0.1.9" reqwest = { version = "0.11.4", default-features = false, features = ["blocking", "rustls-tls", "json"] } serde_json = "1.0.68" @@ -74,6 +76,7 @@ solana-version = { path = "../version", version = "=1.8.0" } static_assertions = "1.1.0" systemstat = "0.1.8" + [build-dependencies] rustc_version = "0.4" @@ -96,4 +99,4 @@ name = "sigverify_stage" name = "retransmit_stage" [package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +targets = ["x86_64-unknown-linux-gnu"] \ No newline at end of file diff --git a/core/src/cluster_info_vote_listener.rs b/core/src/cluster_info_vote_listener.rs index 1c3005d20f..e7fed7a08d 100644 --- a/core/src/cluster_info_vote_listener.rs +++ b/core/src/cluster_info_vote_listener.rs @@ -1530,7 +1530,7 @@ mod tests { let vote_tracker = VoteTracker::new(&bank); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), @@ -1649,7 +1649,7 @@ mod tests { let bank = bank_forks.read().unwrap().get(0).unwrap().clone(); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 7252aa97d0..3825422f8c 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -3014,7 +3014,7 @@ pub mod tests { let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(bank_forks); let exit = Arc::new(AtomicBool::new(false)); - let rpc_subscriptions = Arc::new(RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::default())), @@ -3545,7 +3545,7 @@ pub mod tests { &replay_vote_sender, &VerifyRecyclers::default(), ); - let rpc_subscriptions = Arc::new(RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache, @@ -3613,7 +3613,7 @@ pub mod tests { let exit = Arc::new(AtomicBool::new(false)); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); - let rpc_subscriptions = Arc::new(RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache.clone(), diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 256060570f..45810818fb 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -452,7 +452,7 @@ pub mod tests { }, blockstore, ledger_signal_receiver, - &Arc::new(RpcSubscriptions::new( + &Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache.clone(), diff --git a/core/src/validator.rs b/core/src/validator.rs index 5bbff977f0..9bd7bd072f 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -471,12 +471,12 @@ impl Validator { let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let rpc_subscriptions = Arc::new(RpcSubscriptions::new_with_vote_subscription( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_with_config( &exit, bank_forks.clone(), block_commitment_cache.clone(), optimistically_confirmed_bank.clone(), - config.pubsub_config.enable_vote_subscription, + &config.pubsub_config, )); let max_slots = Arc::new(MaxSlots::default()); @@ -577,12 +577,18 @@ impl Validator { if config.rpc_config.minimal_api { None } else { - Some(PubSubService::new( + let (trigger, pubsub_service) = PubSubService::new( config.pubsub_config.clone(), &rpc_subscriptions, rpc_pubsub_addr, - &exit, - )) + ); + config + .validator_exit + .write() + .unwrap() + .register_exit(Box::new(move || trigger.cancel())); + + Some(pubsub_service) }, Some(OptimisticallyConfirmedBankTracker::new( bank_notification_receiver, diff --git a/core/tests/client.rs b/core/tests/client.rs index 75ea3d75cf..c7692b3cb3 100644 --- a/core/tests/client.rs +++ b/core/tests/client.rs @@ -98,14 +98,14 @@ fn test_slot_subscription() { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), optimistically_confirmed_bank, )); - let pubsub_service = - PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr, &exit); + let (trigger, pubsub_service) = + PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr); std::thread::sleep(Duration::from_millis(400)); let (mut client, receiver) = @@ -138,6 +138,7 @@ fn test_slot_subscription() { } exit.store(true, Ordering::Relaxed); + trigger.cancel(); client.shutdown().unwrap(); pubsub_service.close().unwrap(); diff --git a/core/tests/rpc.rs b/core/tests/rpc.rs index 34fee2df02..47e84565cd 100644 --- a/core/tests/rpc.rs +++ b/core/tests/rpc.rs @@ -1,6 +1,7 @@ use bincode::serialize; use jsonrpc_core::futures::StreamExt; use jsonrpc_core_client::transports::ws; + use log::*; use reqwest::{self, header::CONTENT_TYPE}; use serde_json::{json, Value}; @@ -10,11 +11,12 @@ use solana_client::{ rpc_client::RpcClient, rpc_config::{RpcAccountInfoConfig, RpcSignatureSubscribeConfig}, rpc_request::RpcError, - rpc_response::{Response, RpcSignatureResult, SlotUpdate}, + rpc_response::{Response as RpcResponse, RpcSignatureResult, SlotUpdate}, tpu_client::{TpuClient, TpuClientConfig}, }; use solana_core::test_validator::TestValidator; use solana_rpc::rpc_pubsub::gen_client::Client as PubsubClient; + use solana_sdk::{ commitment_config::CommitmentConfig, hash::Hash, @@ -256,9 +258,9 @@ fn test_rpc_subscriptions() { // Track when subscriptions are ready let (ready_sender, ready_receiver) = channel::<()>(); // Track account notifications are received - let (account_sender, account_receiver) = channel::>(); + let (account_sender, account_receiver) = channel::>(); // Track when status notifications are received - let (status_sender, status_receiver) = channel::<(String, Response)>(); + let (status_sender, status_receiver) = channel::<(String, RpcResponse)>(); // Create the pub sub runtime let rt = Runtime::new().unwrap(); diff --git a/metrics/src/lib.rs b/metrics/src/lib.rs index d0ca7264ce..01c03a4269 100644 --- a/metrics/src/lib.rs +++ b/metrics/src/lib.rs @@ -3,3 +3,61 @@ pub mod counter; pub mod datapoint; mod metrics; pub use crate::metrics::{flush, query, set_host_id, set_panic_hook, submit}; + +use std::sync::Arc; + +/// A helper that sends the count of created tokens as a datapoint. +#[allow(clippy::redundant_allocation)] +pub struct TokenCounter(Arc<&'static str>); + +impl TokenCounter { + /// Creates a new counter with the specified metrics `name`. + pub fn new(name: &'static str) -> Self { + Self(Arc::new(name)) + } + + /// Creates a new token for this counter. The metric's value will be equal + /// to the number of `CounterToken`s. + pub fn create_token(&self) -> CounterToken { + // new_count = strong_count + // - 1 (in TokenCounter) + // + 1 (token that's being created) + datapoint_info!(*self.0, ("count", Arc::strong_count(&self.0), i64)); + CounterToken(self.0.clone()) + } +} + +/// A token for `TokenCounter`. +#[allow(clippy::redundant_allocation)] +pub struct CounterToken(Arc<&'static str>); + +impl Clone for CounterToken { + fn clone(&self) -> Self { + // new_count = strong_count + // - 1 (in TokenCounter) + // + 1 (token that's being created) + datapoint_info!(*self.0, ("count", Arc::strong_count(&self.0), i64)); + CounterToken(self.0.clone()) + } +} + +impl Drop for CounterToken { + fn drop(&mut self) { + // new_count = strong_count + // - 1 (in TokenCounter, if it still exists) + // - 1 (token that's being dropped) + datapoint_info!( + *self.0, + ("count", Arc::strong_count(&self.0).saturating_sub(2), i64) + ); + } +} + +impl Drop for TokenCounter { + fn drop(&mut self) { + datapoint_info!( + *self.0, + ("count", Arc::strong_count(&self.0).saturating_sub(2), i64) + ); + } +} diff --git a/replica-node/src/replica_node.rs b/replica-node/src/replica_node.rs index 3d2df10217..a7a9d2d1a9 100644 --- a/replica-node/src/replica_node.rs +++ b/replica-node/src/replica_node.rs @@ -208,6 +208,17 @@ fn start_client_rpc_services( )); } + let (trigger, pubsub_service) = PubSubService::new( + replica_config.pubsub_config.clone(), + &subscriptions, + replica_config.rpc_pubsub_addr, + ); + replica_config + .replica_exit + .write() + .unwrap() + .register_exit(Box::new(move || trigger.cancel())); + let (_bank_notification_sender, bank_notification_receiver) = unbounded(); ( Some(JsonRpcService::new( @@ -231,18 +242,13 @@ fn start_client_rpc_services( leader_schedule_cache.clone(), max_complete_transaction_status_slot, )), - Some(PubSubService::new( - replica_config.pubsub_config.clone(), - &subscriptions, - replica_config.rpc_pubsub_addr, - &exit, - )), + Some(pubsub_service), Some(OptimisticallyConfirmedBankTracker::new( bank_notification_receiver, &exit, bank_forks.clone(), optimistically_confirmed_bank.clone(), - subscriptions.clone(), + subscriptions, None, )), ) diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 2b03a67baf..2df2be7951 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -14,6 +14,7 @@ base64 = "0.12.3" bincode = "1.3.3" bs58 = "0.4.0" crossbeam-channel = "0.5" +dashmap = "4.0.2" itertools = "0.10.1" jsonrpc-core = "18.0.0" jsonrpc-core-client = { version = "18.0.0", features = ["ipc", "ws"] } @@ -27,6 +28,7 @@ regex = "1.5.4" serde = "1.0.130" serde_derive = "1.0.103" serde_json = "1.0.68" +soketto = "0.6" solana-account-decoder = { path = "../account-decoder", version = "=1.8.0" } solana-client = { path = "../client", version = "=1.8.0" } solana-entry = { path = "../entry", version = "=1.8.0" } @@ -46,8 +48,10 @@ solana-transaction-status = { path = "../transaction-status", version = "=1.8.0" solana-version = { path = "../version", version = "=1.8.0" } solana-vote-program = { path = "../programs/vote", version = "=1.8.0" } spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] } +stream-cancel = "0.8.1" +thiserror = "1.0" tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.6", features = ["codec"] } +tokio-util = { version = "0.6", features = ["codec", "compat"] } [dev-dependencies] serial_test = "0.5.1" diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index ca7e872387..9f2563146a 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -9,6 +9,7 @@ pub mod rpc_health; pub mod rpc_pubsub; pub mod rpc_pubsub_service; pub mod rpc_service; +pub mod rpc_subscription_tracker; pub mod rpc_subscriptions; pub mod transaction_status_service; diff --git a/rpc/src/optimistically_confirmed_bank_tracker.rs b/rpc/src/optimistically_confirmed_bank_tracker.rs index 9e49d1d765..800f9714d2 100644 --- a/rpc/src/optimistically_confirmed_bank_tracker.rs +++ b/rpc/src/optimistically_confirmed_bank_tracker.rs @@ -324,7 +324,7 @@ mod tests { OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache, diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 5580d1efe5..76af567f3b 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -7716,7 +7716,7 @@ pub mod tests { OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache.clone(), diff --git a/rpc/src/rpc_pubsub.rs b/rpc/src/rpc_pubsub.rs index e7755739b5..250cf758b5 100644 --- a/rpc/src/rpc_pubsub.rs +++ b/rpc/src/rpc_pubsub.rs @@ -1,15 +1,20 @@ //! The `pubsub` module implements a threaded subscription service on client RPC request -#[cfg(test)] -use solana_runtime::bank_forks::BankForks; -#[cfg(test)] -use std::sync::RwLock; use { - crate::rpc_subscriptions::{RpcSubscriptions, RpcVote}, + crate::{ + rpc_pubsub_service::PubSubConfig, + rpc_subscription_tracker::{ + AccountSubscriptionParams, LogsSubscriptionKind, LogsSubscriptionParams, + ProgramSubscriptionParams, SignatureSubscriptionParams, SubscriptionControl, + SubscriptionId, SubscriptionParams, SubscriptionToken, + }, + rpc_subscriptions::RpcVote, + }, + dashmap::DashMap, jsonrpc_core::{Error, ErrorCode, Result}, jsonrpc_derive::rpc, - jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId}, - solana_account_decoder::UiAccount, + jsonrpc_pubsub::{typed::Subscriber, SubscriptionId as PubSubSubscriptionId}, + solana_account_decoder::{UiAccount, UiAccountEncoding}, solana_client::{ rpc_config::{ RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig, @@ -21,13 +26,16 @@ use { }, }, solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}, - std::{ - str::FromStr, - sync::{atomic, Arc}, - }, + std::{str::FromStr, sync::Arc}, }; -pub const MAX_ACTIVE_SUBSCRIPTIONS: usize = 100_000; +// We have to keep both of the following traits to not break backwards compatibility. +// `RpcSolPubSubInternal` is actually used by the current PubSub API implementation. +// `RpcSolPubSub` and the corresponding `gen_client` module are preserved +// so the clients reliant on `gen_client::Client` do not break after this implementation is released. +// +// There are no compile-time checks that ensure coherence between traits +// so extra attention is required when adding a new method to the API. // Suppress needless_return due to // https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204 @@ -58,8 +66,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "accountUnsubscribe" )] - fn account_unsubscribe(&self, meta: Option, id: SubscriptionId) - -> Result; + fn account_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get notification every time account data owned by a particular program is changed // Accepts pubkey parameter as base-58 encoded string @@ -82,8 +93,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "programUnsubscribe" )] - fn program_unsubscribe(&self, meta: Option, id: SubscriptionId) - -> Result; + fn program_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get logs for all transactions that reference the specified address #[pubsub(subscription = "logsNotification", subscribe, name = "logsSubscribe")] @@ -101,7 +115,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "logsUnsubscribe" )] - fn logs_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn logs_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get notification when signature is verified // Accepts signature parameter as base-58 encoded string @@ -127,7 +145,7 @@ pub trait RpcSolPubSub { fn signature_unsubscribe( &self, meta: Option, - id: SubscriptionId, + id: PubSubSubscriptionId, ) -> Result; // Get notification when slot is encountered @@ -140,7 +158,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "slotUnsubscribe" )] - fn slot_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn slot_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get series of updates for all slots #[pubsub( @@ -163,7 +185,7 @@ pub trait RpcSolPubSub { fn slots_updates_unsubscribe( &self, meta: Option, - id: SubscriptionId, + id: PubSubSubscriptionId, ) -> Result; // Get notification when vote is encountered @@ -176,7 +198,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "voteUnsubscribe" )] - fn vote_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn vote_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get notification when a new root is set #[pubsub(subscription = "rootNotification", subscribe, name = "rootSubscribe")] @@ -188,53 +214,149 @@ pub trait RpcSolPubSub { unsubscribe, name = "rootUnsubscribe" )] - fn root_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn root_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; +} + +pub use internal::RpcSolPubSubInternal; + +// We have to use a separate module so the code generated by different `rpc` macro invocations do not interfere with each other. +mod internal { + use super::*; + + #[rpc] + pub trait RpcSolPubSubInternal { + // Get notification every time account data is changed + // Accepts pubkey parameter as base-58 encoded string + #[rpc(name = "accountSubscribe")] + fn account_subscribe( + &self, + pubkey_str: String, + config: Option, + ) -> Result; + + // Unsubscribe from account notification subscription. + #[rpc(name = "accountUnsubscribe")] + fn account_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification every time account data owned by a particular program is changed + // Accepts pubkey parameter as base-58 encoded string + #[rpc(name = "programSubscribe")] + fn program_subscribe( + &self, + pubkey_str: String, + config: Option, + ) -> Result; + + // Unsubscribe from account notification subscription. + #[rpc(name = "programUnsubscribe")] + fn program_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get logs for all transactions that reference the specified address + #[rpc(name = "logsSubscribe")] + fn logs_subscribe( + &self, + filter: RpcTransactionLogsFilter, + config: Option, + ) -> Result; + + // Unsubscribe from logs notification subscription. + #[rpc(name = "logsUnsubscribe")] + fn logs_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when signature is verified + // Accepts signature parameter as base-58 encoded string + #[rpc(name = "signatureSubscribe")] + fn signature_subscribe( + &self, + signature_str: String, + config: Option, + ) -> Result; + + // Unsubscribe from signature notification subscription. + #[rpc(name = "signatureUnsubscribe")] + fn signature_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when slot is encountered + #[rpc(name = "slotSubscribe")] + fn slot_subscribe(&self) -> Result; + + // Unsubscribe from slot notification subscription. + #[rpc(name = "slotUnsubscribe")] + fn slot_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get series of updates for all slots + #[rpc(name = "slotsUpdatesSubscribe")] + fn slots_updates_subscribe(&self) -> Result; + + // Unsubscribe from slots updates notification subscription. + #[rpc(name = "slotsUpdatesUnsubscribe")] + fn slots_updates_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when vote is encountered + #[rpc(name = "voteSubscribe")] + fn vote_subscribe(&self) -> Result; + + // Unsubscribe from vote notification subscription. + #[rpc(name = "voteUnsubscribe")] + fn vote_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when a new root is set + #[rpc(name = "rootSubscribe")] + fn root_subscribe(&self) -> Result; + + // Unsubscribe from slot notification subscription. + #[rpc(name = "rootUnsubscribe")] + fn root_unsubscribe(&self, id: SubscriptionId) -> Result; + } } pub struct RpcSolPubSubImpl { - uid: Arc, - subscriptions: Arc, - max_active_subscriptions: usize, + config: PubSubConfig, + subscription_control: SubscriptionControl, + current_subscriptions: Arc>, } impl RpcSolPubSubImpl { - pub fn new(subscriptions: Arc, max_active_subscriptions: usize) -> Self { - let uid = Arc::new(atomic::AtomicUsize::default()); + pub fn new( + config: PubSubConfig, + subscription_control: SubscriptionControl, + current_subscriptions: Arc>, + ) -> Self { Self { - uid, - subscriptions, - max_active_subscriptions, + config, + subscription_control, + current_subscriptions, } } - #[cfg(test)] - fn default_with_bank_forks(bank_forks: Arc>) -> Self { - let uid = Arc::new(atomic::AtomicUsize::default()); - let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); - let max_active_subscriptions = MAX_ACTIVE_SUBSCRIPTIONS; - Self { - uid, - subscriptions, - max_active_subscriptions, - } - } - - fn check_subscription_count(&self) -> Result<()> { - let num_subscriptions = self.subscriptions.total(); - debug!("Total existing subscriptions: {}", num_subscriptions); - if num_subscriptions >= self.max_active_subscriptions { - info!("Node subscription limit reached"); - datapoint_info!("rpc-subscription", ("total", num_subscriptions, i64)); - inc_new_counter_info!("rpc-subscription-refused-limit-reached", 1); - Err(Error { + fn subscribe(&self, params: SubscriptionParams) -> Result { + let token = self + .subscription_control + .subscribe(params) + .map_err(|_| Error { code: ErrorCode::InternalError, message: "Internal Error: Subscription refused. Node subscription limit reached" .into(), data: None, - }) + })?; + let id = token.id(); + self.current_subscriptions.insert(id, token); + Ok(id) + } + + fn unsubscribe(&self, id: SubscriptionId) -> Result { + if self.current_subscriptions.remove(&id).is_some() { + Ok(true) } else { - datapoint_info!("rpc-subscription", ("total", num_subscriptions + 1, i64)); - Ok(()) + Err(Error { + code: ErrorCode::InvalidParams, + message: "Invalid subscription id.".into(), + data: None, + }) } } } @@ -247,325 +369,150 @@ fn param(param_str: &str, thing: &str) -> Result { }) } -impl RpcSolPubSub for RpcSolPubSubImpl { - type Metadata = Arc; - +impl RpcSolPubSubInternal for RpcSolPubSubImpl { fn account_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, pubkey_str: String, config: Option, - ) { - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - match param::(&pubkey_str, "pubkey") { - Ok(pubkey) => { - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("account_subscribe: account={:?} id={:?}", pubkey, sub_id); - self.subscriptions - .add_account_subscription(pubkey, config, sub_id, subscriber) - } - Err(e) => subscriber.reject(e).unwrap_or_default(), - } + ) -> Result { + let config = config.unwrap_or_default(); + let params = AccountSubscriptionParams { + pubkey: param::(&pubkey_str, "pubkey")?, + commitment: config.commitment.unwrap_or_default(), + data_slice: config.data_slice, + encoding: config.encoding.unwrap_or(UiAccountEncoding::Binary), + }; + self.subscribe(SubscriptionParams::Account(params)) } - fn account_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("account_unsubscribe: id={:?}", id); - if self.subscriptions.remove_account_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn account_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } fn program_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, pubkey_str: String, config: Option, - ) { - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - match param::(&pubkey_str, "pubkey") { - Ok(pubkey) => { - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("program_subscribe: account={:?} id={:?}", pubkey, sub_id); - self.subscriptions - .add_program_subscription(pubkey, config, sub_id, subscriber) - } - Err(e) => subscriber.reject(e).unwrap_or_default(), - } + ) -> Result { + let config = config.unwrap_or_default(); + let params = ProgramSubscriptionParams { + pubkey: param::(&pubkey_str, "pubkey")?, + filters: config.filters.unwrap_or_default(), + encoding: config + .account_config + .encoding + .unwrap_or(UiAccountEncoding::Binary), + data_slice: config.account_config.data_slice, + commitment: config.account_config.commitment.unwrap_or_default(), + with_context: config.with_context.unwrap_or_default(), + }; + self.subscribe(SubscriptionParams::Program(params)) } - fn program_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("program_unsubscribe: id={:?}", id); - if self.subscriptions.remove_program_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn program_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } fn logs_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, filter: RpcTransactionLogsFilter, config: Option, - ) { - info!("logs_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - - let (address, include_votes) = match filter { - RpcTransactionLogsFilter::All => (None, false), - RpcTransactionLogsFilter::AllWithVotes => (None, true), - RpcTransactionLogsFilter::Mentions(addresses) => { - match addresses.len() { - 1 => match param::(&addresses[0], "mentions") { - Ok(address) => (Some(address), false), - Err(e) => { - subscriber.reject(e).unwrap_or_default(); - return; - } - }, - _ => { - // Room is reserved in the API to support multiple addresses, but for now - // the implementation only supports one - subscriber - .reject(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Only 1 address supported".into(), - data: None, - }) - .unwrap_or_default(); - return; + ) -> Result { + let params = LogsSubscriptionParams { + kind: match filter { + RpcTransactionLogsFilter::All => LogsSubscriptionKind::All, + RpcTransactionLogsFilter::AllWithVotes => LogsSubscriptionKind::AllWithVotes, + RpcTransactionLogsFilter::Mentions(keys) => { + if keys.len() != 1 { + return Err(Error { + code: ErrorCode::InvalidParams, + message: "Invalid Request: Only 1 address supported".into(), + data: None, + }); } + LogsSubscriptionKind::Single(param::(&keys[0], "mentions")?) } - } + }, + commitment: config.and_then(|c| c.commitment).unwrap_or_default(), }; - - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - self.subscriptions.add_logs_subscription( - address, - include_votes, - config.and_then(|config| config.commitment), - sub_id, - subscriber, - ) + self.subscribe(SubscriptionParams::Logs(params)) } - fn logs_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("logs_unsubscribe: id={:?}", id); - if self.subscriptions.remove_logs_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn logs_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } fn signature_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, signature_str: String, - signature_subscribe_config: Option, - ) { - info!("signature_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - match param::(&signature_str, "signature") { - Ok(signature) => { - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!( - "signature_subscribe: signature={:?} id={:?}", - signature, sub_id - ); - self.subscriptions.add_signature_subscription( - signature, - signature_subscribe_config, - sub_id, - subscriber, - ); - } - Err(e) => subscriber.reject(e).unwrap_or_default(), - } + config: Option, + ) -> Result { + let config = config.unwrap_or_default(); + let params = SignatureSubscriptionParams { + signature: param::(&signature_str, "signature")?, + commitment: config.commitment.unwrap_or_default(), + enable_received_notification: config.enable_received_notification.unwrap_or_default(), + }; + self.subscribe(SubscriptionParams::Signature(params)) } - fn signature_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("signature_unsubscribe"); - if self.subscriptions.remove_signature_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn signature_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } - fn slot_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber) { - info!("slot_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("slot_subscribe: id={:?}", sub_id); - self.subscriptions.add_slot_subscription(sub_id, subscriber); + fn slot_subscribe(&self) -> Result { + self.subscribe(SubscriptionParams::Slot) } - fn slot_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("slot_unsubscribe"); - if self.subscriptions.remove_slot_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn slot_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } - fn slots_updates_subscribe( - &self, - _meta: Self::Metadata, - subscriber: Subscriber>, - ) { - info!("slots_updates_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("slots_updates_subscribe: id={:?}", sub_id); - self.subscriptions - .add_slots_updates_subscription(sub_id, subscriber); + fn slots_updates_subscribe(&self) -> Result { + self.subscribe(SubscriptionParams::SlotsUpdates) } - fn slots_updates_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("slots_updates_unsubscribe"); - if self.subscriptions.remove_slots_updates_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn slots_updates_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } - fn vote_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber) { - info!("vote_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; + fn vote_subscribe(&self) -> Result { + if !self.config.enable_vote_subscription { + return Err(Error::new(jsonrpc_core::ErrorCode::MethodNotFound)); } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("vote_subscribe: id={:?}", sub_id); - self.subscriptions.add_vote_subscription(sub_id, subscriber); + self.subscribe(SubscriptionParams::Vote) } - fn vote_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("vote_unsubscribe"); - if self.subscriptions.remove_vote_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) + fn vote_unsubscribe(&self, id: SubscriptionId) -> Result { + if !self.config.enable_vote_subscription { + return Err(Error::new(jsonrpc_core::ErrorCode::MethodNotFound)); } + self.unsubscribe(id) } - fn root_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber) { - info!("root_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("root_subscribe: id={:?}", sub_id); - self.subscriptions.add_root_subscription(sub_id, subscriber); + fn root_subscribe(&self) -> Result { + self.subscribe(SubscriptionParams::Root) } - fn root_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("root_unsubscribe"); - if self.subscriptions.remove_root_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn root_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } } #[cfg(test)] mod tests { use { - super::*, + super::{RpcSolPubSubInternal, *}, crate::{ - optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, - rpc_subscriptions::tests::robust_poll_or_panic, + optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, rpc_pubsub_service, + rpc_subscriptions::RpcSubscriptions, }, - jsonrpc_core::{futures::channel::mpsc, Response}, - jsonrpc_pubsub::{PubSubHandler, Session}, + jsonrpc_core::{IoHandler, Response}, serial_test::serial, solana_account_decoder::{parse_account_data::parse_account_data, UiAccountEncoding}, - solana_client::rpc_response::{ProcessedSignatureResult, ReceivedSignatureResult}, + solana_client::rpc_response::{ + ProcessedSignatureResult, ReceivedSignatureResult, RpcSignatureResult, SlotInfo, + }, solana_runtime::{ bank::Bank, bank_forks::BankForks, @@ -577,6 +524,7 @@ mod tests { }, solana_sdk::{ account::ReadableAccount, + clock::Slot, commitment_config::CommitmentConfig, hash::Hash, message::Message, @@ -618,10 +566,6 @@ mod tests { Ok(()) } - fn create_session() -> Arc { - Arc::new(Session::new(mpsc::unbounded().0)) - } - #[test] #[serial] fn test_signature_subscribe() { @@ -635,36 +579,31 @@ mod tests { let bank = Bank::new_for_tests(&genesis_config); let blockhash = bank.last_blockhash(); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl { - subscriptions: Arc::new(RpcSubscriptions::new( - &Arc::new(AtomicBool::new(false)), - bank_forks.clone(), - Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), - OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - )), - uid: Arc::new(atomic::AtomicUsize::default()), - max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, - }; + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( + &Arc::new(AtomicBool::new(false)), + bank_forks.clone(), + Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), + )); // Test signature subscriptions let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification"); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.signature_subscribe( - session, - subscriber, tx.signatures[0].to_string(), Some(RpcSignatureSubscribeConfig { commitment: Some(CommitmentConfig::finalized()), ..RpcSignatureSubscribeConfig::default() }), - ); + ) + .unwrap(); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 0).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 0).unwrap(); // Test signature confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = RpcSignatureResult::ProcessedSignature(ProcessedSignatureResult { err: None }); let expected = json!({ @@ -679,26 +618,27 @@ mod tests { } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); // Test "received" - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification"); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.signature_subscribe( - session, - subscriber, tx.signatures[0].to_string(), Some(RpcSignatureSubscribeConfig { commitment: Some(CommitmentConfig::finalized()), enable_received_notification: Some(true), }), - ); + ) + .unwrap(); let received_slot = 1; - rpc.subscriptions - .notify_signatures_received((received_slot, vec![tx.signatures[0]])); + rpc_subscriptions.notify_signatures_received((received_slot, vec![tx.signatures[0]])); // Test signature confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = RpcSignatureResult::ReceivedSignature(ReceivedSignatureResult::ReceivedSignature); let expected = json!({ @@ -712,26 +652,27 @@ mod tests { "subscription": 1, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); // Test "received" for gossip subscription - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification"); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.signature_subscribe( - session, - subscriber, tx.signatures[0].to_string(), Some(RpcSignatureSubscribeConfig { commitment: Some(CommitmentConfig::confirmed()), enable_received_notification: Some(true), }), - ); + ) + .unwrap(); let received_slot = 2; - rpc.subscriptions - .notify_signatures_received((received_slot, vec![tx.signatures[0]])); + rpc_subscriptions.notify_signatures_received((received_slot, vec![tx.signatures[0]])); // Test signature confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = RpcSignatureResult::ReceivedSignature(ReceivedSignatureResult::ReceivedSignature); let expected = json!({ @@ -745,7 +686,10 @@ mod tests { "subscription": 2, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); } #[test] @@ -761,10 +705,10 @@ mod tests { let blockhash = bank.last_blockhash(); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let session = create_session(); + let mut io = IoHandler::<()>::default(); + let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, _receiver) = rpc_pubsub_service::test_connection(&subscriptions); - let mut io = PubSubHandler::default(); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); io.extend_with(rpc.to_delegate()); let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash); @@ -772,10 +716,10 @@ mod tests { r#"{{"jsonrpc":"2.0","id":1,"method":"signatureSubscribe","params":["{}"]}}"#, tx.signatures[0].to_string() ); - let _res = io.handle_request_sync(&req, session.clone()); + let _res = io.handle_request_sync(&req); let req = r#"{"jsonrpc":"2.0","id":1,"method":"signatureUnsubscribe","params":[0]}"#; - let res = io.handle_request_sync(req, session.clone()); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","result":true,"id":1}"#; let expected: Response = serde_json::from_str(expected).unwrap(); @@ -785,7 +729,7 @@ mod tests { // Test bad parameter let req = r#"{"jsonrpc":"2.0","id":1,"method":"signatureUnsubscribe","params":[1]}"#; - let res = io.handle_request_sync(req, session); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid subscription id."},"id":1}"#; let expected: Response = serde_json::from_str(expected).unwrap(); @@ -814,34 +758,37 @@ mod tests { let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); bank_forks.write().unwrap().insert(bank1); - let rpc = RpcSolPubSubImpl { - subscriptions: Arc::new(RpcSubscriptions::new( - &Arc::new(AtomicBool::new(false)), - bank_forks.clone(), - Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( - 1, 1, - ))), - OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - )), - uid: Arc::new(atomic::AtomicUsize::default()), - max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, - }; - let session = create_session(); + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( + &Arc::new(AtomicBool::new(false)), + bank_forks.clone(), + Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( + 1, 1, + ))), + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), + )); + + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + let encoding = UiAccountEncoding::Base64; - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + rpc.account_subscribe( - session, - subscriber, stake_account.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::processed()), encoding: Some(encoding), data_slice: None, }), - ); + ) + .unwrap(); + + // Make sure the subscription is processed before continuing. + let (rpc2, mut receiver2) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc2.slot_subscribe().unwrap(); + rpc_subscriptions.notify_slot(1, 0, 0); + receiver2.recv(); let tx = system_transaction::transfer(&alice, &from.pubkey(), 51, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); let authorized = Authorized::auto(&stake_authority.pubkey()); let ixs = stake_instruction::create_account( @@ -853,8 +800,7 @@ mod tests { ); let message = Message::new(&ixs, Some(&from.pubkey())); let tx = Transaction::new(&[&from, &stake_account], message, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); - sleep(Duration::from_millis(200)); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); // Test signature confirmation notification #1 let account = bank_forks @@ -883,11 +829,14 @@ mod tests { } }); - let (response, _) = robust_poll_or_panic(receiver); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + let response = receiver.recv(); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); let tx = system_transaction::transfer(&alice, &stake_authority.pubkey(), 1, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); sleep(Duration::from_millis(200)); let ix = stake_instruction::authorize( &stake_account.pubkey(), @@ -898,7 +847,7 @@ mod tests { ); let message = Message::new(&[ix], Some(&stake_authority.pubkey())); let tx = Transaction::new(&[&stake_authority], message, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); sleep(Duration::from_millis(200)); let bank = bank_forks.read().unwrap()[1].clone(); @@ -926,30 +875,32 @@ mod tests { let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); bank_forks.write().unwrap().insert(bank1); - let rpc = RpcSolPubSubImpl { - subscriptions: Arc::new(RpcSubscriptions::new( - &Arc::new(AtomicBool::new(false)), - bank_forks.clone(), - Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( - 1, 1, - ))), - OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - )), - uid: Arc::new(atomic::AtomicUsize::default()), - max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, - }; - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( + &Arc::new(AtomicBool::new(false)), + bank_forks.clone(), + Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( + 1, 1, + ))), + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), + )); + + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.account_subscribe( - session, - subscriber, nonce_account.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::processed()), encoding: Some(UiAccountEncoding::JsonParsed), data_slice: None, }), - ); + ) + .unwrap(); + + // Make sure the subscription is processed before continuing. + let (rpc2, mut receiver2) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc2.slot_subscribe().unwrap(); + rpc_subscriptions.notify_slot(1, 0, 0); + receiver2.recv(); let ixs = system_instruction::create_nonce_account( &alice.pubkey(), @@ -959,8 +910,7 @@ mod tests { ); let message = Message::new(&ixs, Some(&alice.pubkey())); let tx = Transaction::new(&[&alice, &nonce_account], message, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); - sleep(Duration::from_millis(200)); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); // Test signature confirmation notification #1 let account = bank_forks @@ -996,22 +946,26 @@ mod tests { } }); - let (response, _) = robust_poll_or_panic(receiver); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + let response = receiver.recv(); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); } #[test] #[serial] fn test_account_unsubscribe() { let bob_pubkey = solana_sdk::pubkey::new_rand(); - let session = create_session(); + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new_for_tests( &genesis_config, )))); - let mut io = PubSubHandler::default(); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); + let mut io = IoHandler::<()>::default(); + let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, _receiver) = rpc_pubsub_service::test_connection(&subscriptions); io.extend_with(rpc.to_delegate()); @@ -1019,10 +973,10 @@ mod tests { r#"{{"jsonrpc":"2.0","id":1,"method":"accountSubscribe","params":["{}"]}}"#, bob_pubkey.to_string() ); - let _res = io.handle_request_sync(&req, session.clone()); + let _res = io.handle_request_sync(&req); let req = r#"{"jsonrpc":"2.0","id":1,"method":"accountUnsubscribe","params":[0]}"#; - let res = io.handle_request_sync(req, session.clone()); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","result":true,"id":1}"#; let expected: Response = serde_json::from_str(expected).unwrap(); @@ -1032,7 +986,7 @@ mod tests { // Test bad parameter let req = r#"{"jsonrpc":"2.0","id":1,"method":"accountUnsubscribe","params":[1]}"#; - let res = io.handle_request_sync(req, session); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid subscription id."},"id":1}"#; let expected: Response = serde_json::from_str(expected).unwrap(); @@ -1053,27 +1007,25 @@ mod tests { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let bob = Keypair::new(); - let mut rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks.clone()); let exit = Arc::new(AtomicBool::new(false)); - let subscriptions = RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - ); - rpc.subscriptions = Arc::new(subscriptions); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + )); + + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.account_subscribe( - session, - subscriber, bob.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::finalized()), encoding: None, data_slice: None, }), - ); + ) + .unwrap(); let tx = system_transaction::transfer(&alice, &bob.pubkey(), 100, blockhash); bank_forks @@ -1083,11 +1035,11 @@ mod tests { .unwrap() .process_transaction(&tx) .unwrap(); - rpc.subscriptions - .notify_subscribers(CommitmentSlots::default()); + rpc_subscriptions.notify_subscribers(CommitmentSlots::default()); + // allow 200ms for notification thread to wake std::thread::sleep(Duration::from_millis(200)); - let _panic = robust_poll_or_panic(receiver); + let _panic = receiver.recv(); } #[test] @@ -1105,29 +1057,26 @@ mod tests { bank_forks.write().unwrap().insert(bank1); let bob = Keypair::new(); - let mut rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks.clone()); let exit = Arc::new(AtomicBool::new(false)); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache, OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - ); - rpc.subscriptions = Arc::new(subscriptions); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + rpc.account_subscribe( - session, - subscriber, bob.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::finalized()), encoding: None, data_slice: None, }), - ); + ) + .unwrap(); let tx = system_transaction::transfer(&alice, &bob.pubkey(), 100, blockhash); bank_forks @@ -1141,7 +1090,7 @@ mod tests { slot: 1, ..CommitmentSlots::default() }; - rpc.subscriptions.notify_subscribers(commitment_slots); + subscriptions.notify_subscribers(commitment_slots); let commitment_slots = CommitmentSlots { slot: 2, @@ -1149,7 +1098,7 @@ mod tests { highest_confirmed_slot: 1, highest_confirmed_root: 1, }; - rpc.subscriptions.notify_subscribers(commitment_slots); + subscriptions.notify_subscribers(commitment_slots); let expected = json!({ "jsonrpc": "2.0", "method": "accountNotification", @@ -1167,8 +1116,11 @@ mod tests { "subscription": 0, } }); - let (response, _) = robust_poll_or_panic(receiver); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + let response = receiver.recv(); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); } #[test] @@ -1177,21 +1129,21 @@ mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification"); - rpc.slot_subscribe(session, subscriber); + let rpc_subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.slot_subscribe().unwrap(); + + rpc_subscriptions.notify_slot(0, 0, 0); - rpc.subscriptions.notify_slot(0, 0, 0); // Test slot confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = SlotInfo { parent: 0, slot: 0, root: 0, }; - let expected_res_str = - serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap(); + let expected_res_str = serde_json::to_string(&expected_res).unwrap(); + let expected = format!( r#"{{"jsonrpc":"2.0","method":"slotNotification","params":{{"result":{},"subscription":0}}}}"#, expected_res_str @@ -1205,34 +1157,27 @@ mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification"); - rpc.slot_subscribe(session, subscriber); - rpc.subscriptions.notify_slot(0, 0, 0); - let (response, _) = robust_poll_or_panic(receiver); + let rpc_subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + let sub_id = rpc.slot_subscribe().unwrap(); + + rpc_subscriptions.notify_slot(0, 0, 0); + let response = receiver.recv(); let expected_res = SlotInfo { parent: 0, slot: 0, root: 0, }; - let expected_res_str = - serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap(); + let expected_res_str = serde_json::to_string(&expected_res).unwrap(); + let expected = format!( r#"{{"jsonrpc":"2.0","method":"slotNotification","params":{{"result":{},"subscription":0}}}}"#, expected_res_str ); assert_eq!(expected, response); - let session = create_session(); - assert!(rpc - .slot_unsubscribe(Some(session), SubscriptionId::Number(42)) - .is_err()); - - let session = create_session(); - assert!(rpc - .slot_unsubscribe(Some(session), SubscriptionId::Number(0)) - .is_ok()); + assert!(rpc.slot_unsubscribe(42.into()).is_err()); + assert!(rpc.slot_unsubscribe(sub_id).is_ok()); } #[test] @@ -1251,23 +1196,18 @@ mod tests { let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - // Setup RPC - let mut rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks.clone()); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("voteNotification"); - // Setup Subscriptions let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new_with_vote_subscription( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, block_commitment_cache, optimistically_confirmed_bank, - true, )); - rpc.subscriptions = subscriptions.clone(); - rpc.vote_subscribe(session, subscriber); + // Setup RPC + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + rpc.vote_subscribe().unwrap(); let vote = Vote { slots: vec![1, 2], @@ -1276,10 +1216,10 @@ mod tests { }; subscriptions.notify_vote(&vote); - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); assert_eq!( response, - r#"{"jsonrpc":"2.0","method":"voteNotification","params":{"result":{"hash":"11111111111111111111111111111111","slots":[1,2],"timestamp":null},"subscription":0}}"# + r#"{"jsonrpc":"2.0","method":"voteNotification","params":{"result":{"slots":[1,2],"hash":"11111111111111111111111111111111","timestamp":null},"subscription":0}}"# ); } @@ -1289,19 +1229,11 @@ mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); - let session = create_session(); - let (subscriber, _id_receiver, _) = Subscriber::new_test("voteNotification"); - rpc.vote_subscribe(session, subscriber); + let rpc_subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, _receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + let sub_id = rpc.vote_subscribe().unwrap(); - let session = create_session(); - assert!(rpc - .vote_unsubscribe(Some(session), SubscriptionId::Number(42)) - .is_err()); - - let session = create_session(); - assert!(rpc - .vote_unsubscribe(Some(session), SubscriptionId::Number(0)) - .is_ok()); + assert!(rpc.vote_unsubscribe(42.into()).is_err()); + assert!(rpc.vote_unsubscribe(sub_id).is_ok()); } } diff --git a/rpc/src/rpc_pubsub_service.rs b/rpc/src/rpc_pubsub_service.rs index 4eca210a88..33fa57f1df 100644 --- a/rpc/src/rpc_pubsub_service.rs +++ b/rpc/src/rpc_pubsub_service.rs @@ -2,45 +2,60 @@ use { crate::{ - rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl, MAX_ACTIVE_SUBSCRIPTIONS}, - rpc_subscriptions::RpcSubscriptions, - }, - jsonrpc_pubsub::{PubSubHandler, Session}, - jsonrpc_ws_server::{RequestContext, ServerBuilder}, - std::{ - net::SocketAddr, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, + rpc_pubsub::{RpcSolPubSubImpl, RpcSolPubSubInternal}, + rpc_subscription_tracker::{ + SubscriptionControl, SubscriptionId, SubscriptionParams, SubscriptionToken, }, - thread::{self, sleep, Builder, JoinHandle}, - time::Duration, + rpc_subscriptions::{RpcNotification, RpcSubscriptions}, }, + dashmap::{mapref::entry::Entry, DashMap}, + jsonrpc_core::IoHandler, + soketto::handshake::{server, Server}, + solana_metrics::TokenCounter, + std::{ + io, + net::SocketAddr, + str, + sync::Arc, + thread::{self, Builder, JoinHandle}, + }, + stream_cancel::{Trigger, Tripwire}, + thiserror::Error, + tokio::{net::TcpStream, pin, select, sync::broadcast}, + tokio_util::compat::TokioAsyncReadCompatExt, }; +pub const MAX_ACTIVE_SUBSCRIPTIONS: usize = 1_000_000; +pub const DEFAULT_QUEUE_CAPACITY_ITEMS: usize = 10_000_000; +pub const DEFAULT_TEST_QUEUE_CAPACITY_ITEMS: usize = 100; +pub const DEFAULT_QUEUE_CAPACITY_BYTES: usize = 256 * 1024 * 1024; + #[derive(Debug, Clone)] pub struct PubSubConfig { pub enable_vote_subscription: bool, - - // See the corresponding fields in - // https://github.com/paritytech/ws-rs/blob/be4d47575bae55c60d9f51b47480d355492a94fc/src/lib.rs#L131 - // for a complete description of each field in this struct - pub max_connections: usize, - pub max_fragment_size: usize, - pub max_in_buffer_capacity: usize, - pub max_out_buffer_capacity: usize, pub max_active_subscriptions: usize, + pub queue_capacity_items: usize, + pub queue_capacity_bytes: usize, } impl Default for PubSubConfig { fn default() -> Self { Self { enable_vote_subscription: false, - max_connections: 1000, // Arbitrary, default of 100 is too low - max_fragment_size: 50 * 1024, // 50KB - max_in_buffer_capacity: 50 * 1024, // 50KB - max_out_buffer_capacity: 15 * 1024 * 1024, // max account size (10MB), then 5MB extra for base64 encoding overhead/etc max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, + queue_capacity_items: DEFAULT_QUEUE_CAPACITY_ITEMS, + queue_capacity_bytes: DEFAULT_QUEUE_CAPACITY_BYTES, + } + } +} + +impl PubSubConfig { + pub fn default_for_tests() -> Self { + Self { + enable_vote_subscription: false, + max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, + queue_capacity_items: DEFAULT_TEST_QUEUE_CAPACITY_ITEMS, + queue_capacity_bytes: DEFAULT_QUEUE_CAPACITY_BYTES, } } } @@ -54,51 +69,30 @@ impl PubSubService { pubsub_config: PubSubConfig, subscriptions: &Arc, pubsub_addr: SocketAddr, - exit: &Arc, - ) -> Self { + ) -> (Trigger, Self) { + let subscription_control = subscriptions.control().clone(); info!("rpc_pubsub bound to {:?}", pubsub_addr); - let rpc = RpcSolPubSubImpl::new( - subscriptions.clone(), - pubsub_config.max_active_subscriptions, - ); - let exit_ = exit.clone(); + let (trigger, tripwire) = Tripwire::new(); let thread_hdl = Builder::new() .name("solana-pubsub".to_string()) .spawn(move || { - let mut io = PubSubHandler::default(); - io.extend_with(rpc.to_delegate()); - - let server = ServerBuilder::with_meta_extractor(io, |context: &RequestContext| { - info!("New pubsub connection"); - let session = Arc::new(Session::new(context.sender())); - session.on_drop(|| { - info!("Pubsub connection dropped"); - }); - session - }) - .max_connections(pubsub_config.max_connections) - .max_payload(pubsub_config.max_fragment_size) - .max_in_buffer_capacity(pubsub_config.max_in_buffer_capacity) - .max_out_buffer_capacity(pubsub_config.max_out_buffer_capacity) - .start(&pubsub_addr); - - if let Err(e) = server { - warn!( - "Pubsub service unavailable error: {:?}. \n\ - Also, check that port {} is not already in use by another application", - e, - pubsub_addr.port() - ); - return; - } - while !exit_.load(Ordering::Relaxed) { - sleep(Duration::from_millis(100)); - } - server.unwrap().close(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime creation failed"); + if let Err(err) = runtime.block_on(listen( + pubsub_addr, + pubsub_config, + subscription_control, + tripwire, + )) { + error!("pubsub service failed: {}", err); + }; }) - .unwrap(); - Self { thread_hdl } + .expect("thread spawn failed"); + + (trigger, Self { thread_hdl }) } pub fn close(self) -> thread::Result<()> { @@ -110,6 +104,244 @@ impl PubSubService { } } +struct BroadcastHandler { + current_subscriptions: Arc>, +} + +fn count_final(params: &SubscriptionParams) { + match params { + SubscriptionParams::Account(_) => { + inc_new_counter_info!("rpc-pubsub-final-accounts", 1); + } + SubscriptionParams::Logs(_) => { + inc_new_counter_info!("rpc-pubsub-final-logs", 1); + } + SubscriptionParams::Program(_) => { + inc_new_counter_info!("rpc-pubsub-final-programs", 1); + } + SubscriptionParams::Signature(_) => { + inc_new_counter_info!("rpc-pubsub-final-signatures", 1); + } + SubscriptionParams::Slot => { + inc_new_counter_info!("rpc-pubsub-final-slots", 1); + } + SubscriptionParams::SlotsUpdates => { + inc_new_counter_info!("rpc-pubsub-final-slots-updates", 1); + } + SubscriptionParams::Root => { + inc_new_counter_info!("rpc-pubsub-final-roots", 1); + } + SubscriptionParams::Vote => { + inc_new_counter_info!("rpc-pubsub-final-votes", 1); + } + } +} + +impl BroadcastHandler { + fn handle(&self, notification: RpcNotification) -> Result>, Error> { + if let Entry::Occupied(entry) = self + .current_subscriptions + .entry(notification.subscription_id) + { + count_final(entry.get().params()); + + if notification.is_final { + entry.remove(); + } + notification + .json + .upgrade() + .ok_or(Error::NotificationIsGone) + .map(Some) + } else { + Ok(None) + } + } +} + +#[cfg(test)] +pub struct TestBroadcastReceiver { + handler: BroadcastHandler, + inner: tokio::sync::broadcast::Receiver, +} + +#[cfg(test)] +impl TestBroadcastReceiver { + pub fn recv(&mut self) -> String { + use std::thread::sleep; + use std::time::{Duration, Instant}; + use tokio::sync::broadcast::error::TryRecvError; + + let timeout = Duration::from_millis(500); + let started = Instant::now(); + + loop { + match self.inner.try_recv() { + Ok(notification) => { + if let Some(json) = self.handler.handle(notification).expect("handler failed") { + return json.to_string(); + } + } + Err(TryRecvError::Empty) => { + if started.elapsed() > timeout { + panic!("TestBroadcastReceiver: no data, timeout reached"); + } + sleep(Duration::from_millis(50)); + } + Err(err) => panic!("broadcast receiver error: {}", err), + } + } + } +} + +#[cfg(test)] +pub fn test_connection( + subscriptions: &Arc, +) -> (RpcSolPubSubImpl, TestBroadcastReceiver) { + let current_subscriptions = Arc::new(DashMap::new()); + + let rpc_impl = RpcSolPubSubImpl::new( + PubSubConfig { + enable_vote_subscription: true, + queue_capacity_items: 100, + ..PubSubConfig::default() + }, + subscriptions.control().clone(), + Arc::clone(¤t_subscriptions), + ); + let broadcast_handler = BroadcastHandler { + current_subscriptions, + }; + let receiver = TestBroadcastReceiver { + inner: subscriptions.control().broadcast_receiver(), + handler: broadcast_handler, + }; + (rpc_impl, receiver) +} + +#[derive(Debug, Error)] +enum Error { + #[error("handshake error: {0}")] + Handshake(#[from] soketto::handshake::Error), + #[error("connection error: {0}")] + Connection(#[from] soketto::connection::Error), + #[error("broadcast queue error: {0}")] + Broadcast(#[from] broadcast::error::RecvError), + #[error("client has lagged behind (notification is gone)")] + NotificationIsGone, +} + +async fn handle_connection( + socket: TcpStream, + subscription_control: SubscriptionControl, + config: PubSubConfig, + mut tripwire: Tripwire, +) -> Result<(), Error> { + let mut server = Server::new(socket.compat()); + let request = server.receive_request().await?; + let accept = server::Response::Accept { + key: request.key(), + protocol: None, + }; + server.send_response(&accept).await?; + let (mut sender, mut receiver) = server.into_builder().finish(); + + let mut broadcast_receiver = subscription_control.broadcast_receiver(); + let mut data = Vec::new(); + let current_subscriptions = Arc::new(DashMap::new()); + + let mut json_rpc_handler = IoHandler::new(); + let rpc_impl = RpcSolPubSubImpl::new( + config, + subscription_control, + Arc::clone(¤t_subscriptions), + ); + json_rpc_handler.extend_with(rpc_impl.to_delegate()); + let broadcast_handler = BroadcastHandler { + current_subscriptions, + }; + loop { + // Extra block for dropping `receive_future`. + { + // soketto is not cancel safe, so we have to introduce an inner loop to poll + // `receive_data` to completion. + let receive_future = receiver.receive_data(&mut data); + pin!(receive_future); + loop { + select! { + result = &mut receive_future => match result { + Ok(_) => break, + Err(soketto::connection::Error::Closed) => return Ok(()), + Err(err) => return Err(err.into()), + }, + result = broadcast_receiver.recv() => { + + // In both possible error cases (closed or lagged) we disconnect the client. + if let Some(json) = broadcast_handler.handle(result?)? { + sender.send_text(&*json).await?; + } + }, + _ = &mut tripwire => { + warn!("disconnecting websocket client: shutting down"); + return Ok(()) + }, + + } + } + } + let data_str = match str::from_utf8(&data) { + Ok(str) => str, + Err(_) => { + // Old implementation just closes the connection, so we preserve that behavior + // for now. It would be more correct to respond with an error. + break; + } + }; + + if let Some(response) = json_rpc_handler.handle_request(data_str).await { + sender.send_text(&response).await?; + } + data.clear(); + } + + Ok(()) +} + +async fn listen( + listen_address: SocketAddr, + config: PubSubConfig, + subscription_control: SubscriptionControl, + mut tripwire: Tripwire, +) -> io::Result<()> { + let listener = tokio::net::TcpListener::bind(&listen_address).await?; + let counter = TokenCounter::new("rpc_pubsub_connections"); + loop { + select! { + result = listener.accept() => match result { + Ok((socket, addr)) => { + debug!("new client ({:?})", addr); + let subscription_control = subscription_control.clone(); + let config = config.clone(); + let tripwire = tripwire.clone(); + let counter_token = counter.create_token(); + tokio::spawn(async move { + let handle = handle_connection( + socket, subscription_control, config, tripwire + ); + match handle.await { + Ok(()) => debug!("connection closed ({:?})", addr), + Err(err) => warn!("connection handler error ({:?}): {}", addr, err), + } + drop(counter_token); // Force moving token into the task. + }); + } + Err(e) => error!("couldn't accept connection: {:?}", e), + }, + _ = &mut tripwire => return Ok(()), + } + } +} + #[cfg(test)] mod tests { use { @@ -123,7 +355,7 @@ mod tests { }, std::{ net::{IpAddr, Ipv4Addr}, - sync::RwLock, + sync::{atomic::AtomicBool, RwLock}, }, }; @@ -136,14 +368,14 @@ mod tests { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, )); - let pubsub_service = - PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr, &exit); + let (_trigger, pubsub_service) = + PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr); let thread = pubsub_service.thread_hdl.thread(); assert_eq!(thread.name().unwrap(), "solana-pubsub"); } diff --git a/rpc/src/rpc_subscription_tracker.rs b/rpc/src/rpc_subscription_tracker.rs new file mode 100644 index 0000000000..97e8031732 --- /dev/null +++ b/rpc/src/rpc_subscription_tracker.rs @@ -0,0 +1,731 @@ +use { + crate::rpc_subscriptions::{NotificationEntry, RpcNotification}, + dashmap::{mapref::entry::Entry as DashEntry, DashMap}, + solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig}, + solana_client::rpc_filter::RpcFilterType, + solana_metrics::{CounterToken, TokenCounter}, + solana_runtime::{ + bank::{TransactionLogCollectorConfig, TransactionLogCollectorFilter}, + bank_forks::BankForks, + }, + solana_sdk::{ + clock::Slot, commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature, + }, + std::{ + collections::{ + hash_map::{Entry, HashMap}, + HashSet, + }, + fmt, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, RwLock, Weak, + }, + }, + thiserror::Error, + tokio::sync::broadcast, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct SubscriptionId(u64); + +impl From for SubscriptionId { + fn from(value: u64) -> Self { + SubscriptionId(value) + } +} + +impl From for u64 { + fn from(value: SubscriptionId) -> Self { + value.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SubscriptionParams { + Account(AccountSubscriptionParams), + Logs(LogsSubscriptionParams), + Program(ProgramSubscriptionParams), + Signature(SignatureSubscriptionParams), + Slot, + SlotsUpdates, + Root, + Vote, +} + +impl SubscriptionParams { + fn method(&self) -> &'static str { + match self { + SubscriptionParams::Account(_) => "accountNotification", + SubscriptionParams::Logs(_) => "logsNotification", + SubscriptionParams::Program(_) => "programNotification", + SubscriptionParams::Signature(_) => "signatureNotification", + SubscriptionParams::Slot => "slotNotification", + SubscriptionParams::SlotsUpdates => "slotsUpdatesNotification", + SubscriptionParams::Root => "rootNotification", + SubscriptionParams::Vote => "voteNotification", + } + } + + fn commitment(&self) -> Option { + match self { + SubscriptionParams::Account(params) => Some(params.commitment), + SubscriptionParams::Logs(params) => Some(params.commitment), + SubscriptionParams::Program(params) => Some(params.commitment), + SubscriptionParams::Signature(params) => Some(params.commitment), + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => None, + } + } + + fn is_commitment_watcher(&self) -> bool { + let commitment = match self { + SubscriptionParams::Account(params) => ¶ms.commitment, + SubscriptionParams::Logs(params) => ¶ms.commitment, + SubscriptionParams::Program(params) => ¶ms.commitment, + SubscriptionParams::Signature(params) => ¶ms.commitment, + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => return false, + }; + !commitment.is_confirmed() + } + + fn is_gossip_watcher(&self) -> bool { + let commitment = match self { + SubscriptionParams::Account(params) => ¶ms.commitment, + SubscriptionParams::Logs(params) => ¶ms.commitment, + SubscriptionParams::Program(params) => ¶ms.commitment, + SubscriptionParams::Signature(params) => ¶ms.commitment, + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => return false, + }; + commitment.is_confirmed() + } + + fn is_node_progress_watcher(&self) -> bool { + matches!( + self, + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AccountSubscriptionParams { + pub pubkey: Pubkey, + pub encoding: UiAccountEncoding, + pub data_slice: Option, + pub commitment: CommitmentConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct LogsSubscriptionParams { + pub kind: LogsSubscriptionKind, + pub commitment: CommitmentConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum LogsSubscriptionKind { + All, + AllWithVotes, + Single(Pubkey), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ProgramSubscriptionParams { + pub pubkey: Pubkey, + pub filters: Vec, + pub encoding: UiAccountEncoding, + pub data_slice: Option, + pub commitment: CommitmentConfig, + pub with_context: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SignatureSubscriptionParams { + pub signature: Signature, + pub commitment: CommitmentConfig, + pub enable_received_notification: bool, +} + +#[derive(Clone)] +pub struct SubscriptionControl(Arc); + +struct SubscriptionControlInner { + subscriptions: DashMap>, + next_id: AtomicU64, + max_active_subscriptions: usize, + sender: crossbeam_channel::Sender, + broadcast_sender: broadcast::Sender, + counter: TokenCounter, +} + +impl SubscriptionControl { + pub fn new( + max_active_subscriptions: usize, + sender: crossbeam_channel::Sender, + broadcast_sender: broadcast::Sender, + ) -> Self { + Self(Arc::new(SubscriptionControlInner { + subscriptions: DashMap::new(), + next_id: AtomicU64::new(0), + max_active_subscriptions, + sender, + broadcast_sender, + counter: TokenCounter::new("rpc_pubsub_total_subscriptions"), + })) + } + + pub fn broadcast_receiver(&self) -> broadcast::Receiver { + self.0.broadcast_sender.subscribe() + } + + pub fn subscribe(&self, params: SubscriptionParams) -> Result { + debug!( + "Total existing subscriptions: {}", + self.0.subscriptions.len() + ); + let count = self.0.subscriptions.len(); + match self.0.subscriptions.entry(params) { + DashEntry::Occupied(entry) => Ok(SubscriptionToken( + entry + .get() + .upgrade() + .expect("dead subscription encountered in SubscriptionControl"), + self.0.counter.create_token(), + )), + DashEntry::Vacant(entry) => { + if count >= self.0.max_active_subscriptions { + inc_new_counter_info!("rpc-subscription-refused-limit-reached", 1); + return Err(Error::TooManySubscriptions); + } + let id = SubscriptionId::from(self.0.next_id.fetch_add(1, Ordering::AcqRel)); + let token = SubscriptionToken( + Arc::new(SubscriptionTokenInner { + control: Arc::clone(&self.0), + params: entry.key().clone(), + id, + }), + self.0.counter.create_token(), + ); + let _ = self + .0 + .sender + .send(NotificationEntry::Subscribed(token.0.params.clone(), id)); + entry.insert(Arc::downgrade(&token.0)); + datapoint_info!( + "rpc-subscription", + ("total", self.0.subscriptions.len(), i64) + ); + Ok(token) + } + } + } + + pub fn total(&self) -> usize { + self.0.subscriptions.len() + } + + #[cfg(test)] + pub fn assert_subscribed(&self, params: &SubscriptionParams) { + assert!(self.0.subscriptions.contains_key(params)); + } + + #[cfg(test)] + pub fn assert_unsubscribed(&self, params: &SubscriptionParams) { + assert!(!self.0.subscriptions.contains_key(params)); + } + + #[cfg(test)] + pub fn account_subscribed(&self, pubkey: &Pubkey) -> bool { + self.0.subscriptions.iter().any(|item| { + if let SubscriptionParams::Account(params) = item.key() { + ¶ms.pubkey == pubkey + } else { + false + } + }) + } + + #[cfg(test)] + pub fn signature_subscribed(&self, signature: &Signature) -> bool { + self.0.subscriptions.iter().any(|item| { + if let SubscriptionParams::Signature(params) = item.key() { + ¶ms.signature == signature + } else { + false + } + }) + } +} + +#[derive(Debug)] +pub struct SubscriptionInfo { + id: SubscriptionId, + params: SubscriptionParams, + method: &'static str, + pub last_notified_slot: RwLock, + commitment: Option, +} + +impl SubscriptionInfo { + pub fn id(&self) -> SubscriptionId { + self.id + } + + pub fn method(&self) -> &'static str { + self.method + } + + pub fn params(&self) -> &SubscriptionParams { + &self.params + } + + pub fn commitment(&self) -> Option { + self.commitment + } +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("node subscription limit reached")] + TooManySubscriptions, +} + +struct LogsSubscriptionsIndex { + all_count: usize, + all_with_votes_count: usize, + single_count: HashMap, + + bank_forks: Arc>, +} + +impl LogsSubscriptionsIndex { + fn add(&mut self, params: &LogsSubscriptionParams) { + match params.kind { + LogsSubscriptionKind::All => self.all_count += 1, + LogsSubscriptionKind::AllWithVotes => self.all_with_votes_count += 1, + LogsSubscriptionKind::Single(key) => { + *self.single_count.entry(key).or_default() += 1; + } + } + self.update_config(); + } + + fn remove(&mut self, params: &LogsSubscriptionParams) { + match params.kind { + LogsSubscriptionKind::All => self.all_count -= 1, + LogsSubscriptionKind::AllWithVotes => self.all_with_votes_count -= 1, + LogsSubscriptionKind::Single(key) => match self.single_count.entry(key) { + Entry::Occupied(mut entry) => { + *entry.get_mut() -= 1; + if *entry.get() == 0 { + entry.remove(); + } + } + Entry::Vacant(_) => error!("missing entry in single_count"), + }, + } + self.update_config(); + } + + fn update_config(&self) { + let config = if self.all_with_votes_count > 0 { + TransactionLogCollectorConfig { + filter: TransactionLogCollectorFilter::AllWithVotes, + mentioned_addresses: HashSet::new(), + } + } else if self.all_count > 0 { + TransactionLogCollectorConfig { + filter: TransactionLogCollectorFilter::All, + mentioned_addresses: HashSet::new(), + } + } else { + TransactionLogCollectorConfig { + filter: TransactionLogCollectorFilter::OnlyMentionedAddresses, + mentioned_addresses: self.single_count.keys().copied().collect(), + } + }; + + *self + .bank_forks + .read() + .unwrap() + .root_bank() + .transaction_log_collector_config + .write() + .unwrap() = config; + } +} + +pub struct SubscriptionsTracker { + logs_subscriptions_index: LogsSubscriptionsIndex, + by_signature: HashMap>>, + // Accounts, logs, programs, signatures (not gossip) + commitment_watchers: HashMap>, + // Accounts, logs, programs, signatures (gossip) + gossip_watchers: HashMap>, + // Slots, slots updates, roots, votes. + node_progress_watchers: HashMap>, +} + +impl SubscriptionsTracker { + pub fn new(bank_forks: Arc>) -> Self { + SubscriptionsTracker { + logs_subscriptions_index: LogsSubscriptionsIndex { + all_count: 0, + all_with_votes_count: 0, + single_count: HashMap::new(), + bank_forks, + }, + by_signature: HashMap::new(), + commitment_watchers: HashMap::new(), + gossip_watchers: HashMap::new(), + node_progress_watchers: HashMap::new(), + } + } + + pub fn subscribe( + &mut self, + params: SubscriptionParams, + id: SubscriptionId, + last_notified_slot: impl FnOnce() -> Slot, + ) { + let info = Arc::new(SubscriptionInfo { + last_notified_slot: RwLock::new(last_notified_slot()), + id, + commitment: params.commitment(), + method: params.method(), + params: params.clone(), + }); + match ¶ms { + SubscriptionParams::Logs(params) => { + self.logs_subscriptions_index.add(params); + } + SubscriptionParams::Signature(params) => { + self.by_signature + .entry(params.signature) + .or_default() + .insert(id, Arc::clone(&info)); + } + _ => {} + } + if info.params.is_commitment_watcher() { + self.commitment_watchers.insert(id, Arc::clone(&info)); + } + if info.params.is_gossip_watcher() { + self.gossip_watchers.insert(id, Arc::clone(&info)); + } + if info.params.is_node_progress_watcher() { + self.node_progress_watchers + .insert(info.params.clone(), Arc::clone(&info)); + } + } + + #[allow(clippy::collapsible_if)] + pub fn unsubscribe(&mut self, params: SubscriptionParams, id: SubscriptionId) { + match ¶ms { + SubscriptionParams::Logs(params) => { + self.logs_subscriptions_index.remove(params); + } + SubscriptionParams::Signature(params) => { + if let Entry::Occupied(mut entry) = self.by_signature.entry(params.signature) { + if entry.get_mut().remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in by_signature)"); + } + if entry.get_mut().is_empty() { + entry.remove(); + } + } else { + warn!("Subscriptions inconsistency (missing entry in by_signature)"); + } + } + _ => {} + } + if params.is_commitment_watcher() { + if self.commitment_watchers.remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in commitment_watchers)"); + } + } + if params.is_gossip_watcher() { + if self.gossip_watchers.remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in gossip_watchers)"); + } + } + if params.is_node_progress_watcher() { + if self.node_progress_watchers.remove(¶ms).is_none() { + warn!("Subscriptions inconsistency (missing entry in node_progress_watchers)"); + } + } + } + + pub fn by_signature( + &self, + ) -> &HashMap>> { + &self.by_signature + } + pub fn commitment_watchers(&self) -> &HashMap> { + &self.commitment_watchers + } + pub fn gossip_watchers(&self) -> &HashMap> { + &self.gossip_watchers + } + pub fn node_progress_watchers(&self) -> &HashMap> { + &self.node_progress_watchers + } +} + +struct SubscriptionTokenInner { + control: Arc, + params: SubscriptionParams, + id: SubscriptionId, +} + +impl fmt::Debug for SubscriptionTokenInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SubscriptionTokenInner") + .field("id", &self.id) + .finish() + } +} + +impl Drop for SubscriptionTokenInner { + #[allow(clippy::collapsible_if)] + fn drop(&mut self) { + match self.control.subscriptions.entry(self.params.clone()) { + DashEntry::Vacant(_) => { + warn!("Subscriptions inconsistency (missing entry in by_params)"); + } + DashEntry::Occupied(entry) => { + let _ = self.control.sender.send(NotificationEntry::Unsubscribed( + self.params.clone(), + self.id, + )); + entry.remove(); + datapoint_info!( + "rpc-subscription", + ("total", self.control.subscriptions.len(), i64) + ); + } + } + } +} + +#[derive(Clone)] +pub struct SubscriptionToken(Arc, CounterToken); + +impl SubscriptionToken { + pub fn id(&self) -> SubscriptionId { + self.0.id + } + + pub fn params(&self) -> &SubscriptionParams { + &self.0.params + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_pubsub_service::PubSubConfig; + use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo}; + use solana_runtime::bank::Bank; + use std::str::FromStr; + + struct ControlWrapper { + control: SubscriptionControl, + receiver: crossbeam_channel::Receiver, + } + + impl ControlWrapper { + fn new() -> Self { + let (sender, receiver) = crossbeam_channel::unbounded(); + let (broadcast_sender, _broadcast_receiver) = broadcast::channel(42); + + let control = SubscriptionControl::new( + PubSubConfig::default().max_active_subscriptions, + sender, + broadcast_sender, + ); + Self { control, receiver } + } + + fn assert_subscribed(&self, expected_params: &SubscriptionParams, expected_id: u64) { + if let NotificationEntry::Subscribed(params, id) = self.receiver.recv().unwrap() { + assert_eq!(¶ms, expected_params); + assert_eq!(id, SubscriptionId::from(expected_id)); + } else { + panic!("unexpected notification"); + } + self.assert_silence(); + } + + fn assert_unsubscribed(&self, expected_params: &SubscriptionParams, expected_id: u64) { + if let NotificationEntry::Unsubscribed(params, id) = self.receiver.recv().unwrap() { + assert_eq!(¶ms, expected_params); + assert_eq!(id, SubscriptionId::from(expected_id)); + } else { + panic!("unexpected notification"); + } + self.assert_silence(); + } + + fn assert_silence(&self) { + assert!(self.receiver.try_recv().is_err()); + } + } + + #[test] + fn notify_subscribe() { + let control = ControlWrapper::new(); + let token1 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 0); + drop(token1); + control.assert_unsubscribed(&SubscriptionParams::Slot, 0); + } + + #[test] + fn notify_subscribe_multiple() { + let control = ControlWrapper::new(); + let token1 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 0); + let token2 = token1.clone(); + drop(token1); + let token3 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + drop(token3); + control.assert_silence(); + drop(token2); + control.assert_unsubscribed(&SubscriptionParams::Slot, 0); + } + + #[test] + fn notify_subscribe_two_subscriptions() { + let control = ControlWrapper::new(); + let token_slot1 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 0); + + let signature_params = SubscriptionParams::Signature(SignatureSubscriptionParams { + signature: Signature::default(), + commitment: CommitmentConfig::processed(), + enable_received_notification: false, + }); + let token_signature1 = control.control.subscribe(signature_params.clone()).unwrap(); + control.assert_subscribed(&signature_params, 1); + + let token_slot2 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + let token_signature2 = control.control.subscribe(signature_params.clone()).unwrap(); + drop(token_slot1); + control.assert_silence(); + drop(token_slot2); + control.assert_unsubscribed(&SubscriptionParams::Slot, 0); + drop(token_signature2); + control.assert_silence(); + drop(token_signature1); + control.assert_unsubscribed(&signature_params, 1); + + let token_slot3 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 2); + drop(token_slot3); + control.assert_unsubscribed(&SubscriptionParams::Slot, 2); + } + + #[test] + fn subscription_info() { + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); + let bank = Bank::new_for_tests(&genesis_config); + let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); + let mut tracker = SubscriptionsTracker::new(bank_forks); + + tracker.subscribe(SubscriptionParams::Slot, 0.into(), || 0); + let info = tracker + .node_progress_watchers + .get(&SubscriptionParams::Slot) + .unwrap(); + assert_eq!(info.commitment, None); + assert_eq!(info.params, SubscriptionParams::Slot); + assert_eq!(info.method, SubscriptionParams::Slot.method()); + assert_eq!(info.id, SubscriptionId::from(0)); + assert_eq!(*info.last_notified_slot.read().unwrap(), 0); + + let account_params = SubscriptionParams::Account(AccountSubscriptionParams { + pubkey: Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), + commitment: CommitmentConfig::finalized(), + encoding: UiAccountEncoding::Base64Zstd, + data_slice: None, + }); + tracker.subscribe(account_params.clone(), 1.into(), || 42); + + let info = tracker + .commitment_watchers + .get(&SubscriptionId::from(1)) + .unwrap(); + assert_eq!(info.commitment, Some(CommitmentConfig::finalized())); + assert_eq!(info.params, account_params); + assert_eq!(info.method, account_params.method()); + assert_eq!(info.id, SubscriptionId::from(1)); + assert_eq!(*info.last_notified_slot.read().unwrap(), 42); + } + + #[test] + fn subscription_indexes() { + fn counts(tracker: &SubscriptionsTracker) -> (usize, usize, usize, usize) { + ( + tracker.by_signature.len(), + tracker.commitment_watchers.len(), + tracker.gossip_watchers.len(), + tracker.node_progress_watchers.len(), + ) + } + + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); + let bank = Bank::new_for_tests(&genesis_config); + let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); + let mut tracker = SubscriptionsTracker::new(bank_forks); + + tracker.subscribe(SubscriptionParams::Slot, 0.into(), || 0); + assert_eq!(counts(&tracker), (0, 0, 0, 1)); + tracker.unsubscribe(SubscriptionParams::Slot, 0.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + + let account_params = SubscriptionParams::Account(AccountSubscriptionParams { + pubkey: Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), + commitment: CommitmentConfig::finalized(), + encoding: UiAccountEncoding::Base64Zstd, + data_slice: None, + }); + tracker.subscribe(account_params.clone(), 1.into(), || 0); + assert_eq!(counts(&tracker), (0, 1, 0, 0)); + tracker.unsubscribe(account_params, 1.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + + let account_params2 = SubscriptionParams::Account(AccountSubscriptionParams { + pubkey: Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), + commitment: CommitmentConfig::confirmed(), + encoding: UiAccountEncoding::Base64Zstd, + data_slice: None, + }); + tracker.subscribe(account_params2.clone(), 2.into(), || 0); + assert_eq!(counts(&tracker), (0, 0, 1, 0)); + tracker.unsubscribe(account_params2, 2.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + + let signature_params = SubscriptionParams::Signature(SignatureSubscriptionParams { + signature: Signature::default(), + commitment: CommitmentConfig::processed(), + enable_received_notification: false, + }); + tracker.subscribe(signature_params.clone(), 3.into(), || 0); + assert_eq!(counts(&tracker), (1, 1, 0, 0)); + tracker.unsubscribe(signature_params, 3.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + } +} diff --git a/rpc/src/rpc_subscriptions.rs b/rpc/src/rpc_subscriptions.rs index 018012b504..982c7152d4 100644 --- a/rpc/src/rpc_subscriptions.rs +++ b/rpc/src/rpc_subscriptions.rs @@ -4,16 +4,17 @@ use { crate::{ optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, parsed_token_accounts::{get_parsed_token_account, get_parsed_token_accounts}, + rpc_pubsub_service::PubSubConfig, + rpc_subscription_tracker::{ + AccountSubscriptionParams, LogsSubscriptionKind, LogsSubscriptionParams, + ProgramSubscriptionParams, SignatureSubscriptionParams, SubscriptionControl, + SubscriptionId, SubscriptionInfo, SubscriptionParams, SubscriptionsTracker, + }, }, - core::hash::Hash, - jsonrpc_pubsub::{ - typed::{Sink, Subscriber}, - SubscriptionId, - }, + crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender}, serde::Serialize, solana_account_decoder::{parse_token::spl_token_id_v2_0, UiAccount, UiAccountEncoding}, solana_client::{ - rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig}, rpc_filter::RpcFilterType, rpc_response::{ ProcessedSignatureResult, ReceivedSignatureResult, Response, RpcKeyedAccount, @@ -22,16 +23,13 @@ use { }, solana_measure::measure::Measure, solana_runtime::{ - bank::{ - Bank, TransactionLogCollectorConfig, TransactionLogCollectorFilter, TransactionLogInfo, - }, + bank::{Bank, TransactionLogInfo}, bank_forks::BankForks, commitment::{BlockCommitmentCache, CommitmentSlots}, }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::{Slot, UnixTimestamp}, - commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature, timing::timestamp, @@ -39,40 +37,37 @@ use { }, solana_vote_program::vote_state::Vote, std::{ - collections::{HashMap, HashSet}, - iter, + collections::{HashMap, VecDeque}, + io::Cursor, + iter, str, sync::{ atomic::{AtomicBool, Ordering}, - mpsc::{Receiver, RecvTimeoutError, SendError, Sender}, + Arc, RwLock, Weak, }, - sync::{Arc, Mutex, RwLock}, thread::{Builder, JoinHandle}, time::Duration, }, + tokio::sync::broadcast, }; const RECEIVE_DELAY_MILLIS: u64 = 100; -trait BankGetTransactionLogsAdapter { - fn get_transaction_logs_adapter( - &self, - stuff: &(Option, bool), - ) -> Option>; -} - -impl BankGetTransactionLogsAdapter for Bank { - fn get_transaction_logs_adapter( - &self, - config: &(Option, bool), - ) -> Option> { - let mut logs = self.get_transaction_logs(config.0.as_ref()); - - if config.0.is_none() && !config.1 { - // Filter out votes if the subscriber doesn't want them - logs = logs.map(|logs| logs.into_iter().filter(|log| !log.is_vote).collect()); +fn get_transaction_logs( + bank: &Bank, + params: &LogsSubscriptionParams, +) -> Option> { + let pubkey = match ¶ms.kind { + LogsSubscriptionKind::All | LogsSubscriptionKind::AllWithVotes => None, + LogsSubscriptionKind::Single(pubkey) => Some(pubkey), + }; + let mut logs = bank.get_transaction_logs(pubkey); + if matches!(params.kind, LogsSubscriptionKind::All) { + // Filter out votes if the subscriber doesn't want them + if let Some(logs) = &mut logs { + logs.retain(|log| !log.is_vote); } - logs } + logs } // A more human-friendly version of Vote, with the bank state signature base58 encoded. @@ -83,7 +78,7 @@ pub struct RpcVote { pub timestamp: Option, } -enum NotificationEntry { +pub enum NotificationEntry { Slot(SlotInfo), SlotUpdate(SlotUpdate), Vote(Vote), @@ -91,6 +86,8 @@ enum NotificationEntry { Bank(CommitmentSlots), Gossip(Slot), SignaturesReceived((Slot, Vec)), + Subscribed(SubscriptionParams, SubscriptionId), + Unsubscribed(SubscriptionParams, SubscriptionId), } impl std::fmt::Debug for NotificationEntry { @@ -109,179 +106,174 @@ impl std::fmt::Debug for NotificationEntry { write!(f, "SignaturesReceived({:?})", slot_signatures) } NotificationEntry::Gossip(slot) => write!(f, "Gossip({:?})", slot), + NotificationEntry::Subscribed(params, id) => { + write!(f, "Subscribed({:?}, {:?})", params, id) + } + NotificationEntry::Unsubscribed(params, id) => { + write!(f, "Unsubscribed({:?}, {:?})", params, id) + } } } } -struct SubscriptionData { - sink: Sink, - commitment: CommitmentConfig, - last_notified_slot: RwLock, - config: Option, -} #[derive(Default, Clone)] struct ProgramConfig { filters: Vec, encoding: Option, } -type RpcAccountSubscriptions = RwLock< - HashMap< - Pubkey, - HashMap, UiAccountEncoding>>, - >, ->; -type RpcLogsSubscriptions = RwLock< - HashMap< - (Option, bool), - HashMap, ()>>, - >, ->; -type RpcProgramSubscriptions = RwLock< - HashMap< - Pubkey, - HashMap, ProgramConfig>>, - >, ->; -type RpcSignatureSubscriptions = RwLock< - HashMap< - Signature, - HashMap, bool>>, - >, ->; -type RpcSlotSubscriptions = RwLock>>; -type RpcSlotUpdateSubscriptions = RwLock>>>; -type RpcVoteSubscriptions = RwLock>>; -type RpcRootSubscriptions = RwLock>>; - -fn add_subscription( - subscriptions: &mut HashMap>>, - hashmap_key: K, - commitment: CommitmentConfig, - sub_id: SubscriptionId, - subscriber: Subscriber, - last_notified_slot: Slot, - config: Option, -) where - K: Eq + Hash, - S: Clone, -{ - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let subscription_data = SubscriptionData { - sink, - commitment, - last_notified_slot: RwLock::new(last_notified_slot), - config, - }; - - subscriptions - .entry(hashmap_key) - .or_default() - .insert(sub_id, subscription_data); -} - -fn remove_subscription( - subscriptions: &mut HashMap>>, - sub_id: &SubscriptionId, -) -> bool -where - K: Eq + Hash, - S: Clone, -{ - let mut found = false; - subscriptions.retain(|_, v| { - v.retain(|k, _| { - let retain = k != sub_id; - if !retain { - found = true; - } - retain - }); - !v.is_empty() - }); - found -} #[allow(clippy::type_complexity)] -fn check_commitment_and_notify( - subscriptions: &HashMap, T>>>, - hashmap_key: &K, +fn check_commitment_and_notify( + params: &P, + subscription: &SubscriptionInfo, bank_forks: &Arc>, commitment_slots: &CommitmentSlots, bank_method: B, filter_results: F, - notifier: &RpcNotifier, -) -> HashSet + notifier: &mut RpcNotifier, + is_final: bool, +) -> bool where - K: Eq + Hash + Clone + Copy, S: Clone + Serialize, - B: Fn(&Bank, &K) -> X, - F: Fn(X, &K, Slot, Option, Arc) -> (Box>, Slot), + B: Fn(&Bank, &P) -> X, + F: Fn(X, &P, Slot, Arc) -> (Box>, Slot), X: Clone + Default, - T: Clone, { - let mut notified_set: HashSet = HashSet::new(); - if let Some(hashmap) = subscriptions.get(hashmap_key) { - for ( - sub_id, - SubscriptionData { - sink, - commitment, - last_notified_slot, - config, - }, - ) in hashmap.iter() - { - let slot = if commitment.is_finalized() { - commitment_slots.highest_confirmed_root - } else if commitment.is_confirmed() { - commitment_slots.highest_confirmed_slot - } else { - commitment_slots.slot - }; + let commitment = if let Some(commitment) = subscription.commitment() { + commitment + } else { + error!("missing commitment in check_commitment_and_notify"); + return false; + }; + let slot = if commitment.is_finalized() { + commitment_slots.highest_confirmed_root + } else if commitment.is_confirmed() { + commitment_slots.highest_confirmed_slot + } else { + commitment_slots.slot + }; - if let Some(bank) = bank_forks.read().unwrap().get(slot).cloned() { - let results = bank_method(&bank, hashmap_key); - let mut w_last_notified_slot = last_notified_slot.write().unwrap(); - let (filter_results, result_slot) = filter_results( - results, - hashmap_key, - *w_last_notified_slot, - config.as_ref().cloned(), - bank, - ); - for result in filter_results { - notifier.notify( - Response { - context: RpcResponseContext { slot }, - value: result, - }, - sink, - ); - notified_set.insert(sub_id.clone()); - *w_last_notified_slot = result_slot; - } - } + let mut notified = false; + if let Some(bank) = bank_forks.read().unwrap().get(slot).cloned() { + let results = bank_method(&bank, params); + let mut w_last_notified_slot = subscription.last_notified_slot.write().unwrap(); + let (filter_results, result_slot) = + filter_results(results, params, *w_last_notified_slot, bank); + for result in filter_results { + notifier.notify( + Response { + context: RpcResponseContext { slot }, + value: result, + }, + subscription, + is_final, + ); + *w_last_notified_slot = result_slot; + notified = true; } } - notified_set + notified } -struct RpcNotifier; +#[derive(Debug, Clone)] +pub struct RpcNotification { + pub subscription_id: SubscriptionId, + pub is_final: bool, + pub json: Weak, +} + +struct RecentItems { + queue: VecDeque>, + total_bytes: usize, + max_len: usize, + max_total_bytes: usize, +} + +impl RecentItems { + fn new(max_len: usize, max_total_bytes: usize) -> Self { + Self { + queue: VecDeque::new(), + total_bytes: 0, + max_len, + max_total_bytes, + } + } + + fn push(&mut self, item: Arc) { + self.total_bytes = self + .total_bytes + .checked_add(item.len()) + .expect("total bytes overflow"); + self.queue.push_back(item); + + while self.total_bytes > self.max_total_bytes || self.queue.len() > self.max_len { + let item = self.queue.pop_front().expect("can't be empty"); + self.total_bytes = self + .total_bytes + .checked_sub(item.len()) + .expect("total bytes underflow"); + } + } +} + +struct RpcNotifier { + sender: broadcast::Sender, + buf: Vec, + recent_items: RecentItems, +} + +#[derive(Debug, Serialize)] +struct NotificationParams { + result: T, + subscription: SubscriptionId, +} + +#[derive(Debug, Serialize)] +struct Notification { + jsonrpc: Option, + method: &'static str, + params: NotificationParams, +} impl RpcNotifier { - fn notify(&self, value: T, sink: &Sink) + fn notify(&mut self, value: T, subscription: &SubscriptionInfo, is_final: bool) where T: serde::Serialize, { - let _ = sink.notify(Ok(value)); + self.buf.clear(); + let notification = Notification { + jsonrpc: Some(jsonrpc_core::Version::V2), + method: subscription.method(), + params: NotificationParams { + result: value, + subscription: subscription.id(), + }, + }; + serde_json::to_writer(Cursor::new(&mut self.buf), ¬ification) + .expect("serialization never fails"); + let buf_str = str::from_utf8(&self.buf).expect("json is always utf-8"); + let buf_arc = Arc::new(String::from(buf_str)); + + let notification = RpcNotification { + subscription_id: subscription.id(), + json: Arc::downgrade(&buf_arc), + is_final, + }; + // There is an unlikely case where this can fail: if the last subscription is closed + // just as the notifier generates a notification for it. + let _ = self.sender.send(notification); + + inc_new_counter_info!("rpc-pubsub-messages", 1); + inc_new_counter_info!("rpc-pubsub-bytes", buf_arc.len()); + + self.recent_items.push(buf_arc); } } fn filter_account_result( result: Option<(AccountSharedData, Slot)>, - pubkey: &Pubkey, + params: &AccountSubscriptionParams, last_notified_slot: Slot, - encoding: Option, bank: Arc, ) -> (Box>, Slot) { // If the account is not found, `last_modified_slot` will default to zero and @@ -291,12 +283,21 @@ fn filter_account_result( // If last_modified_slot < last_notified_slot this means that we last notified for a fork // and should notify that the account state has been reverted. let results: Box> = if last_modified_slot != last_notified_slot { - let encoding = encoding.unwrap_or(UiAccountEncoding::Binary); - if account.owner() == &spl_token_id_v2_0() && encoding == UiAccountEncoding::JsonParsed { - Box::new(iter::once(get_parsed_token_account(bank, pubkey, account))) + if account.owner() == &spl_token_id_v2_0() + && params.encoding == UiAccountEncoding::JsonParsed + { + Box::new(iter::once(get_parsed_token_account( + bank, + ¶ms.pubkey, + account, + ))) } else { Box::new(iter::once(UiAccount::encode( - pubkey, &account, encoding, None, None, + ¶ms.pubkey, + &account, + params.encoding, + None, + None, ))) } } else { @@ -308,9 +309,8 @@ fn filter_account_result( fn filter_signature_result( result: Option>, - _signature: &Signature, + _params: &SignatureSubscriptionParams, last_notified_slot: Slot, - _config: Option, _bank: Arc, ) -> (Box>, Slot) { ( @@ -323,23 +323,22 @@ fn filter_signature_result( fn filter_program_results( accounts: Vec<(Pubkey, AccountSharedData)>, - program_id: &Pubkey, + params: &ProgramSubscriptionParams, last_notified_slot: Slot, - config: Option, bank: Arc, ) -> (Box>, Slot) { - let config = config.unwrap_or_default(); - let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary); - let filters = config.filters; let accounts_is_empty = accounts.is_empty(); + let encoding = params.encoding; + let filters = params.filters.clone(); let keyed_accounts = accounts.into_iter().filter(move |(_, account)| { filters.iter().all(|filter_type| match filter_type { RpcFilterType::DataSize(size) => account.data().len() as u64 == *size, RpcFilterType::Memcmp(compare) => compare.bytes_match(account.data()), }) }); - let accounts: Box> = if program_id == &spl_token_id_v2_0() - && encoding == UiAccountEncoding::JsonParsed + let accounts: Box> = if params.pubkey + == spl_token_id_v2_0() + && params.encoding == UiAccountEncoding::JsonParsed && !accounts_is_empty { Box::new(get_parsed_token_accounts(bank, keyed_accounts)) @@ -356,9 +355,8 @@ fn filter_program_results( fn filter_logs_results( logs: Option>, - _address: &(Option, bool), + _params: &LogsSubscriptionParams, last_notified_slot: Slot, - _config: Option<()>, _bank: Arc, ) -> (Box>, Slot) { match logs { @@ -374,59 +372,54 @@ fn filter_logs_results( } } -fn total_nested_subscriptions( - subscription_map: &RwLock>>, -) -> usize { - subscription_map - .read() - .unwrap() - .iter() - .fold(0, |acc, x| acc + x.1.len()) -} +fn initial_last_notified_slot( + params: &SubscriptionParams, + bank_forks: &RwLock, + block_commitment_cache: &RwLock, + optimistically_confirmed_bank: &RwLock, +) -> Slot { + match params { + SubscriptionParams::Account(params) => { + let slot = if params.commitment.is_finalized() { + block_commitment_cache + .read() + .unwrap() + .highest_confirmed_root() + } else if params.commitment.is_confirmed() { + optimistically_confirmed_bank.read().unwrap().bank.slot() + } else { + block_commitment_cache.read().unwrap().slot() + }; -#[derive(Clone)] -struct Subscriptions { - account_subscriptions: Arc, - program_subscriptions: Arc, - logs_subscriptions: Arc, - signature_subscriptions: Arc, - gossip_account_subscriptions: Arc, - gossip_logs_subscriptions: Arc, - gossip_program_subscriptions: Arc, - gossip_signature_subscriptions: Arc, - slot_subscriptions: Arc, - slots_updates_subscriptions: Arc, - vote_subscriptions: Arc, - root_subscriptions: Arc, -} - -impl Subscriptions { - fn total(&self) -> usize { - let mut total = 0; - total += total_nested_subscriptions(&self.account_subscriptions); - total += total_nested_subscriptions(&self.program_subscriptions); - total += total_nested_subscriptions(&self.logs_subscriptions); - total += total_nested_subscriptions(&self.signature_subscriptions); - total += total_nested_subscriptions(&self.gossip_account_subscriptions); - total += total_nested_subscriptions(&self.gossip_logs_subscriptions); - total += total_nested_subscriptions(&self.gossip_program_subscriptions); - total += total_nested_subscriptions(&self.gossip_signature_subscriptions); - total += self.slot_subscriptions.read().unwrap().len(); - total += self.vote_subscriptions.read().unwrap().len(); - total += self.root_subscriptions.read().unwrap().len(); - total + if let Some((_account, slot)) = bank_forks + .read() + .unwrap() + .get(slot) + .and_then(|bank| bank.get_account_modified_slot(¶ms.pubkey)) + { + slot + } else { + 0 + } + } + // last_notified_slot is not utilized for these subscriptions + SubscriptionParams::Logs(_) + | SubscriptionParams::Program(_) + | SubscriptionParams::Signature(_) + | SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => 0, } } pub struct RpcSubscriptions { - subscriptions: Subscriptions, - notification_sender: Arc>>, + notification_sender: Sender, + t_cleanup: Option>, - bank_forks: Arc>, - block_commitment_cache: Arc>, - optimistically_confirmed_bank: Arc>, + exit: Arc, - enable_vote_subscription: bool, + control: SubscriptionControl, } impl Drop for RpcSubscriptions { @@ -444,61 +437,52 @@ impl RpcSubscriptions { block_commitment_cache: Arc>, optimistically_confirmed_bank: Arc>, ) -> Self { - Self::new_with_vote_subscription( + Self::new_with_config( exit, bank_forks, block_commitment_cache, optimistically_confirmed_bank, - false, + &PubSubConfig::default(), ) } - pub fn new_with_vote_subscription( + pub fn new_for_tests( exit: &Arc, bank_forks: Arc>, block_commitment_cache: Arc>, optimistically_confirmed_bank: Arc>, - enable_vote_subscription: bool, ) -> Self { - let (notification_sender, notification_receiver): ( - Sender, - Receiver, - ) = std::sync::mpsc::channel(); + Self::new_with_config( + exit, + bank_forks, + block_commitment_cache, + optimistically_confirmed_bank, + &PubSubConfig::default_for_tests(), + ) + } - let account_subscriptions = Arc::new(RpcAccountSubscriptions::default()); - let logs_subscriptions = Arc::new(RpcLogsSubscriptions::default()); - let program_subscriptions = Arc::new(RpcProgramSubscriptions::default()); - let signature_subscriptions = Arc::new(RpcSignatureSubscriptions::default()); - let gossip_account_subscriptions = Arc::new(RpcAccountSubscriptions::default()); - let gossip_logs_subscriptions = Arc::new(RpcLogsSubscriptions::default()); - let gossip_program_subscriptions = Arc::new(RpcProgramSubscriptions::default()); - let gossip_signature_subscriptions = Arc::new(RpcSignatureSubscriptions::default()); - let slot_subscriptions = Arc::new(RpcSlotSubscriptions::default()); - let slots_updates_subscriptions = Arc::new(RpcSlotUpdateSubscriptions::default()); - let vote_subscriptions = Arc::new(RpcVoteSubscriptions::default()); - let root_subscriptions = Arc::new(RpcRootSubscriptions::default()); - let notification_sender = Arc::new(Mutex::new(notification_sender)); + pub fn new_with_config( + exit: &Arc, + bank_forks: Arc>, + block_commitment_cache: Arc>, + optimistically_confirmed_bank: Arc>, + config: &PubSubConfig, + ) -> Self { + let (notification_sender, notification_receiver) = crossbeam_channel::unbounded(); - let _bank_forks = bank_forks.clone(); - let _block_commitment_cache = block_commitment_cache.clone(); let exit_clone = exit.clone(); - let subscriptions = Subscriptions { - account_subscriptions, - program_subscriptions, - logs_subscriptions, - signature_subscriptions, - gossip_account_subscriptions, - gossip_logs_subscriptions, - gossip_program_subscriptions, - gossip_signature_subscriptions, - slot_subscriptions, - slots_updates_subscriptions, - vote_subscriptions, - root_subscriptions, - }; - let _subscriptions = subscriptions.clone(); + let subscriptions = SubscriptionsTracker::new(bank_forks.clone()); - let notifier = RpcNotifier {}; + let (broadcast_sender, _) = broadcast::channel(config.queue_capacity_items); + + let notifier = RpcNotifier { + sender: broadcast_sender.clone(), + buf: Vec::new(), + recent_items: RecentItems::new( + config.queue_capacity_items, + config.queue_capacity_bytes, + ), + }; let t_cleanup = Builder::new() .name("solana-rpc-notifications".to_string()) .spawn(move || { @@ -506,21 +490,26 @@ impl RpcSubscriptions { exit_clone, notifier, notification_receiver, - _subscriptions, - _bank_forks, + subscriptions, + bank_forks, + block_commitment_cache, + optimistically_confirmed_bank, ); }) .unwrap(); + let control = SubscriptionControl::new( + PubSubConfig::default().max_active_subscriptions, + notification_sender.clone(), + broadcast_sender, + ); + Self { - subscriptions, notification_sender, t_cleanup: Some(t_cleanup), - bank_forks, - block_commitment_cache, - optimistically_confirmed_bank, + exit: exit.clone(), - enable_vote_subscription, + control, } } @@ -528,373 +517,16 @@ impl RpcSubscriptions { pub fn default_with_bank_forks(bank_forks: Arc>) -> Self { let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - Self::new_with_vote_subscription( + Self::new( &Arc::new(AtomicBool::new(false)), bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), optimistically_confirmed_bank, - true, ) } - fn check_account( - pubkey: &Pubkey, - bank_forks: &Arc>, - account_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let subscriptions = account_subscriptions.read().unwrap(); - check_commitment_and_notify( - &subscriptions, - pubkey, - bank_forks, - commitment_slots, - Bank::get_account_modified_slot, - filter_account_result, - notifier, - ) - } - - fn check_logs( - address_with_enable_votes_flag: &(Option, bool), - bank_forks: &Arc>, - logs_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let subscriptions = logs_subscriptions.read().unwrap(); - check_commitment_and_notify( - &subscriptions, - address_with_enable_votes_flag, - bank_forks, - commitment_slots, - Bank::get_transaction_logs_adapter, - filter_logs_results, - notifier, - ) - } - - fn check_program( - program_id: &Pubkey, - bank_forks: &Arc>, - program_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let subscriptions = program_subscriptions.read().unwrap(); - check_commitment_and_notify( - &subscriptions, - program_id, - bank_forks, - commitment_slots, - Bank::get_program_accounts_modified_since_parent, - filter_program_results, - notifier, - ) - } - - fn check_signature( - signature: &Signature, - bank_forks: &Arc>, - signature_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let mut subscriptions = signature_subscriptions.write().unwrap(); - let notified_ids = check_commitment_and_notify( - &subscriptions, - signature, - bank_forks, - commitment_slots, - Bank::get_signature_status_processed_since_parent, - filter_signature_result, - notifier, - ); - if let Some(subscription_ids) = subscriptions.get_mut(signature) { - subscription_ids.retain(|k, _| !notified_ids.contains(k)); - if subscription_ids.is_empty() { - subscriptions.remove(signature); - } - } - notified_ids - } - - pub fn total(&self) -> usize { - self.subscriptions.total() - } - - pub fn add_account_subscription( - &self, - pubkey: Pubkey, - config: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let config = config.unwrap_or_default(); - let commitment = config.commitment.unwrap_or_default(); - - let slot = if commitment.is_finalized() { - self.block_commitment_cache - .read() - .unwrap() - .highest_confirmed_root() - } else if commitment.is_confirmed() { - self.optimistically_confirmed_bank - .read() - .unwrap() - .bank - .slot() - } else { - self.block_commitment_cache.read().unwrap().slot() - }; - - let last_notified_slot = if let Some((_account, slot)) = self - .bank_forks - .read() - .unwrap() - .get(slot) - .and_then(|bank| bank.get_account_modified_slot(&pubkey)) - { - slot - } else { - 0 - }; - - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_account_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.account_subscriptions.write().unwrap() - }; - - add_subscription( - &mut subscriptions, - pubkey, - commitment, - sub_id, - subscriber, - last_notified_slot, - config.encoding, - ); - } - - pub fn remove_account_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.account_subscriptions.write().unwrap(); - if remove_subscription(&mut subscriptions, id) { - true - } else { - let mut subscriptions = self - .subscriptions - .gossip_account_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - } - } - - pub fn add_program_subscription( - &self, - program_id: Pubkey, - config: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let config = config.unwrap_or_default(); - let commitment = config.account_config.commitment.unwrap_or_default(); - - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_program_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.program_subscriptions.write().unwrap() - }; - - add_subscription( - &mut subscriptions, - program_id, - commitment, - sub_id, - subscriber, - 0, // last_notified_slot is not utilized for program subscriptions - Some(ProgramConfig { - filters: config.filters.unwrap_or_default(), - encoding: config.account_config.encoding, - }), - ); - } - - pub fn remove_program_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.program_subscriptions.write().unwrap(); - if remove_subscription(&mut subscriptions, id) { - true - } else { - let mut subscriptions = self - .subscriptions - .gossip_program_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - } - } - - pub fn add_logs_subscription( - &self, - address: Option, - include_votes: bool, - commitment: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let commitment = commitment.unwrap_or_default(); - - { - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_logs_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.logs_subscriptions.write().unwrap() - }; - add_subscription( - &mut subscriptions, - (address, include_votes), - commitment, - sub_id, - subscriber, - 0, // last_notified_slot is not utilized for logs subscriptions - None, - ); - } - self.update_bank_transaction_log_keys(); - } - - pub fn remove_logs_subscription(&self, id: &SubscriptionId) -> bool { - let mut removed = { - let mut subscriptions = self.subscriptions.logs_subscriptions.write().unwrap(); - remove_subscription(&mut subscriptions, id) - }; - - if !removed { - removed = { - let mut subscriptions = self - .subscriptions - .gossip_logs_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - }; - } - - if removed { - self.update_bank_transaction_log_keys(); - } - removed - } - - fn update_bank_transaction_log_keys(&self) { - // Grab a write lock for both `logs_subscriptions` and `gossip_logs_subscriptions`, to - // ensure `Bank::transaction_log_collector_config` is updated atomically. - let logs_subscriptions = self.subscriptions.logs_subscriptions.write().unwrap(); - let gossip_logs_subscriptions = self - .subscriptions - .gossip_logs_subscriptions - .write() - .unwrap(); - - let mut config = TransactionLogCollectorConfig::default(); - - let mut all = false; - let mut all_with_votes = false; - let mut mentioned_address = false; - for (address, with_votes) in logs_subscriptions - .keys() - .chain(gossip_logs_subscriptions.keys()) - { - match address { - None => { - if *with_votes { - all_with_votes = true; - } else { - all = true; - } - } - Some(address) => { - config.mentioned_addresses.insert(*address); - mentioned_address = true; - } - } - } - config.filter = if all_with_votes { - TransactionLogCollectorFilter::AllWithVotes - } else if all { - TransactionLogCollectorFilter::All - } else if mentioned_address { - TransactionLogCollectorFilter::OnlyMentionedAddresses - } else { - TransactionLogCollectorFilter::None - }; - - *self - .bank_forks - .read() - .unwrap() - .root_bank() - .transaction_log_collector_config - .write() - .unwrap() = config; - } - - pub fn add_signature_subscription( - &self, - signature: Signature, - signature_subscribe_config: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let (commitment, enable_received_notification) = signature_subscribe_config - .map(|config| (config.commitment, config.enable_received_notification)) - .unwrap_or_default(); - - let commitment = commitment.unwrap_or_default(); - - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_signature_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.signature_subscriptions.write().unwrap() - }; - - add_subscription( - &mut subscriptions, - signature, - commitment, - sub_id, - subscriber, - 0, // last_notified_slot is not utilized for signature subscriptions - enable_received_notification, - ); - } - - pub fn remove_signature_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.signature_subscriptions.write().unwrap(); - if remove_subscription(&mut subscriptions, id) { - true - } else { - let mut subscriptions = self - .subscriptions - .gossip_signature_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - } + pub fn control(&self) -> &SubscriptionControl { + &self.control } /// Notify subscribers of changes to any accounts or new signatures since @@ -913,40 +545,6 @@ impl RpcSubscriptions { self.enqueue_notification(NotificationEntry::SlotUpdate(slot_update)); } - pub fn add_slot_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber) { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self.subscriptions.slot_subscriptions.write().unwrap(); - subscriptions.insert(sub_id, sink); - } - - pub fn remove_slot_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.slot_subscriptions.write().unwrap(); - subscriptions.remove(id).is_some() - } - - pub fn add_slots_updates_subscription( - &self, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self - .subscriptions - .slots_updates_subscriptions - .write() - .unwrap(); - subscriptions.insert(sub_id, sink); - } - - pub fn remove_slots_updates_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self - .subscriptions - .slots_updates_subscriptions - .write() - .unwrap(); - subscriptions.remove(id).is_some() - } - pub fn notify_slot(&self, slot: Slot, parent: Slot, root: Slot) { self.enqueue_notification(NotificationEntry::Slot(SlotInfo { slot, parent, root })); self.enqueue_notification(NotificationEntry::SlotUpdate(SlotUpdate::CreatedBank { @@ -960,38 +558,10 @@ impl RpcSubscriptions { self.enqueue_notification(NotificationEntry::SignaturesReceived(slot_signatures)); } - pub fn add_vote_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber) { - if self.enable_vote_subscription { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self.subscriptions.vote_subscriptions.write().unwrap(); - subscriptions.insert(sub_id, sink); - } else { - let _ = subscriber.reject(jsonrpc_core::Error::new( - jsonrpc_core::ErrorCode::MethodNotFound, - )); - } - } - - pub fn remove_vote_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.vote_subscriptions.write().unwrap(); - subscriptions.remove(id).is_some() - } - pub fn notify_vote(&self, vote: &Vote) { self.enqueue_notification(NotificationEntry::Vote(vote.clone())); } - pub fn add_root_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber) { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self.subscriptions.root_subscriptions.write().unwrap(); - subscriptions.insert(sub_id, sink); - } - - pub fn remove_root_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.root_subscriptions.write().unwrap(); - subscriptions.remove(id).is_some() - } - pub fn notify_roots(&self, mut rooted_slots: Vec) { rooted_slots.sort_unstable(); rooted_slots.into_iter().for_each(|root| { @@ -1004,12 +574,7 @@ impl RpcSubscriptions { } fn enqueue_notification(&self, notification_entry: NotificationEntry) { - match self - .notification_sender - .lock() - .unwrap() - .send(notification_entry) - { + match self.notification_sender.send(notification_entry) { Ok(()) => (), Err(SendError(notification)) => { warn!( @@ -1022,112 +587,133 @@ impl RpcSubscriptions { fn process_notifications( exit: Arc, - notifier: RpcNotifier, + mut notifier: RpcNotifier, notification_receiver: Receiver, - subscriptions: Subscriptions, + mut subscriptions: SubscriptionsTracker, bank_forks: Arc>, + block_commitment_cache: Arc>, + optimistically_confirmed_bank: Arc>, ) { loop { if exit.load(Ordering::Relaxed) { break; } match notification_receiver.recv_timeout(Duration::from_millis(RECEIVE_DELAY_MILLIS)) { - Ok(notification_entry) => match notification_entry { - NotificationEntry::Slot(slot_info) => { - let subscriptions = subscriptions.slot_subscriptions.read().unwrap(); - let num_subscriptions = subscriptions.len(); - if num_subscriptions > 0 { - debug!( - "slot notify: {:?}, num_subscriptions: {:?}", - slot_info, num_subscriptions - ); + Ok(notification_entry) => { + match notification_entry { + NotificationEntry::Subscribed(params, id) => { + subscriptions.subscribe(params.clone(), id, || { + initial_last_notified_slot( + ¶ms, + &bank_forks, + &block_commitment_cache, + &optimistically_confirmed_bank, + ) + }); } - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-slot", 1); - notifier.notify(slot_info, sink); + NotificationEntry::Unsubscribed(params, id) => { + subscriptions.unsubscribe(params, id); + } + NotificationEntry::Slot(slot_info) => { + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::Slot) + { + debug!("slot notify: {:?}", slot_info); + inc_new_counter_info!("rpc-subscription-notify-slot", 1); + notifier.notify(&slot_info, sub, false); + } + } + NotificationEntry::SlotUpdate(slot_update) => { + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::SlotsUpdates) + { + inc_new_counter_info!("rpc-subscription-notify-slots-updates", 1); + notifier.notify(&slot_update, sub, false); + } + } + // These notifications are only triggered by votes observed on gossip, + // unlike `NotificationEntry::Gossip`, which also accounts for slots seen + // in VoteState's from bank states built in ReplayStage. + NotificationEntry::Vote(ref vote_info) => { + let rpc_vote = RpcVote { + // TODO: Remove clones + slots: vote_info.slots.clone(), + hash: bs58::encode(vote_info.hash).into_string(), + timestamp: vote_info.timestamp, + }; + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::Vote) + { + debug!("vote notify: {:?}", vote_info); + inc_new_counter_info!("rpc-subscription-notify-vote", 1); + notifier.notify(&rpc_vote, sub, false); + } + } + NotificationEntry::Root(root) => { + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::Root) + { + debug!("root notify: {:?}", root); + inc_new_counter_info!("rpc-subscription-notify-root", 1); + notifier.notify(&root, sub, false); + } + } + NotificationEntry::Bank(commitment_slots) => { + RpcSubscriptions::notify_accounts_logs_programs_signatures( + subscriptions.commitment_watchers(), + &bank_forks, + &commitment_slots, + &mut notifier, + "bank", + ) + } + NotificationEntry::Gossip(slot) => { + let commitment_slots = CommitmentSlots { + highest_confirmed_slot: slot, + ..CommitmentSlots::default() + }; + + RpcSubscriptions::notify_accounts_logs_programs_signatures( + subscriptions.gossip_watchers(), + &bank_forks, + &commitment_slots, + &mut notifier, + "gossip", + ) + } + NotificationEntry::SignaturesReceived((slot, slot_signatures)) => { + for slot_signature in &slot_signatures { + if let Some(subs) = subscriptions.by_signature().get(slot_signature) + { + for subscription in subs.values() { + if let SubscriptionParams::Signature(params) = + subscription.params() + { + if params.enable_received_notification { + notifier.notify( + Response { + context: RpcResponseContext { slot }, + value: RpcSignatureResult::ReceivedSignature( + ReceivedSignatureResult::ReceivedSignature, + ), + }, + subscription, + false, + ); + } + } else { + error!("invalid params type in visit_by_signature"); + } + } + } + } } } - NotificationEntry::SlotUpdate(slot_update) => { - let subscriptions = - subscriptions.slots_updates_subscriptions.read().unwrap(); - let slot_update = Arc::new(slot_update); - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-slots-updates", 1); - notifier.notify(slot_update.clone(), sink); - } - } - // These notifications are only triggered by votes observed on gossip, - // unlike `NotificationEntry::Gossip`, which also accounts for slots seen - // in VoteState's from bank states built in ReplayStage. - NotificationEntry::Vote(ref vote_info) => { - let subscriptions = subscriptions.vote_subscriptions.read().unwrap(); - let num_subscriptions = subscriptions.len(); - if num_subscriptions > 0 { - debug!( - "vote notify: {:?}, num_subscriptions: {:?}", - vote_info, num_subscriptions - ); - } - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-vote", 1); - notifier.notify( - RpcVote { - // TODO: Remove clones - slots: vote_info.slots.clone(), - hash: bs58::encode(vote_info.hash).into_string(), - timestamp: vote_info.timestamp, - }, - sink, - ); - } - } - NotificationEntry::Root(root) => { - let subscriptions = subscriptions.root_subscriptions.read().unwrap(); - let num_subscriptions = subscriptions.len(); - if num_subscriptions > 0 { - debug!( - "root notify: {:?}, num_subscriptions: {:?}", - root, num_subscriptions - ); - } - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-root", 1); - notifier.notify(root, sink); - } - } - NotificationEntry::Bank(commitment_slots) => { - RpcSubscriptions::notify_accounts_logs_programs_signatures( - &subscriptions.account_subscriptions, - &subscriptions.logs_subscriptions, - &subscriptions.program_subscriptions, - &subscriptions.signature_subscriptions, - &bank_forks, - &commitment_slots, - ¬ifier, - "bank", - ) - } - NotificationEntry::Gossip(slot) => { - Self::process_gossip_notification( - slot, - ¬ifier, - &subscriptions, - &bank_forks, - ); - } - NotificationEntry::SignaturesReceived(slot_signatures) => { - RpcSubscriptions::process_signatures_received( - &slot_signatures, - &subscriptions.gossip_signature_subscriptions, - ¬ifier, - ); - RpcSubscriptions::process_signatures_received( - &slot_signatures, - &subscriptions.signature_subscriptions, - ¬ifier, - ); - } - }, + } Err(RecvTimeoutError::Timeout) => { // not a problem - try reading again } @@ -1139,176 +725,155 @@ impl RpcSubscriptions { } } - fn process_gossip_notification( - slot: Slot, - notifier: &RpcNotifier, - subscriptions: &Subscriptions, - bank_forks: &Arc>, - ) { - let commitment_slots = CommitmentSlots { - highest_confirmed_slot: slot, - ..CommitmentSlots::default() - }; - RpcSubscriptions::notify_accounts_logs_programs_signatures( - &subscriptions.gossip_account_subscriptions, - &subscriptions.gossip_logs_subscriptions, - &subscriptions.gossip_program_subscriptions, - &subscriptions.gossip_signature_subscriptions, - bank_forks, - &commitment_slots, - notifier, - "gossip", - ); - } - fn notify_accounts_logs_programs_signatures( - account_subscriptions: &Arc, - logs_subscriptions: &Arc, - program_subscriptions: &Arc, - signature_subscriptions: &Arc, + subscriptions: &HashMap>, bank_forks: &Arc>, commitment_slots: &CommitmentSlots, - notifier: &RpcNotifier, + notifier: &mut RpcNotifier, source: &'static str, ) { - let mut accounts_time = Measure::start("accounts"); - let pubkeys: Vec<_> = { - let subs = account_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; - let mut num_pubkeys_notified = 0; - for pubkey in &pubkeys { - num_pubkeys_notified += Self::check_account( - pubkey, - bank_forks, - account_subscriptions.clone(), - notifier, - commitment_slots, - ) - .len(); - } - accounts_time.stop(); + let mut total_time = Measure::start("notify_accounts_logs_programs_signatures"); + let mut num_accounts_found = 0; + let mut num_accounts_notified = 0; - let mut logs_time = Measure::start("logs"); - let logs: Vec<_> = { - let subs = logs_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; + let mut num_logs_found = 0; let mut num_logs_notified = 0; - for address in &logs { - num_logs_notified += Self::check_logs( - address, - bank_forks, - logs_subscriptions.clone(), - notifier, - commitment_slots, - ) - .len(); - } - logs_time.stop(); - let mut programs_time = Measure::start("programs"); - let programs: Vec<_> = { - let subs = program_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; - let mut num_programs_notified = 0; - for program_id in &programs { - num_programs_notified += Self::check_program( - program_id, - bank_forks, - program_subscriptions.clone(), - notifier, - commitment_slots, - ) - .len(); - } - programs_time.stop(); - - let mut signatures_time = Measure::start("signatures"); - let signatures: Vec<_> = { - let subs = signature_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; + let mut num_signatures_found = 0; let mut num_signatures_notified = 0; - for signature in &signatures { - num_signatures_notified += Self::check_signature( - signature, - bank_forks, - signature_subscriptions.clone(), - notifier, - commitment_slots, - ) - .len(); + + let mut num_programs_found = 0; + let mut num_programs_notified = 0; + + for subscription in subscriptions.values() { + match subscription.params() { + SubscriptionParams::Account(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + |bank, params| bank.get_account_modified_slot(¶ms.pubkey), + filter_account_result, + notifier, + false, + ); + + num_accounts_found += 1; + + if notified { + num_accounts_notified += 1; + } + } + SubscriptionParams::Logs(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + get_transaction_logs, + filter_logs_results, + notifier, + false, + ); + num_logs_found += 1; + + if notified { + num_logs_notified += 1; + } + } + SubscriptionParams::Program(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + |bank, params| { + bank.get_program_accounts_modified_since_parent(¶ms.pubkey) + }, + filter_program_results, + notifier, + false, + ); + num_programs_found += 1; + + if notified { + num_programs_notified += 1; + } + } + SubscriptionParams::Signature(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + |bank, params| { + bank.get_signature_status_processed_since_parent(¶ms.signature) + }, + filter_signature_result, + notifier, + true, // Unsubscribe. + ); + num_signatures_found += 1; + + if notified { + num_signatures_notified += 1; + } + } + _ => error!("wrong subscription type in alps map"), + } } - signatures_time.stop(); - let total_notified = num_pubkeys_notified + num_programs_notified + num_signatures_notified; - let total_ms = accounts_time.as_ms() + programs_time.as_ms() + signatures_time.as_ms(); + + total_time.stop(); + + let total_notified = num_accounts_notified + + num_logs_notified + + num_programs_notified + + num_signatures_notified; + let total_ms = total_time.as_ms(); if total_notified > 0 || total_ms > 10 { debug!( - "notified({}): accounts: {} / {} ({}) programs: {} / {} ({}) signatures: {} / {} ({})", + "notified({}): accounts: {} / {} logs: {} / {} programs: {} / {} signatures: {} / {}", source, - pubkeys.len(), - num_pubkeys_notified, - accounts_time, - programs.len(), + num_accounts_found, + num_accounts_notified, + num_logs_found, + num_logs_notified, + num_programs_found, num_programs_notified, - programs_time, - signatures.len(), + num_signatures_found, num_signatures_notified, - signatures_time, ); inc_new_counter_info!("rpc-subscription-notify-bank-or-gossip", total_notified); datapoint_info!( "rpc_subscriptions", ("source", source.to_string(), String), - ("num_account_subscriptions", pubkeys.len(), i64), - ("num_account_pubkeys_notified", num_pubkeys_notified, i64), - ("accounts_time", accounts_time.as_us() as i64, i64), - ("num_logs_subscriptions", logs.len(), i64), + ("num_account_subscriptions", num_accounts_found, i64), + ("num_account_pubkeys_notified", num_accounts_notified, i64), + ("num_logs_subscriptions", num_logs_found, i64), ("num_logs_notified", num_logs_notified, i64), - ("logs_time", logs_time.as_us() as i64, i64), - ("num_program_subscriptions", programs.len(), i64), + ("num_program_subscriptions", num_programs_found, i64), ("num_programs_notified", num_programs_notified, i64), - ("programs_time", programs_time.as_us() as i64, i64), - ("num_signature_subscriptions", signatures.len(), i64), + ("num_signature_subscriptions", num_signatures_found, i64), ("num_signatures_notified", num_signatures_notified, i64), - ("signatures_time", signatures_time.as_us() as i64, i64) + ("notifications_time", total_time.as_us() as i64, i64), + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_accounts_notified", + num_accounts_notified + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_logs_notified", + num_logs_notified + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_programs_notified", + num_programs_notified + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_signatures_notified", + num_signatures_notified ); - } - } - - fn process_signatures_received( - (received_slot, signatures): &(Slot, Vec), - signature_subscriptions: &Arc, - notifier: &RpcNotifier, - ) { - for signature in signatures { - if let Some(hashmap) = signature_subscriptions.read().unwrap().get(signature) { - for ( - _, - SubscriptionData { - sink, - config: is_received_notification_enabled, - .. - }, - ) in hashmap.iter() - { - if is_received_notification_enabled.unwrap_or_default() { - notifier.notify( - Response { - context: RpcResponseContext { - slot: *received_slot, - }, - value: RpcSignatureResult::ReceivedSignature( - ReceivedSignatureResult::ReceivedSignature, - ), - }, - sink, - ); - } - } - } } } @@ -1324,66 +889,43 @@ impl RpcSubscriptions { Ok(()) } } + + #[cfg(test)] + fn total(&self) -> usize { + self.control.total() + } } #[cfg(test)] pub(crate) mod tests { use { super::*, - crate::optimistically_confirmed_bank_tracker::{ - BankNotification, OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker, + crate::{ + optimistically_confirmed_bank_tracker::{ + BankNotification, OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker, + }, + rpc_pubsub::RpcSolPubSubInternal, + rpc_pubsub_service, }, - jsonrpc_core::futures::StreamExt, - jsonrpc_pubsub::typed::Subscriber, serial_test::serial, + solana_client::rpc_config::{ + RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig, + RpcTransactionLogsFilter, + }, solana_runtime::{ commitment::BlockCommitment, genesis_utils::{create_genesis_config, GenesisConfigInfo}, }, solana_sdk::{ + commitment_config::CommitmentConfig, message::Message, signature::{Keypair, Signer}, stake, system_instruction, system_program, system_transaction, transaction::Transaction, }, - std::{ - fmt::Debug, - sync::{atomic::Ordering::Relaxed, mpsc::channel}, - }, - tokio::{ - runtime::Runtime, - time::{sleep, timeout}, - }, + std::{collections::HashSet, sync::atomic::Ordering::Relaxed}, }; - pub(crate) fn robust_poll_or_panic( - receiver: jsonrpc_core::futures::channel::mpsc::UnboundedReceiver, - ) -> ( - T, - jsonrpc_core::futures::channel::mpsc::UnboundedReceiver, - ) { - let (inner_sender, inner_receiver) = channel(); - let rt = Runtime::new().unwrap(); - rt.spawn(async move { - let result = timeout( - Duration::from_millis(RECEIVE_DELAY_MILLIS), - receiver.into_future(), - ) - .await - .unwrap_or_else(|err| panic!("stream error {:?}", err)); - - match result { - (Some(value), receiver) => { - inner_sender.send((value, receiver)).expect("send error") - } - (None, _) => panic!("unexpected end of stream"), - } - - sleep(Duration::from_millis(RECEIVE_DELAY_MILLIS * 2)).await; - }); - inner_receiver.recv().expect("recv error") - } - fn make_account_result(lamports: u64, subscription: u64, data: &str) -> serde_json::Value { json!({ "jsonrpc": "2.0", @@ -1421,14 +963,14 @@ pub(crate) mod tests { let alice = Keypair::new(); let exit = Arc::new(AtomicBool::new(false)); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( 1, 1, ))), OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - ); + )); let tx0 = system_transaction::create_account( &mint_keypair, @@ -1464,50 +1006,57 @@ pub(crate) mod tests { (alice.pubkey(), tx2, expected2), ]; - for (i, (pubkey, tx, expected)) in subscribe_cases.iter().enumerate() { - let (sub, _id_receiver, recv) = Subscriber::new_test("accountNotification"); - let sub_id = SubscriptionId::Number(i as u64); - subscriptions.add_account_subscription( - *pubkey, - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::processed()), - encoding: None, - data_slice: None, - }), - sub_id.clone(), - sub, - ); + for (pubkey, tx, expected) in subscribe_cases { + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); - assert!(subscriptions - .subscriptions - .account_subscriptions - .read() - .unwrap() - .contains_key(pubkey)); + let sub_id = rpc + .account_subscribe( + pubkey.to_string(), + Some(RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::processed()), + encoding: None, + data_slice: None, + }), + ) + .unwrap(); + + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Account(AccountSubscriptionParams { + pubkey, + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + })); bank_forks .read() .unwrap() .get(1) .unwrap() - .process_transaction(tx) + .process_transaction(&tx) .unwrap(); let commitment_slots = CommitmentSlots { slot: 1, ..CommitmentSlots::default() }; subscriptions.notify_subscribers(commitment_slots); - let (response, _) = robust_poll_or_panic(recv); + let response = receiver.recv(); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&sub_id); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc.account_unsubscribe(sub_id).unwrap(); - assert!(!subscriptions - .subscriptions - .account_subscriptions - .read() - .unwrap() - .contains_key(pubkey)); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Account(AccountSubscriptionParams { + pubkey, + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + })); } } @@ -1539,40 +1088,42 @@ pub(crate) mod tests { .process_transaction(&tx) .unwrap(); - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, - ); - subscriptions.add_program_subscription( - stake::program::id(), - Some(RpcProgramAccountsConfig { - account_config: RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::processed()), - ..RpcAccountInfoConfig::default() - }, - ..RpcProgramAccountsConfig::default() - }), - sub_id.clone(), - subscriber, - ); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc + .program_subscribe( + stake::program::id().to_string(), + Some(RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::processed()), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }), + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .program_subscriptions - .read() - .unwrap() - .contains_key(&stake::program::id())); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: stake::program::id(), + filters: Vec::new(), + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + with_context: false, + })); subscriptions.notify_subscribers(CommitmentSlots::default()); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = json!({ "jsonrpc": "2.0", "method": "programNotification", @@ -1593,15 +1144,22 @@ pub(crate) mod tests { "subscription": 0, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - subscriptions.remove_program_subscription(&sub_id); - assert!(!subscriptions - .subscriptions - .program_subscriptions - .read() - .unwrap() - .contains_key(&stake::program::id())); + rpc.program_unsubscribe(sub_id).unwrap(); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: stake::program::id(), + filters: Vec::new(), + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + with_context: false, + })); } #[test] @@ -1674,15 +1232,12 @@ pub(crate) mod tests { bank3.process_transaction(&tx).unwrap(); // now add programSubscribe at the "confirmed" commitment level - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -1690,25 +1245,32 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - subscriptions.add_program_subscription( - stake::program::id(), - Some(RpcProgramAccountsConfig { - account_config: RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - ..RpcAccountInfoConfig::default() - }, - ..RpcProgramAccountsConfig::default() - }), - sub_id.clone(), - subscriber, - ); - assert!(subscriptions - .subscriptions - .gossip_program_subscriptions - .read() - .unwrap() - .contains_key(&stake::program::id())); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + + let sub_id = rpc + .program_subscribe( + stake::program::id().to_string(), + Some(RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }), + ) + .unwrap(); + + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: stake::program::id(), + filters: Vec::new(), + encoding: UiAccountEncoding::Binary, + data_slice: None, + commitment: CommitmentConfig::confirmed(), + with_context: false, + })); let mut highest_confirmed_slot: Slot = 0; let mut last_notified_confirmed_slot: Slot = 0; @@ -1749,13 +1311,19 @@ pub(crate) mod tests { }) }; - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(1, 1, &alice.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(2, 2, &bob.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); bank3.freeze(); OptimisticallyConfirmedBankTracker::process_notification( @@ -1769,10 +1337,13 @@ pub(crate) mod tests { &None, ); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(3, 3, &joe.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_program_subscription(&sub_id); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc.program_unsubscribe(sub_id).unwrap(); } #[test] @@ -1830,15 +1401,12 @@ pub(crate) mod tests { bank2.process_transaction(&tx).unwrap(); // now add programSubscribe at the "confirmed" commitment level - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -1846,8 +1414,9 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - subscriptions.add_program_subscription( - stake::program::id(), + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + rpc.program_subscribe( + stake::program::id().to_string(), Some(RpcProgramAccountsConfig { account_config: RpcAccountInfoConfig { commitment: Some(CommitmentConfig::confirmed()), @@ -1855,16 +1424,19 @@ pub(crate) mod tests { }, ..RpcProgramAccountsConfig::default() }), - sub_id, - subscriber, - ); + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .gossip_program_subscriptions - .read() - .unwrap() - .contains_key(&stake::program::id())); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: stake::program::id(), + filters: Vec::new(), + encoding: UiAccountEncoding::Binary, + data_slice: None, + commitment: CommitmentConfig::confirmed(), + with_context: false, + })); let mut highest_confirmed_slot: Slot = 0; let mut last_notified_confirmed_slot: Slot = 0; @@ -1882,7 +1454,7 @@ pub(crate) mod tests { ); // The following should panic - let (_response, _transport_receiver) = robust_poll_or_panic(transport_receiver); + let _response = receiver.recv(); } #[test] @@ -1939,15 +1511,12 @@ pub(crate) mod tests { bank2.process_transaction(&tx).unwrap(); // now add programSubscribe at the "confirmed" commitment level - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -1955,25 +1524,30 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - subscriptions.add_program_subscription( - stake::program::id(), - Some(RpcProgramAccountsConfig { - account_config: RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - ..RpcAccountInfoConfig::default() - }, - ..RpcProgramAccountsConfig::default() - }), - sub_id.clone(), - subscriber, - ); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc + .program_subscribe( + stake::program::id().to_string(), + Some(RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }), + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .gossip_program_subscriptions - .read() - .unwrap() - .contains_key(&stake::program::id())); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: stake::program::id(), + filters: Vec::new(), + encoding: UiAccountEncoding::Binary, + data_slice: None, + commitment: CommitmentConfig::confirmed(), + with_context: false, + })); let mut highest_confirmed_slot: Slot = 0; let mut last_notified_confirmed_slot: Slot = 0; @@ -2043,18 +1617,27 @@ pub(crate) mod tests { &None, ); - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(1, 1, &alice.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(2, 2, &bob.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(3, 3, &joe.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_program_subscription(&sub_id); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc.program_unsubscribe(sub_id).unwrap(); } #[test] @@ -2118,79 +1701,78 @@ pub(crate) mod tests { let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(block_commitment_cache)), optimistically_confirmed_bank, - ); + )); - let (past_bank_sub1, _id_receiver, past_bank_recv1) = - Subscriber::new_test("signatureNotification"); - let (past_bank_sub2, _id_receiver, past_bank_recv2) = - Subscriber::new_test("signatureNotification"); - let (processed_sub, _id_receiver, processed_recv) = - Subscriber::new_test("signatureNotification"); - let (processed_sub3, _id_receiver, processed_recv3) = - Subscriber::new_test("signatureNotification"); + let (past_bank_rpc1, mut past_bank_receiver1) = + rpc_pubsub_service::test_connection(&subscriptions); + let (past_bank_rpc2, mut past_bank_receiver2) = + rpc_pubsub_service::test_connection(&subscriptions); + let (processed_rpc, mut processed_receiver) = + rpc_pubsub_service::test_connection(&subscriptions); + let (another_rpc, _another_receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let (processed_rpc3, mut processed_receiver3) = + rpc_pubsub_service::test_connection(&subscriptions); + + let past_bank_sub_id1 = past_bank_rpc1 + .signature_subscribe( + past_bank_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); + let past_bank_sub_id2 = past_bank_rpc2 + .signature_subscribe( + past_bank_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::finalized()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); + let processed_sub_id = processed_rpc + .signature_subscribe( + processed_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); + another_rpc + .signature_subscribe( + unprocessed_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); - subscriptions.add_signature_subscription( - past_bank_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(1), - past_bank_sub1, - ); - subscriptions.add_signature_subscription( - past_bank_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::finalized()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(2), - past_bank_sub2, - ); - subscriptions.add_signature_subscription( - processed_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(3), - processed_sub, - ); - subscriptions.add_signature_subscription( - unprocessed_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(4), - Subscriber::new_test("signatureNotification").0, - ); // Add a subscription that gets `received` notifications - subscriptions.add_signature_subscription( - unprocessed_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(true), - }), - SubscriptionId::Number(5), - processed_sub3, - ); + let processed_sub_id3 = processed_rpc3 + .signature_subscribe( + unprocessed_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(true), + }), + ) + .unwrap(); + + assert!(subscriptions + .control + .signature_subscribed(&unprocessed_tx.signatures[0])); + assert!(subscriptions + .control + .signature_subscribed(&processed_tx.signatures[0])); - { - let sig_subs = subscriptions - .subscriptions - .signature_subscriptions - .read() - .unwrap(); - assert_eq!(sig_subs.get(&past_bank_tx.signatures[0]).unwrap().len(), 2); - assert!(sig_subs.contains_key(&unprocessed_tx.signatures[0])); - assert!(sig_subs.contains_key(&processed_tx.signatures[0])); - } let mut commitment_slots = CommitmentSlots::default(); let received_slot = 1; commitment_slots.slot = received_slot; @@ -2224,130 +1806,135 @@ pub(crate) mod tests { // Expect to receive a notification from bank 1 because this subscription is // looking for 0 confirmations and so checks the current bank - let expected = expected_notification(Notification { slot: 1, id: 1 }, &expected_res); - let (response, _) = robust_poll_or_panic(past_bank_recv1); + let expected = expected_notification( + Notification { + slot: 1, + id: past_bank_sub_id1.into(), + }, + &expected_res, + ); + let response = past_bank_receiver1.recv(); assert_eq!(expected, response); // Expect to receive a notification from bank 0 because this subscription is // looking for 1 confirmation and so checks the past bank - let expected = expected_notification(Notification { slot: 0, id: 2 }, &expected_res); - let (response, _) = robust_poll_or_panic(past_bank_recv2); + let expected = expected_notification( + Notification { + slot: 0, + id: past_bank_sub_id2.into(), + }, + &expected_res, + ); + let response = past_bank_receiver2.recv(); assert_eq!(expected, response); - let expected = expected_notification(Notification { slot: 1, id: 3 }, &expected_res); - let (response, _) = robust_poll_or_panic(processed_recv); + let expected = expected_notification( + Notification { + slot: 1, + id: processed_sub_id.into(), + }, + &expected_res, + ); + let response = processed_receiver.recv(); assert_eq!(expected, response); // Expect a "received" notification let expected = expected_notification( Notification { slot: received_slot, - id: 5, + id: processed_sub_id3.into(), }, &received_expected_res, ); - let (response, _) = robust_poll_or_panic(processed_recv3); + let response = processed_receiver3.recv(); assert_eq!(expected, response); // Subscription should be automatically removed after notification - let sig_subs = subscriptions - .subscriptions - .signature_subscriptions - .read() - .unwrap(); - assert!(!sig_subs.contains_key(&processed_tx.signatures[0])); - assert!(!sig_subs.contains_key(&past_bank_tx.signatures[0])); + + assert!(!subscriptions + .control + .signature_subscribed(&processed_tx.signatures[0])); + assert!(!subscriptions + .control + .signature_subscribed(&past_bank_tx.signatures[0])); // Unprocessed signature subscription should not be removed - assert_eq!( - sig_subs.get(&unprocessed_tx.signatures[0]).unwrap().len(), - 2 - ); + assert!(subscriptions + .control + .signature_subscribed(&unprocessed_tx.signatures[0])); } #[test] #[serial] fn test_check_slot_subscribe() { - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("slotNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, - ); - subscriptions.add_slot_subscription(sub_id.clone(), subscriber); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc.slot_subscribe().unwrap(); - assert!(subscriptions - .subscriptions - .slot_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Slot); subscriptions.notify_slot(0, 0, 0); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); + let expected_res = SlotInfo { parent: 0, slot: 0, root: 0, }; - let expected_res_str = - serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap(); + let expected_res_str = serde_json::to_string(&expected_res).unwrap(); + let expected = format!( r#"{{"jsonrpc":"2.0","method":"slotNotification","params":{{"result":{},"subscription":0}}}}"#, expected_res_str ); assert_eq!(expected, response); - subscriptions.remove_slot_subscription(&sub_id); - assert!(!subscriptions - .subscriptions - .slot_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); + rpc.slot_unsubscribe(sub_id).unwrap(); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Slot); } #[test] #[serial] fn test_check_root_subscribe() { - let (subscriber, _id_receiver, mut transport_receiver) = - Subscriber::new_test("rootNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, - ); - subscriptions.add_root_subscription(sub_id.clone(), subscriber); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc.root_subscribe().unwrap(); - assert!(subscriptions - .subscriptions - .root_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Root); subscriptions.notify_roots(vec![2, 1, 3]); for expected_root in 1..=3 { - let (response, receiver) = robust_poll_or_panic(transport_receiver); - transport_receiver = receiver; + let response = receiver.recv(); + let expected_res_str = serde_json::to_string(&serde_json::to_value(expected_root).unwrap()).unwrap(); let expected = format!( @@ -2357,69 +1944,10 @@ pub(crate) mod tests { assert_eq!(expected, response); } - subscriptions.remove_root_subscription(&sub_id); - assert!(!subscriptions - .subscriptions - .root_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); - } - - #[test] - #[serial] - fn test_add_and_remove_subscription() { - let mut subscriptions: HashMap>> = - HashMap::new(); - let commitment = CommitmentConfig::confirmed(); - - let num_keys = 5; - for key in 0..num_keys { - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("notification"); - let sub_id = SubscriptionId::Number(key); - add_subscription( - &mut subscriptions, - key, - commitment, - sub_id, - subscriber, - 0, - None, - ); - } - - // Add another subscription to the "0" key - let (subscriber, _id_receiver, _transport_receiver) = Subscriber::new_test("notification"); - let extra_sub_id = SubscriptionId::Number(num_keys); - add_subscription( - &mut subscriptions, - 0, - commitment, - extra_sub_id.clone(), - subscriber, - 0, - None, - ); - - assert_eq!(subscriptions.len(), num_keys as usize); - assert_eq!(subscriptions.get(&0).unwrap().len(), 2); - assert_eq!(subscriptions.get(&1).unwrap().len(), 1); - - assert!(remove_subscription( - &mut subscriptions, - &SubscriptionId::Number(0) - )); - assert_eq!(subscriptions.len(), num_keys as usize); - assert_eq!(subscriptions.get(&0).unwrap().len(), 1); - assert!(!remove_subscription( - &mut subscriptions, - &SubscriptionId::Number(0) - )); - - assert!(remove_subscription(&mut subscriptions, &extra_sub_id)); - assert_eq!(subscriptions.len(), (num_keys - 1) as usize); - assert!(subscriptions.get(&0).is_none()); + rpc.root_unsubscribe(sub_id).unwrap(); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Root); } #[test] @@ -2444,12 +1972,8 @@ pub(crate) mod tests { OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let (subscriber0, _id_receiver, transport_receiver0) = - Subscriber::new_test("accountNotification"); - let (subscriber1, _id_receiver, transport_receiver1) = - Subscriber::new_test("accountNotification"); let exit = Arc::new(AtomicBool::new(false)); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -2457,24 +1981,20 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - let sub_id0 = SubscriptionId::Number(0); - subscriptions.add_account_subscription( - alice.pubkey(), - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - encoding: None, - data_slice: None, - }), - sub_id0.clone(), - subscriber0, - ); + let (rpc0, mut receiver0) = rpc_pubsub_service::test_connection(&subscriptions); + let (rpc1, mut receiver1) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id0 = rpc0 + .account_subscribe( + alice.pubkey().to_string(), + Some(RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + encoding: None, + data_slice: None, + }), + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .gossip_account_subscriptions - .read() - .unwrap() - .contains_key(&alice.pubkey())); + assert!(subscriptions.control.account_subscribed(&alice.pubkey())); let tx = system_transaction::create_account( &mint_keypair, @@ -2526,7 +2046,7 @@ pub(crate) mod tests { &None, ); - let (response, _) = robust_poll_or_panic(transport_receiver0); + let response = receiver0.recv(); let expected = json!({ "jsonrpc": "2.0", "method": "accountNotification", @@ -2544,20 +2064,22 @@ pub(crate) mod tests { "subscription": 0, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&sub_id0); - - let sub_id1 = SubscriptionId::Number(1); - subscriptions.add_account_subscription( - alice.pubkey(), - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - encoding: None, - data_slice: None, - }), - sub_id1.clone(), - subscriber1, + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), ); + rpc0.account_unsubscribe(sub_id0).unwrap(); + + let sub_id1 = rpc1 + .account_subscribe( + alice.pubkey().to_string(), + Some(RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + encoding: None, + data_slice: None, + }), + ) + .unwrap(); let bank2 = bank_forks.read().unwrap().get(2).unwrap().clone(); bank2.freeze(); @@ -2572,7 +2094,7 @@ pub(crate) mod tests { &mut highest_confirmed_slot, &None, ); - let (response, _) = robust_poll_or_panic(transport_receiver1); + let response = receiver1.recv(); let expected = json!({ "jsonrpc": "2.0", "method": "accountNotification", @@ -2590,60 +2112,13 @@ pub(crate) mod tests { "subscription": 1, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&sub_id1); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc1.account_unsubscribe(sub_id1).unwrap(); - assert!(!subscriptions - .subscriptions - .gossip_account_subscriptions - .read() - .unwrap() - .contains_key(&alice.pubkey())); - } - - #[test] - fn test_total_nested_subscriptions() { - let mock_subscriptions = RwLock::new(HashMap::new()); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 0); - - mock_subscriptions - .write() - .unwrap() - .insert(0, HashMap::new()); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 0); - - mock_subscriptions - .write() - .unwrap() - .entry(0) - .and_modify(|map| { - map.insert(0, "test"); - }); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 1); - - mock_subscriptions - .write() - .unwrap() - .entry(0) - .and_modify(|map| { - map.insert(1, "test"); - }); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 2); - - mock_subscriptions - .write() - .unwrap() - .insert(1, HashMap::new()); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 2); - - mock_subscriptions - .write() - .unwrap() - .entry(1) - .and_modify(|map| { - map.insert(0, "test"); - }); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 3); + assert!(!subscriptions.control.account_subscribed(&alice.pubkey())); } #[test] @@ -2651,99 +2126,79 @@ pub(crate) mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100); let bank = Bank::new_for_tests(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let subscriptions = RpcSubscriptions::default_with_bank_forks(bank_forks); + let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + + let (rpc1, _receiver1) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id1 = rpc1 + .account_subscribe(Pubkey::default().to_string(), None) + .unwrap(); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("accountNotification"); - let account_sub_id = SubscriptionId::Number(0u64); - subscriptions.add_account_subscription( - Pubkey::default(), - None, - account_sub_id.clone(), - subscriber, - ); assert_eq!(subscriptions.total(), 1); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("programNotification"); - let program_sub_id = SubscriptionId::Number(1u64); - subscriptions.add_program_subscription( - Pubkey::default(), - None, - program_sub_id.clone(), - subscriber, - ); + let (rpc2, _receiver2) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id2 = rpc2 + .program_subscribe(Pubkey::default().to_string(), None) + .unwrap(); + assert_eq!(subscriptions.total(), 2); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("logsNotification"); - let logs_sub_id = SubscriptionId::Number(2u64); - subscriptions.add_logs_subscription(None, false, None, logs_sub_id.clone(), subscriber); + let (rpc3, _receiver3) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id3 = rpc3 + .logs_subscribe(RpcTransactionLogsFilter::All, None) + .unwrap(); assert_eq!(subscriptions.total(), 3); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("signatureNotification"); - let sig_sub_id = SubscriptionId::Number(3u64); - subscriptions.add_signature_subscription( - Signature::default(), - None, - sig_sub_id.clone(), - subscriber, - ); + let (rpc4, _receiver4) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id4 = rpc4 + .signature_subscribe(Signature::default().to_string(), None) + .unwrap(); + assert_eq!(subscriptions.total(), 4); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("slotNotification"); - let slot_sub_id = SubscriptionId::Number(4u64); - subscriptions.add_slot_subscription(slot_sub_id.clone(), subscriber); + let (rpc5, _receiver5) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id5 = rpc5.slot_subscribe().unwrap(); + assert_eq!(subscriptions.total(), 5); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("voteNotification"); - let vote_sub_id = SubscriptionId::Number(5u64); - subscriptions.add_vote_subscription(vote_sub_id.clone(), subscriber); + let (rpc6, _receiver6) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id6 = rpc6.vote_subscribe().unwrap(); + assert_eq!(subscriptions.total(), 6); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("rootNotification"); - let root_sub_id = SubscriptionId::Number(6u64); - subscriptions.add_root_subscription(root_sub_id.clone(), subscriber); + let (rpc7, _receiver7) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id7 = rpc7.root_subscribe().unwrap(); + assert_eq!(subscriptions.total(), 7); - // Add duplicate account subscription to ensure totals include all subscriptions on all keys - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("accountNotification2"); - let account_dupe_sub_id = SubscriptionId::Number(7u64); - subscriptions.add_account_subscription( - Pubkey::default(), - None, - account_dupe_sub_id.clone(), - subscriber, - ); - assert_eq!(subscriptions.total(), 8); - - subscriptions.remove_account_subscription(&account_sub_id); + // Add duplicate account subscription, but it shouldn't increment the count. + let (rpc8, _receiver8) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id8 = rpc8 + .account_subscribe(Pubkey::default().to_string(), None) + .unwrap(); assert_eq!(subscriptions.total(), 7); - subscriptions.remove_account_subscription(&account_dupe_sub_id); + rpc1.account_unsubscribe(sub_id1).unwrap(); + assert_eq!(subscriptions.total(), 7); + + rpc8.account_unsubscribe(sub_id8).unwrap(); assert_eq!(subscriptions.total(), 6); - subscriptions.remove_program_subscription(&program_sub_id); + rpc2.program_unsubscribe(sub_id2).unwrap(); assert_eq!(subscriptions.total(), 5); - subscriptions.remove_logs_subscription(&logs_sub_id); + rpc3.logs_unsubscribe(sub_id3).unwrap(); assert_eq!(subscriptions.total(), 4); - subscriptions.remove_signature_subscription(&sig_sub_id); + rpc4.signature_unsubscribe(sub_id4).unwrap(); assert_eq!(subscriptions.total(), 3); - subscriptions.remove_slot_subscription(&slot_sub_id); + rpc5.slot_unsubscribe(sub_id5).unwrap(); assert_eq!(subscriptions.total(), 2); - subscriptions.remove_vote_subscription(&vote_sub_id); + rpc6.vote_unsubscribe(sub_id6).unwrap(); assert_eq!(subscriptions.total(), 1); - subscriptions.remove_root_subscription(&root_sub_id); + rpc7.root_unsubscribe(sub_id7).unwrap(); assert_eq!(subscriptions.total(), 0); } } diff --git a/sdk/src/commitment_config.rs b/sdk/src/commitment_config.rs index 0551a068c0..1e77c65968 100644 --- a/sdk/src/commitment_config.rs +++ b/sdk/src/commitment_config.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use thiserror::Error; -#[derive(Serialize, Deserialize, Default, Clone, Copy, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Default, Clone, Copy, Debug, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] pub struct CommitmentConfig { pub commitment: CommitmentLevel, diff --git a/validator/src/main.rs b/validator/src/main.rs index ee598272e1..fe2ebe2e13 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1059,15 +1059,13 @@ pub fn main() { &format!("{}-{}", VALIDATOR_PORT_RANGE.0, VALIDATOR_PORT_RANGE.1); let default_genesis_archive_unpacked_size = &MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string(); let default_rpc_max_multiple_accounts = &MAX_MULTIPLE_ACCOUNTS.to_string(); - let default_rpc_pubsub_max_connections = PubSubConfig::default().max_connections.to_string(); - let default_rpc_pubsub_max_fragment_size = - PubSubConfig::default().max_fragment_size.to_string(); - let default_rpc_pubsub_max_in_buffer_capacity = - PubSubConfig::default().max_in_buffer_capacity.to_string(); - let default_rpc_pubsub_max_out_buffer_capacity = - PubSubConfig::default().max_out_buffer_capacity.to_string(); + let default_rpc_pubsub_max_active_subscriptions = PubSubConfig::default().max_active_subscriptions.to_string(); + let default_rpc_pubsub_queue_capacity_items = + PubSubConfig::default().queue_capacity_items.to_string(); + let default_rpc_pubsub_queue_capacity_bytes = + PubSubConfig::default().queue_capacity_bytes.to_string(); let default_rpc_send_transaction_retry_ms = ValidatorConfig::default() .send_transaction_retry_ms .to_string(); @@ -1737,10 +1735,10 @@ pub fn main() { .value_name("NUMBER") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_connections) + .hidden(true) .help("The maximum number of connections that RPC PubSub will support. \ This is a hard limit and no new connections beyond this limit can \ - be made until an old connection is dropped."), + be made until an old connection is dropped. (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_fragment_size") @@ -1748,9 +1746,9 @@ pub fn main() { .value_name("BYTES") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_fragment_size) + .hidden(true) .help("The maximum length in bytes of acceptable incoming frames. Messages longer \ - than this will be rejected."), + than this will be rejected. (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_in_buffer_capacity") @@ -1758,8 +1756,9 @@ pub fn main() { .value_name("BYTES") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_in_buffer_capacity) - .help("The maximum size in bytes to which the incoming websocket buffer can grow."), + .hidden(true) + .help("The maximum size in bytes to which the incoming websocket buffer can grow. \ + (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_out_buffer_capacity") @@ -1767,8 +1766,9 @@ pub fn main() { .value_name("BYTES") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_out_buffer_capacity) - .help("The maximum size in bytes to which the outgoing websocket buffer can grow."), + .hidden(true) + .help("The maximum size in bytes to which the outgoing websocket buffer can grow. \ + (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_active_subscriptions") @@ -1780,6 +1780,26 @@ pub fn main() { .help("The maximum number of active subscriptions that RPC PubSub will accept \ across all connections."), ) + .arg( + Arg::with_name("rpc_pubsub_queue_capacity_items") + .long("rpc-pubsub-queue-capacity-items") + .takes_value(true) + .value_name("NUMBER") + .validator(is_parsable::) + .default_value(&default_rpc_pubsub_queue_capacity_items) + .help("The maximum number of notifications that RPC PubSub will store \ + across all connections."), + ) + .arg( + Arg::with_name("rpc_pubsub_queue_capacity_bytes") + .long("rpc-pubsub-queue-capacity-bytes") + .takes_value(true) + .value_name("BYTES") + .validator(is_parsable::) + .default_value(&default_rpc_pubsub_queue_capacity_bytes) + .help("The maximum total size of notifications that RPC PubSub will store \ + across all connections."), + ) .arg( Arg::with_name("rpc_send_transaction_retry_ms") .long("rpc-send-retry-ms") @@ -2569,23 +2589,21 @@ pub fn main() { }), pubsub_config: PubSubConfig { enable_vote_subscription: matches.is_present("rpc_pubsub_enable_vote_subscription"), - max_connections: value_t_or_exit!(matches, "rpc_pubsub_max_connections", usize), - max_fragment_size: value_t_or_exit!(matches, "rpc_pubsub_max_fragment_size", usize), - max_in_buffer_capacity: value_t_or_exit!( - matches, - "rpc_pubsub_max_in_buffer_capacity", - usize - ), - max_out_buffer_capacity: value_t_or_exit!( - matches, - "rpc_pubsub_max_out_buffer_capacity", - usize - ), max_active_subscriptions: value_t_or_exit!( matches, "rpc_pubsub_max_active_subscriptions", usize ), + queue_capacity_items: value_t_or_exit!( + matches, + "rpc_pubsub_queue_capacity_items", + usize + ), + queue_capacity_bytes: value_t_or_exit!( + matches, + "rpc_pubsub_queue_capacity_bytes", + usize + ), }, voting_disabled: matches.is_present("no_voting") || restricted_repair_only_mode, wait_for_supermajority: value_t!(matches, "wait_for_supermajority", Slot).ok(),