Merge pull request #195 from blockworks-foundation/restructuring_core

restructuring core library into multiple directories

cluster-endpoints/src/endpoint_stremers.rs

@@ -1,6 +1,4 @@
-use solana_lite_rpc_core::streams::{
+use solana_lite_rpc_core::types::{BlockStream, ClusterInfoStream, SlotStream, VoteAccountStream};
-    BlockStream, ClusterInfoStream, SlotStream, VoteAccountStream,
-};
 pub struct EndpointStreaming {
     pub blocks_notifier: BlockStream,
     pub slot_notifier: SlotStream,

cluster-endpoints/src/json_rpc_leaders_getter.rs

@@ -2,7 +2,9 @@ use anyhow::{bail, Context};
 use async_trait::async_trait;
 use itertools::Itertools;
 use solana_client::nonblocking::rpc_client::RpcClient;
-use solana_lite_rpc_core::leaders_fetcher_trait::{LeaderData, LeaderFetcherInterface};
+use solana_lite_rpc_core::{
+    structures::leader_data::LeaderData, traits::leaders_fetcher_interface::LeaderFetcherInterface,
+};
 use std::{collections::VecDeque, sync::Arc};
 use tokio::sync::RwLock;
 

core/src/lib.rs

@@ -1,18 +1,10 @@
-pub mod block_information_store;
-pub mod cluster_info;
-pub mod data_cache;
 pub mod keypair_loader;
-pub mod leaders_fetcher_trait;
-pub mod notifications;
-pub mod proxy_request_format;
 pub mod quic_connection;
 pub mod quic_connection_utils;
-pub mod rotating_queue;
 pub mod solana_utils;
-pub mod streams;
+pub mod stores;
 pub mod structures;
-pub mod subscription_handler;
+pub mod traits;
-pub mod subscription_sink;
+pub mod types;
-pub mod tx_store;
 
 pub type AnyhowJoinHandle = tokio::task::JoinHandle<anyhow::Result<()>>;

core/src/quic_connection.rs

@@ -1,6 +1,6 @@
 use crate::{
     quic_connection_utils::{QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils},
-    rotating_queue::RotatingQueue,
+    structures::rotating_queue::RotatingQueue,
 };
 use anyhow::bail;
 use log::warn;

core/src/stores/cluster_info_store.rs

@@ -3,7 +3,7 @@ use solana_rpc_client_api::response::RpcContactInfo;
 use solana_sdk::pubkey::Pubkey;
 use std::{str::FromStr, sync::Arc};
 
-use crate::streams::ClusterInfoStream;
+use crate::types::ClusterInfoStream;
 
 #[derive(Debug, Clone, Default)]
 pub struct ClusterInfo {

core/src/stores/data_cache.rs

@@ -4,14 +4,14 @@ use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::slot_history::Slot;
 
 use crate::{
-    block_information_store::BlockInformationStore,
+    stores::{
-    cluster_info::ClusterInfo,
+        block_information_store::BlockInformationStore, cluster_info_store::ClusterInfo,
+        subscription_store::SubscriptionStore, tx_store::TxStore,
+    },
     structures::{
         identity_stakes::IdentityStakes,
         slot_notification::{AtomicSlot, SlotNotification},
     },
-    subscription_handler::SubscriptionHandler,
-    tx_store::TxStore,
 };
 pub type TxSubKey = (String, CommitmentConfig);
 
@@ -26,7 +26,7 @@ pub struct SlotCache {
 pub struct DataCache {
     pub block_store: BlockInformationStore,
     pub txs: TxStore,
-    pub tx_subs: SubscriptionHandler,
+    pub tx_subs: SubscriptionStore,
     pub slot_cache: SlotCache,
     pub identity_stakes: IdentityStakes,
     pub cluster_info: ClusterInfo,

core/src/stores/mod.rs

@@ -0,0 +1,7 @@
+// this mod will contain all the different stores that are used by lite-rpc
+
+pub mod block_information_store;
+pub mod cluster_info_store;
+pub mod data_cache;
+pub mod subscription_store;
+pub mod tx_store;

core/src/stores/subscription_store.rs

@@ -1,23 +1,19 @@
-use std::{sync::Arc, time::Duration};
+use crate::{structures::processed_block::TransactionInfo, types::SubscptionHanderSink};
-
 use dashmap::DashMap;
 use solana_sdk::{
     commitment_config::{CommitmentConfig, CommitmentLevel},
     slot_history::Slot,
 };
+use std::{sync::Arc, time::Duration};
 use tokio::time::Instant;
 
-use crate::{structures::processed_block::TransactionInfo, subscription_sink::SubscriptionSink};
-
-pub type SubscptionHanderSink = Arc<dyn SubscriptionSink>;
-
 #[derive(Clone, Default)]
-pub struct SubscriptionHandler {
+pub struct SubscriptionStore {
     pub signature_subscribers:
         Arc<DashMap<(String, CommitmentConfig), (SubscptionHanderSink, Instant)>>,
 }
 
-impl SubscriptionHandler {
+impl SubscriptionStore {
     #[allow(deprecated)]
     pub fn get_supported_commitment_config(
         commitment_config: CommitmentConfig,

core/src/structures/leader_data.rs

@@ -0,0 +1,7 @@
+use solana_sdk::{pubkey::Pubkey, slot_history::Slot};
+
+#[derive(Debug, Clone)]
+pub struct LeaderData {
+    pub leader_slot: Slot,
+    pub pubkey: Pubkey,
+}

core/src/structures/mod.rs

@@ -1,3 +1,9 @@
+// this mod will contain all the core structures that are defined for lite-rpc
+
 pub mod identity_stakes;
+pub mod leader_data;
+pub mod notifications;
 pub mod processed_block;
+pub mod proxy_request_format;
+pub mod rotating_queue;
 pub mod slot_notification;

core/src/traits/block_storage_interface.rs

@@ -0,0 +1,12 @@
+use async_trait::async_trait;
+use solana_sdk::{commitment_config::CommitmentLevel, slot_history::Slot};
+use solana_transaction_status::UiConfirmedBlock;
+use std::sync::Arc;
+
+#[async_trait]
+pub trait BlockStorageInterface: Send + Sync {
+    async fn save(&self, slot: Slot, block: UiConfirmedBlock, commitment: CommitmentLevel);
+    async fn get(&self, slot: Slot) -> Option<UiConfirmedBlock>;
+}
+
+pub type BlockStorageImpl = Arc<dyn BlockStorageInterface>;

core/src/traits/leaders_fetcher_interface.rs

@@ -1,11 +1,6 @@
+use crate::structures::leader_data::LeaderData;
 use async_trait::async_trait;
-use solana_sdk::{pubkey::Pubkey, slot_history::Slot};
+use solana_sdk::slot_history::Slot;
-
-#[derive(Debug, Clone)]
-pub struct LeaderData {
-    pub leader_slot: Slot,
-    pub pubkey: Pubkey,
-}
 
 #[async_trait]
 pub trait LeaderFetcherInterface: Send + Sync {

core/src/traits/mod.rs

@@ -0,0 +1,3 @@
+pub mod block_storage_interface;
+pub mod leaders_fetcher_interface;
+pub mod subscription_sink;

core/src/types.rs

@@ -1,9 +1,15 @@
+use std::sync::Arc;
+
 use solana_rpc_client_api::response::{RpcContactInfo, RpcVoteAccountStatus};
 use tokio::sync::broadcast::Receiver;
 
-use crate::structures::{processed_block::ProcessedBlock, slot_notification::SlotNotification};
+use crate::{
+    structures::{processed_block::ProcessedBlock, slot_notification::SlotNotification},
+    traits::subscription_sink::SubscriptionSink,
+};
 
 pub type BlockStream = Receiver<ProcessedBlock>;
 pub type SlotStream = Receiver<SlotNotification>;
 pub type VoteAccountStream = Receiver<RpcVoteAccountStatus>;
 pub type ClusterInfoStream = Receiver<Vec<RpcContactInfo>>;
+pub type SubscptionHanderSink = Arc<dyn SubscriptionSink>;

lite-rpc/src/bridge.rs

@@ -13,7 +13,8 @@ use jsonrpsee::{core::SubscriptionResult, server::ServerBuilder, PendingSubscrip
 use log::info;
 use prometheus::{opts, register_int_counter, IntCounter};
 use solana_lite_rpc_core::{
-    block_information_store::BlockInformation, data_cache::DataCache, AnyhowJoinHandle,
+    stores::{block_information_store::BlockInformation, data_cache::DataCache, tx_store::TxProps},
+    AnyhowJoinHandle,
 };
 use solana_rpc_client::nonblocking::rpc_client::RpcClient;
 use solana_rpc_client_api::{
@@ -293,7 +294,7 @@ impl LiteRpcServer for LiteBridge {
         {
             self.data_cache.txs.insert(
                 airdrop_sig.clone(),
-                solana_lite_rpc_core::tx_store::TxProps {
+                TxProps {
                     status: None,
                     last_valid_blockheight: block_height,
                 },

lite-rpc/src/jsonrpsee_subscrption_handler_sink.rs

@@ -13,7 +13,7 @@ impl JsonRpseeSubscriptionHandlerSink {
 }
 
 #[async_trait]
-impl solana_lite_rpc_core::subscription_sink::SubscriptionSink
+impl solana_lite_rpc_core::traits::subscription_sink::SubscriptionSink
     for JsonRpseeSubscriptionHandlerSink
 {
     async fn send(&self, slot: solana_sdk::slot_history::Slot, message: serde_json::Value) {

lite-rpc/src/main.rs

@@ -14,17 +14,20 @@ use solana_lite_rpc_cluster_endpoints::endpoint_stremers::EndpointStreaming;
 use solana_lite_rpc_cluster_endpoints::grpc_subscription::create_grpc_subscription;
 use solana_lite_rpc_cluster_endpoints::json_rpc_leaders_getter::JsonRpcLeaderGetter;
 use solana_lite_rpc_cluster_endpoints::json_rpc_subscription::create_json_rpc_polling_subscription;
-use solana_lite_rpc_core::block_information_store::{BlockInformation, BlockInformationStore};
-use solana_lite_rpc_core::cluster_info::ClusterInfo;
-use solana_lite_rpc_core::data_cache::{DataCache, SlotCache};
 use solana_lite_rpc_core::keypair_loader::load_identity_keypair;
-use solana_lite_rpc_core::notifications::NotificationSender;
 use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters;
-use solana_lite_rpc_core::streams::BlockStream;
+use solana_lite_rpc_core::stores::{
-use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes;
+    block_information_store::{BlockInformation, BlockInformationStore},
-use solana_lite_rpc_core::structures::processed_block::ProcessedBlock;
+    cluster_info_store::ClusterInfo,
-use solana_lite_rpc_core::subscription_handler::SubscriptionHandler;
+    data_cache::{DataCache, SlotCache},
-use solana_lite_rpc_core::tx_store::TxStore;
+    subscription_store::SubscriptionStore,
+    tx_store::TxStore,
+};
+use solana_lite_rpc_core::structures::{
+    identity_stakes::IdentityStakes, notifications::NotificationSender,
+    processed_block::ProcessedBlock,
+};
+use solana_lite_rpc_core::types::BlockStream;
 use solana_lite_rpc_core::AnyhowJoinHandle;
 use solana_lite_rpc_services::data_caching_service::DataCachingService;
 use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath;
@@ -120,7 +123,7 @@ pub async fn start_lite_rpc(args: Args, rpc_client: Arc<RpcClient>) -> anyhow::R
         cluster_info: ClusterInfo::default(),
         identity_stakes: IdentityStakes::new(validator_identity.pubkey()),
         slot_cache: SlotCache::new(finalized_block.slot),
-        tx_subs: SubscriptionHandler::default(),
+        tx_subs: SubscriptionStore::default(),
         txs: TxStore::default(),
     };
 

lite-rpc/src/postgres.rs

@@ -13,7 +13,7 @@ use native_tls::{Certificate, Identity, TlsConnector};
 
 use crate::encoding::BinaryEncoding;
 use solana_lite_rpc_core::{
-    notifications::{
+    structures::notifications::{
         BlockNotification, NotificationMsg, NotificationReciever, TransactionNotification,
         TransactionUpdateNotification,
     },

lite-rpc/src/service_spawner.rs

@@ -1,7 +1,7 @@
 use solana_lite_rpc_core::{
-    data_cache::DataCache,
+    stores::data_cache::DataCache,
-    notifications::NotificationSender,
+    structures::notifications::NotificationSender,
-    streams::{BlockStream, ClusterInfoStream, SlotStream, VoteAccountStream},
+    types::{BlockStream, ClusterInfoStream, SlotStream, VoteAccountStream},
     AnyhowJoinHandle,
 };
 use solana_lite_rpc_services::{

quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs

@@ -5,8 +5,8 @@ use log::{debug, error, info, trace, warn};
 
 use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters;
 use solana_lite_rpc_core::solana_utils::SerializableTransaction;
+use solana_lite_rpc_core::stores::tx_store::empty_tx_store;
 use solana_lite_rpc_core::structures::identity_stakes::IdentityStakesData;
-use solana_lite_rpc_core::tx_store::empty_tx_store;
 use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager;
 use solana_sdk::hash::Hash;
 use solana_sdk::instruction::Instruction;

services/src/data_caching_service.rs

@@ -3,11 +3,10 @@ use std::time::Duration;
 use anyhow::{bail, Context};
 use prometheus::core::GenericGauge;
 use prometheus::{opts, register_int_counter, register_int_gauge, IntCounter};
-use solana_lite_rpc_core::block_information_store::BlockInformation;
+use solana_lite_rpc_core::stores::{
-use solana_lite_rpc_core::data_cache::DataCache;
+    block_information_store::BlockInformation, data_cache::DataCache,
-use solana_lite_rpc_core::streams::{
-    BlockStream, ClusterInfoStream, SlotStream, VoteAccountStream,
 };
+use solana_lite_rpc_core::types::{BlockStream, ClusterInfoStream, SlotStream, VoteAccountStream};
 use solana_lite_rpc_core::AnyhowJoinHandle;
 use solana_sdk::commitment_config::CommitmentLevel;
 use solana_transaction_status::{TransactionConfirmationStatus, TransactionStatus};

services/src/metrics_capture.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use log::info;
 use prometheus::{core::GenericGauge, opts, register_int_gauge};
 use serde::{Deserialize, Serialize};
-use solana_lite_rpc_core::tx_store::TxStore;
+use solana_lite_rpc_core::stores::tx_store::TxStore;
 use solana_transaction_status::TransactionConfirmationStatus;
 use tokio::{sync::RwLock, task::JoinHandle};
 

services/src/tpu_utils/quic_proxy_connection_manager.rs

@@ -15,10 +15,10 @@ use solana_sdk::pubkey::Pubkey;
 use tokio::sync::broadcast::error::TryRecvError;
 use tokio::sync::{broadcast::Receiver, RwLock};
 
-use solana_lite_rpc_core::proxy_request_format::{TpuForwardingRequest, TxData};
 use solana_lite_rpc_core::quic_connection_utils::{
     QuicConnectionParameters, SkipServerVerification,
 };
+use solana_lite_rpc_core::structures::proxy_request_format::{TpuForwardingRequest, TxData};
 
 use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect;
 

services/src/tpu_utils/tpu_connection_manager.rs

@@ -5,9 +5,8 @@ use quinn::Endpoint;
 use solana_lite_rpc_core::{
     quic_connection::QuicConnectionPool,
     quic_connection_utils::{QuicConnectionParameters, QuicConnectionUtils},
-    rotating_queue::RotatingQueue,
+    stores::tx_store::TxStore,
-    structures::identity_stakes::IdentityStakesData,
+    structures::{identity_stakes::IdentityStakesData, rotating_queue::RotatingQueue},
-    tx_store::TxStore,
 };
 use solana_sdk::pubkey::Pubkey;
 use solana_streamer::nonblocking::quic::compute_max_allowed_uni_streams;

services/src/tpu_utils/tpu_service.rs

@@ -5,10 +5,10 @@ use super::tpu_connection_manager::TpuConnectionManager;
 use crate::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager;
 use crate::tpu_utils::tpu_connection_path::TpuConnectionPath;
 use crate::tpu_utils::tpu_service::ConnectionManager::{DirectTpu, QuicProxy};
-use solana_lite_rpc_core::data_cache::DataCache;
-use solana_lite_rpc_core::leaders_fetcher_trait::LeaderFetcherInterface;
 use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters;
-use solana_lite_rpc_core::streams::SlotStream;
+use solana_lite_rpc_core::stores::data_cache::DataCache;
+use solana_lite_rpc_core::traits::leaders_fetcher_interface::LeaderFetcherInterface;
+use solana_lite_rpc_core::types::SlotStream;
 use solana_lite_rpc_core::AnyhowJoinHandle;
 use solana_sdk::{quic::QUIC_PORT_OFFSET, signature::Keypair, slot_history::Slot};
 use solana_streamer::tls_certificates::new_self_signed_tls_certificate;

services/src/transaction_replayer.rs

@@ -2,7 +2,7 @@ use crate::tpu_utils::tpu_service::TpuService;
 use anyhow::bail;
 use log::error;
 use prometheus::{core::GenericGauge, opts, register_int_gauge};
-use solana_lite_rpc_core::{tx_store::TxStore, AnyhowJoinHandle};
+use solana_lite_rpc_core::{stores::tx_store::TxStore, AnyhowJoinHandle};
 use std::time::Duration;
 use tokio::{
     sync::mpsc::{UnboundedReceiver, UnboundedSender},

services/src/transaction_service.rs

@@ -9,12 +9,12 @@ use crate::{
     tx_sender::{TransactionInfo, TxSender},
 };
 use anyhow::bail;
+use solana_lite_rpc_core::{solana_utils::SerializableTransaction, types::SlotStream};
 use solana_lite_rpc_core::{
-    block_information_store::{BlockInformation, BlockInformationStore},
+    stores::block_information_store::{BlockInformation, BlockInformationStore},
-    notifications::NotificationSender,
+    structures::notifications::NotificationSender,
     AnyhowJoinHandle,
 };
-use solana_lite_rpc_core::{solana_utils::SerializableTransaction, streams::SlotStream};
 use solana_sdk::transaction::VersionedTransaction;
 use tokio::{
     sync::mpsc::{self, Sender, UnboundedSender},

services/src/tx_sender.rs

@@ -13,9 +13,8 @@ use tokio::sync::mpsc::Receiver;
 
 use crate::tpu_utils::tpu_service::TpuService;
 use solana_lite_rpc_core::{
-    data_cache::DataCache,
+    stores::{data_cache::DataCache, tx_store::TxProps},
-    notifications::{NotificationMsg, NotificationSender, TransactionNotification},
+    structures::notifications::{NotificationMsg, NotificationSender, TransactionNotification},
-    tx_store::TxProps,
     AnyhowJoinHandle,
 };