diff --git a/Cargo.lock b/Cargo.lock index aa4eff2..4d9ad4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,9 +65,9 @@ dependencies = [ [[package]] name = "agave-geyser-plugin-interface" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6df27237b95d4eb1b9188da9425391bb7093cf24c2d3bb3705b6e27f863e31" +checksum = "d440e759d65ccedd8b6aa2b0b4e3821f68a91ede65c6a43b0aab663b3c539ba1" dependencies = [ "log", "solana-sdk", @@ -3814,9 +3814,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52346da8fbbac45fdfbb9c09f7a4e263fabd13f401352e1feedc55e1178a8ba2" +checksum = "b94ceb26c7d19530cb1bb49bf0f817647cb5fee691dae6779e19d33ac1d4fda1" dependencies = [ "Inflector", "base64 0.21.7", @@ -3839,9 +3839,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2010ba6fe2a1c4270ca3d3ef23ebfd893e3d2c980b9c0fc04451c4ce2f6b3deb" +checksum = "378259800dc9dad34828d8be4ce0de71146bac1cbbd310f8901f6f19d92c5ea3" dependencies = [ "bincode", "chrono", @@ -3853,9 +3853,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c00a6aca244dfa904e2c4a26406ba7b0987344ceaec932f3cda0b35eff0babc" +checksum = "780402262644f9efe9ac7d885812d245007fe65fd56a3dfed83ed30d61b44c74" dependencies = [ "block-buffer 0.10.4", "bs58", @@ -3878,9 +3878,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bed58b27b9b8877893f69bc5cfd1c62e984315e0229d83cf8a32ad0933c0d6c9" +checksum = "df836de37aba77234c7afa1d857dc450fb9983572e4c6f595c84cdda65a63792" dependencies = [ "proc-macro2", "quote", @@ -3890,9 +3890,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee2daf61ae582edf9634adf8e5021faf002df0d3f69078ecbcd6c7b41bdf833" +checksum = "9906be6edd0e1b579510736c153dbc51e5968808098d1b1f8c89dbea960aba58" dependencies = [ "env_logger", "lazy_static", @@ -3901,9 +3901,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "148512f384b82cf9e8bfe80503b688340d42a4cc17cfd572b88a6d803a488527" +checksum = "7dc77e7f99fa5e845437ac9a593cd4bd67b5f9e4ba4a9578355eef25d3e839e9" dependencies = [ "log", "solana-sdk", @@ -3911,9 +3911,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55d734099c26f81621bd1aaddb8788908e20fd7fac28fb00402d564964eae4ea" +checksum = "b0a9f68887ac31f84ef69365bdc2d7ca6bf19d50a9c6ee10806adb033e24e318" dependencies = [ "crossbeam-channel", "gethostname", @@ -3926,9 +3926,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4908f360900d0a1aa81c7bad7937c78f0825c3f08ff0b22f1de0e43e5946f2" +checksum = "48ecc7af7594674687260a4d7bcfb0588cefdbe9d0f6c4e9f3140999107566c4" dependencies = [ "ark-bn254", "ark-ec", @@ -3981,9 +3981,9 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c8ace7f999a8278351ea86ed93f57e7833cb65fb04167a9ba9ea593e995288" +checksum = "0ef9218f50470228ebca12bb147650ca7052678aad915a4e19687ee215f8d947" dependencies = [ "base64 0.21.7", "bincode", @@ -4009,9 +4009,9 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2259b63faca1132e3a0c8b98438fb60e5d25897260dd3655bcf4ec8c6f2bf8" +checksum = "d1adab0dcdc851fc7bc6ca8c6926d9f56ed3982f1e4fabd67d362647b57143d3" dependencies = [ "async-trait", "base64 0.21.7", @@ -4035,9 +4035,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0aea25d581de77ba256b81f4ebd8d963b85ec01d70a74829365e85f6403d497" +checksum = "6d6764712822bbc0259bbb5413377798a11462221863d000082f39968ce5ad03" dependencies = [ "base64 0.21.7", "bs58", @@ -4057,9 +4057,9 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50ec330850953d4971b052ff98c74a8e67e7618b4aed9f4971b8d3b68fcd1cd" +checksum = "73bb113fa17e0607343afdc795c2c5230981c5b51c99b2c54fba91755879d65b" dependencies = [ "assert_matches", "base64 0.21.7", @@ -4112,9 +4112,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ef2ea49002d1bf52a4a8509570b2c3b88e7b6d0a131b11bbd637ca1e1df0ff" +checksum = "8fcdb3a94e2f04d856d2fba6feb4f6887a1da21a3ee0b64b69c08d15dc22d46c" dependencies = [ "bs58", "proc-macro2", @@ -4131,9 +4131,9 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-transaction-status" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0439563ffb7471a0b51446f0fff5c8b2108e31248bf7dbab8b9efaa2af3a4c27" +checksum = "6adbd8f3fccddeae87278a105dcf8a8792f8816c0f4fb5f7ae8f307af279ac49" dependencies = [ "Inflector", "base64 0.21.7", @@ -4156,9 +4156,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb25449b519a334103778e2fc1c5c0e3ea7862ae2c1ffe90fc82ce3c96058171" +checksum = "ac8714cf9f6caefc403e19770ad73ed2e4c866b7201e31dd17a9e06b6a693a57" dependencies = [ "log", "rustc_version", @@ -4172,9 +4172,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78899849d1131b2fbbe9f826080cc18cec5598da63a77357642c9cd8b1a86a86" +checksum = "b5de2428939c6e279901d4357bf02c809739e5b97164e8620e09a9e0b55c2327" dependencies = [ "bincode", "log", @@ -4194,9 +4194,9 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.18.15" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cafb3df56516086f65e2a08a8cd03f504236f3b5348299abd45415d1d18ba32" +checksum = "8dad1753ec3b189879c8756ac35471467116dfc93d7aeb68cfd28f22a02c850d" dependencies = [ "aes-gcm-siv", "base64 0.21.7", diff --git a/Cargo.toml b/Cargo.toml index cf653eb..3d86674 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,13 +22,14 @@ license = "AGPL" edition = "2021" [workspace.dependencies] -solana-sdk = "~1.18.15" -solana-program = "~1.18.15" -solana-transaction-status = "~1.18.15" -solana-logger = "~1.18.15" -solana-rpc-client = "~1.18.15" -solana-rpc-client-api = "~1.18.15" -solana-account-decoder = "~1.18.15" +solana-sdk = "~1.18.16" +solana-program = "~1.18.16" +solana-transaction-status = "~1.18.16" +solana-logger = "~1.18.16" +solana-rpc-client = "~1.18.16" +solana-rpc-client-api = "~1.18.16" +solana-account-decoder = "~1.18.16" +agave-geyser-plugin-interface = "=1.18.16" itertools = "0.10.5" serde = "1.0.201" diff --git a/plugin/Cargo.toml b/plugin/Cargo.toml index 7ee322f..74e7c40 100644 --- a/plugin/Cargo.toml +++ b/plugin/Cargo.toml @@ -13,7 +13,7 @@ name = "config-check" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -agave-geyser-plugin-interface = "=1.18.15" +agave-geyser-plugin-interface = {workspace = true} clap = { workspace = true, features = ["derive"] } serde = { workspace = true } diff --git a/snapshot/Cargo.toml b/snapshot/Cargo.toml new file mode 100644 index 0000000..7439e94 --- /dev/null +++ b/snapshot/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "quic-geyser-snapshot" +version = "0.1.5" +edition = "2021" + +[dependencies] +solana-sdk = { workspace = true } +solana-program = { workspace = true } +solana-transaction-status = { workspace = true } + +quic-geyser-common = {workspace = true} + +serde = { workspace = true } +serde_json = { workspace = true } +anyhow = { workspace = true } +log = { workspace = true } +thiserror = {workspace = true} +itertools = { workspace = true } +lz4 = { workspace = true } +bincode = { workspace = true } +tokio = {workspace = true} + +lite-account-manager-common = { workspace = true } +lite-account-storage = { workspace = true } +lite-token-account-storage = { workspace = true } diff --git a/snapshot/src/lib.rs b/snapshot/src/lib.rs new file mode 100644 index 0000000..d0c460f --- /dev/null +++ b/snapshot/src/lib.rs @@ -0,0 +1,2 @@ +pub mod snapshot_config; +pub mod snapshot_creator; diff --git a/snapshot/src/snapshot_config.rs b/snapshot/src/snapshot_config.rs new file mode 100644 index 0000000..1b7c36e --- /dev/null +++ b/snapshot/src/snapshot_config.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SnapshotConfig { + pub enabled: bool, + pub load_program_ids_except: Vec, +} + +impl Default for SnapshotConfig { + fn default() -> Self { + Self { + enabled: false, + load_program_ids_except: vec![ + solana_program::system_program::id().to_string(), + solana_program::stake::program::id().to_string(), + solana_program::vote::program::id().to_string(), + ], + } + } +} diff --git a/snapshot/src/snapshot_creator.rs b/snapshot/src/snapshot_creator.rs new file mode 100644 index 0000000..a14826c --- /dev/null +++ b/snapshot/src/snapshot_creator.rs @@ -0,0 +1,180 @@ +use std::{collections::HashMap, str::FromStr, sync::Arc}; + +use itertools::Itertools; +use lite_account_manager_common::{ + account_data::{Account as AccountManagerAccount, AccountData as AccountManagerAccountData}, + account_filter::{AccountFilter, AccountFilterType}, + account_filters_interface::AccountFiltersStoreInterface, + account_store_interface::{AccountLoadingError, AccountStorageInterface}, + commitment::Commitment, + except_filter_store::ExceptFilterStore, + slot_info::SlotInfo, +}; +use lite_account_storage::{ + inmemory_account_store::InmemoryAccountStore, storage_by_program_id::StorageByProgramId, +}; +use lite_token_account_storage::{ + inmemory_token_account_storage::InmemoryTokenAccountStorage, + inmemory_token_storage::TokenProgramAccountsStorage, TOKEN_PROGRAM_2022_ID, TOKEN_PROGRAM_ID, +}; +use quic_geyser_common::{channel_message::ChannelMessage, config::CompressionParameters}; +use solana_sdk::pubkey::Pubkey; + +use crate::snapshot_config::SnapshotConfig; + +pub struct SnapshotCreator { + storage: Arc, + filters: Arc, + compression_mode: CompressionParameters, +} + +impl SnapshotCreator { + pub fn new(config: SnapshotConfig, compression_mode: CompressionParameters) -> SnapshotCreator { + let program_ids_except = config + .load_program_ids_except + .iter() + .map(|pk| Pubkey::from_str(pk).unwrap()) + .collect_vec(); + let mut program_storage_map = HashMap::new(); + let token_account_storage = Arc::new(InmemoryTokenAccountStorage::default()); + let token_program_storage: Arc = + Arc::new(TokenProgramAccountsStorage::new(token_account_storage)); + program_storage_map.insert(TOKEN_PROGRAM_2022_ID, token_program_storage.clone()); + program_storage_map.insert(TOKEN_PROGRAM_ID, token_program_storage); + let mut filters = ExceptFilterStore::default(); + + let account_filters = program_ids_except + .iter() + .map(|f| AccountFilter { + accounts: vec![], + program_id: Some(*f), + filters: None, + }) + .collect_vec(); + filters.add_account_filters(&account_filters); + let filters = Arc::new(filters); + let default = Arc::new(InmemoryAccountStore::new(filters.clone())); + + SnapshotCreator { + storage: Arc::new(StorageByProgramId::new(program_storage_map, default)), + filters, + compression_mode, + } + } + + pub fn start_listening( + &self, + mut recieve_channel: tokio::sync::mpsc::UnboundedReceiver, + ) { + let storage = self.storage.clone(); + let filters = self.filters.clone(); + let compression = self.compression_mode.clone(); + tokio::spawn(async move { + while let Some(message) = recieve_channel.recv().await { + match message { + ChannelMessage::Account(account, slot, is_init) => { + let tmp_acc = AccountManagerAccountData { + pubkey: account.pubkey, + account: Arc::new(AccountManagerAccount { + lamports: account.account.lamports, + data: lite_account_manager_common::account_data::Data::Uncompressed( + vec![], + ), + owner: account.account.owner, + executable: account.account.executable, + rent_epoch: account.account.rent_epoch, + }), + updated_slot: slot, + write_version: 0, + }; + // check first if filter is satified + if !filters.satisfies(&tmp_acc).await { + continue; + } + + let is_token_program = account.account.owner == TOKEN_PROGRAM_2022_ID + || account.account.owner == TOKEN_PROGRAM_ID; + let data = account.account.data; + let data_len = data.len(); + let may_be_compressed_data = if is_token_program { + lite_account_manager_common::account_data::Data::Uncompressed(data) + } else { + match compression.compression_type { + quic_geyser_common::compression::CompressionType::None => { + lite_account_manager_common::account_data::Data::Uncompressed( + data, + ) + } + quic_geyser_common::compression::CompressionType::Lz4Fast(_) + | quic_geyser_common::compression::CompressionType::Lz4(_) => { + lite_account_manager_common::account_data::Data::Lz4 { + binary: compression.compression_type.compress(&data), + len: data_len, + } + } + } + }; + + let account_to_save = AccountManagerAccountData { + pubkey: account.pubkey, + account: Arc::new(AccountManagerAccount { + lamports: account.account.lamports, + data: may_be_compressed_data, + owner: account.account.owner, + executable: account.account.executable, + rent_epoch: account.account.rent_epoch, + }), + updated_slot: slot, + write_version: 0, + }; + if is_init { + storage.initilize_or_update_account(account_to_save).await + } else { + storage + .update_account( + account_to_save, + lite_account_manager_common::commitment::Commitment::Processed, + ) + .await; + } + } + ChannelMessage::Slot(slot, parent, commitment) => { + storage + .process_slot_data( + SlotInfo { + slot, + parent, + root: 0, + }, + commitment.into(), + ) + .await; + } + _ => { + // other message are not treated + } + } + } + log::error!("snapshot creator listening thread stopped"); + panic!(); + }); + } + + pub async fn create_snapshot( + &self, + program_id: Pubkey, + ) -> Result, AccountLoadingError> { + self.storage.create_snapshot(program_id).await + } + + pub async fn get_program_accounts( + &self, + program_pubkey: Pubkey, + account_filters: Option>, + commitment: Commitment, + ) -> Result, AccountLoadingError> { + self.storage + .get_program_accounts(program_pubkey, account_filters, commitment) + .await + } +}