adding missing snapshot library in repo
This commit is contained in:
godmodegalactus
parent
181071acb7
commit
fb59cb7aa5
|
|
@ -65,9 +65,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "agave-geyser-plugin-interface"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee6df27237b95d4eb1b9188da9425391bb7093cf24c2d3bb3705b6e27f863e31"
|
||||
checksum = "d440e759d65ccedd8b6aa2b0b4e3821f68a91ede65c6a43b0aab663b3c539ba1"
|
||||
dependencies = [
|
||||
"log",
|
||||
"solana-sdk",
|
||||
|
|
@ -3814,9 +3814,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-account-decoder"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52346da8fbbac45fdfbb9c09f7a4e263fabd13f401352e1feedc55e1178a8ba2"
|
||||
checksum = "b94ceb26c7d19530cb1bb49bf0f817647cb5fee691dae6779e19d33ac1d4fda1"
|
||||
dependencies = [
|
||||
"Inflector",
|
||||
"base64 0.21.7",
|
||||
|
|
@ -3839,9 +3839,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-config-program"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2010ba6fe2a1c4270ca3d3ef23ebfd893e3d2c980b9c0fc04451c4ce2f6b3deb"
|
||||
checksum = "378259800dc9dad34828d8be4ce0de71146bac1cbbd310f8901f6f19d92c5ea3"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"chrono",
|
||||
|
|
@ -3853,9 +3853,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-frozen-abi"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c00a6aca244dfa904e2c4a26406ba7b0987344ceaec932f3cda0b35eff0babc"
|
||||
checksum = "780402262644f9efe9ac7d885812d245007fe65fd56a3dfed83ed30d61b44c74"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"bs58",
|
||||
|
|
@ -3878,9 +3878,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-frozen-abi-macro"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bed58b27b9b8877893f69bc5cfd1c62e984315e0229d83cf8a32ad0933c0d6c9"
|
||||
checksum = "df836de37aba77234c7afa1d857dc450fb9983572e4c6f595c84cdda65a63792"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -3890,9 +3890,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-logger"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bee2daf61ae582edf9634adf8e5021faf002df0d3f69078ecbcd6c7b41bdf833"
|
||||
checksum = "9906be6edd0e1b579510736c153dbc51e5968808098d1b1f8c89dbea960aba58"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"lazy_static",
|
||||
|
|
@ -3901,9 +3901,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-measure"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "148512f384b82cf9e8bfe80503b688340d42a4cc17cfd572b88a6d803a488527"
|
||||
checksum = "7dc77e7f99fa5e845437ac9a593cd4bd67b5f9e4ba4a9578355eef25d3e839e9"
|
||||
dependencies = [
|
||||
"log",
|
||||
"solana-sdk",
|
||||
|
|
@ -3911,9 +3911,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-metrics"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55d734099c26f81621bd1aaddb8788908e20fd7fac28fb00402d564964eae4ea"
|
||||
checksum = "b0a9f68887ac31f84ef69365bdc2d7ca6bf19d50a9c6ee10806adb033e24e318"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"gethostname",
|
||||
|
|
@ -3926,9 +3926,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-program"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f4908f360900d0a1aa81c7bad7937c78f0825c3f08ff0b22f1de0e43e5946f2"
|
||||
checksum = "48ecc7af7594674687260a4d7bcfb0588cefdbe9d0f6c4e9f3140999107566c4"
|
||||
dependencies = [
|
||||
"ark-bn254",
|
||||
"ark-ec",
|
||||
|
|
@ -3981,9 +3981,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-program-runtime"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47c8ace7f999a8278351ea86ed93f57e7833cb65fb04167a9ba9ea593e995288"
|
||||
checksum = "0ef9218f50470228ebca12bb147650ca7052678aad915a4e19687ee215f8d947"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"bincode",
|
||||
|
|
@ -4009,9 +4009,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-rpc-client"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f2259b63faca1132e3a0c8b98438fb60e5d25897260dd3655bcf4ec8c6f2bf8"
|
||||
checksum = "d1adab0dcdc851fc7bc6ca8c6926d9f56ed3982f1e4fabd67d362647b57143d3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.21.7",
|
||||
|
|
@ -4035,9 +4035,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-rpc-client-api"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0aea25d581de77ba256b81f4ebd8d963b85ec01d70a74829365e85f6403d497"
|
||||
checksum = "6d6764712822bbc0259bbb5413377798a11462221863d000082f39968ce5ad03"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"bs58",
|
||||
|
|
@ -4057,9 +4057,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-sdk"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c50ec330850953d4971b052ff98c74a8e67e7618b4aed9f4971b8d3b68fcd1cd"
|
||||
checksum = "73bb113fa17e0607343afdc795c2c5230981c5b51c99b2c54fba91755879d65b"
|
||||
dependencies = [
|
||||
"assert_matches",
|
||||
"base64 0.21.7",
|
||||
|
|
@ -4112,9 +4112,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-sdk-macro"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95ef2ea49002d1bf52a4a8509570b2c3b88e7b6d0a131b11bbd637ca1e1df0ff"
|
||||
checksum = "8fcdb3a94e2f04d856d2fba6feb4f6887a1da21a3ee0b64b69c08d15dc22d46c"
|
||||
dependencies = [
|
||||
"bs58",
|
||||
"proc-macro2",
|
||||
|
|
@ -4131,9 +4131,9 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183"
|
|||
|
||||
[[package]]
|
||||
name = "solana-transaction-status"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0439563ffb7471a0b51446f0fff5c8b2108e31248bf7dbab8b9efaa2af3a4c27"
|
||||
checksum = "6adbd8f3fccddeae87278a105dcf8a8792f8816c0f4fb5f7ae8f307af279ac49"
|
||||
dependencies = [
|
||||
"Inflector",
|
||||
"base64 0.21.7",
|
||||
|
|
@ -4156,9 +4156,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-version"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb25449b519a334103778e2fc1c5c0e3ea7862ae2c1ffe90fc82ce3c96058171"
|
||||
checksum = "ac8714cf9f6caefc403e19770ad73ed2e4c866b7201e31dd17a9e06b6a693a57"
|
||||
dependencies = [
|
||||
"log",
|
||||
"rustc_version",
|
||||
|
|
@ -4172,9 +4172,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-vote-program"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78899849d1131b2fbbe9f826080cc18cec5598da63a77357642c9cd8b1a86a86"
|
||||
checksum = "b5de2428939c6e279901d4357bf02c809739e5b97164e8620e09a9e0b55c2327"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"log",
|
||||
|
|
@ -4194,9 +4194,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "solana-zk-token-sdk"
|
||||
version = "1.18.15"
|
||||
version = "1.18.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5cafb3df56516086f65e2a08a8cd03f504236f3b5348299abd45415d1d18ba32"
|
||||
checksum = "8dad1753ec3b189879c8756ac35471467116dfc93d7aeb68cfd28f22a02c850d"
|
||||
dependencies = [
|
||||
"aes-gcm-siv",
|
||||
"base64 0.21.7",
|
||||
|
|
|
|||
15
Cargo.toml
15
Cargo.toml
|
|
@ -22,13 +22,14 @@ license = "AGPL"
|
|||
edition = "2021"
|
||||
|
||||
[workspace.dependencies]
|
||||
solana-sdk = "~1.18.15"
|
||||
solana-program = "~1.18.15"
|
||||
solana-transaction-status = "~1.18.15"
|
||||
solana-logger = "~1.18.15"
|
||||
solana-rpc-client = "~1.18.15"
|
||||
solana-rpc-client-api = "~1.18.15"
|
||||
solana-account-decoder = "~1.18.15"
|
||||
solana-sdk = "~1.18.16"
|
||||
solana-program = "~1.18.16"
|
||||
solana-transaction-status = "~1.18.16"
|
||||
solana-logger = "~1.18.16"
|
||||
solana-rpc-client = "~1.18.16"
|
||||
solana-rpc-client-api = "~1.18.16"
|
||||
solana-account-decoder = "~1.18.16"
|
||||
agave-geyser-plugin-interface = "=1.18.16"
|
||||
|
||||
itertools = "0.10.5"
|
||||
serde = "1.0.201"
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ name = "config-check"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
agave-geyser-plugin-interface = "=1.18.15"
|
||||
agave-geyser-plugin-interface = {workspace = true}
|
||||
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
serde = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
[package]
|
||||
name = "quic-geyser-snapshot"
|
||||
version = "0.1.5"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
solana-sdk = { workspace = true }
|
||||
solana-program = { workspace = true }
|
||||
solana-transaction-status = { workspace = true }
|
||||
|
||||
quic-geyser-common = {workspace = true}
|
||||
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
log = { workspace = true }
|
||||
thiserror = {workspace = true}
|
||||
itertools = { workspace = true }
|
||||
lz4 = { workspace = true }
|
||||
bincode = { workspace = true }
|
||||
tokio = {workspace = true}
|
||||
|
||||
lite-account-manager-common = { workspace = true }
|
||||
lite-account-storage = { workspace = true }
|
||||
lite-token-account-storage = { workspace = true }
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
pub mod snapshot_config;
|
||||
pub mod snapshot_creator;
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct SnapshotConfig {
|
||||
pub enabled: bool,
|
||||
pub load_program_ids_except: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for SnapshotConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
load_program_ids_except: vec![
|
||||
solana_program::system_program::id().to_string(),
|
||||
solana_program::stake::program::id().to_string(),
|
||||
solana_program::vote::program::id().to_string(),
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
use std::{collections::HashMap, str::FromStr, sync::Arc};
|
||||
|
||||
use itertools::Itertools;
|
||||
use lite_account_manager_common::{
|
||||
account_data::{Account as AccountManagerAccount, AccountData as AccountManagerAccountData},
|
||||
account_filter::{AccountFilter, AccountFilterType},
|
||||
account_filters_interface::AccountFiltersStoreInterface,
|
||||
account_store_interface::{AccountLoadingError, AccountStorageInterface},
|
||||
commitment::Commitment,
|
||||
except_filter_store::ExceptFilterStore,
|
||||
slot_info::SlotInfo,
|
||||
};
|
||||
use lite_account_storage::{
|
||||
inmemory_account_store::InmemoryAccountStore, storage_by_program_id::StorageByProgramId,
|
||||
};
|
||||
use lite_token_account_storage::{
|
||||
inmemory_token_account_storage::InmemoryTokenAccountStorage,
|
||||
inmemory_token_storage::TokenProgramAccountsStorage, TOKEN_PROGRAM_2022_ID, TOKEN_PROGRAM_ID,
|
||||
};
|
||||
use quic_geyser_common::{channel_message::ChannelMessage, config::CompressionParameters};
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use crate::snapshot_config::SnapshotConfig;
|
||||
|
||||
pub struct SnapshotCreator {
|
||||
storage: Arc<dyn AccountStorageInterface>,
|
||||
filters: Arc<dyn AccountFiltersStoreInterface>,
|
||||
compression_mode: CompressionParameters,
|
||||
}
|
||||
|
||||
impl SnapshotCreator {
|
||||
pub fn new(config: SnapshotConfig, compression_mode: CompressionParameters) -> SnapshotCreator {
|
||||
let program_ids_except = config
|
||||
.load_program_ids_except
|
||||
.iter()
|
||||
.map(|pk| Pubkey::from_str(pk).unwrap())
|
||||
.collect_vec();
|
||||
let mut program_storage_map = HashMap::new();
|
||||
let token_account_storage = Arc::new(InmemoryTokenAccountStorage::default());
|
||||
let token_program_storage: Arc<dyn AccountStorageInterface> =
|
||||
Arc::new(TokenProgramAccountsStorage::new(token_account_storage));
|
||||
program_storage_map.insert(TOKEN_PROGRAM_2022_ID, token_program_storage.clone());
|
||||
program_storage_map.insert(TOKEN_PROGRAM_ID, token_program_storage);
|
||||
let mut filters = ExceptFilterStore::default();
|
||||
|
||||
let account_filters = program_ids_except
|
||||
.iter()
|
||||
.map(|f| AccountFilter {
|
||||
accounts: vec![],
|
||||
program_id: Some(*f),
|
||||
filters: None,
|
||||
})
|
||||
.collect_vec();
|
||||
filters.add_account_filters(&account_filters);
|
||||
let filters = Arc::new(filters);
|
||||
let default = Arc::new(InmemoryAccountStore::new(filters.clone()));
|
||||
|
||||
SnapshotCreator {
|
||||
storage: Arc::new(StorageByProgramId::new(program_storage_map, default)),
|
||||
filters,
|
||||
compression_mode,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_listening(
|
||||
&self,
|
||||
mut recieve_channel: tokio::sync::mpsc::UnboundedReceiver<ChannelMessage>,
|
||||
) {
|
||||
let storage = self.storage.clone();
|
||||
let filters = self.filters.clone();
|
||||
let compression = self.compression_mode.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(message) = recieve_channel.recv().await {
|
||||
match message {
|
||||
ChannelMessage::Account(account, slot, is_init) => {
|
||||
let tmp_acc = AccountManagerAccountData {
|
||||
pubkey: account.pubkey,
|
||||
account: Arc::new(AccountManagerAccount {
|
||||
lamports: account.account.lamports,
|
||||
data: lite_account_manager_common::account_data::Data::Uncompressed(
|
||||
vec![],
|
||||
),
|
||||
owner: account.account.owner,
|
||||
executable: account.account.executable,
|
||||
rent_epoch: account.account.rent_epoch,
|
||||
}),
|
||||
updated_slot: slot,
|
||||
write_version: 0,
|
||||
};
|
||||
// check first if filter is satified
|
||||
if !filters.satisfies(&tmp_acc).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
let is_token_program = account.account.owner == TOKEN_PROGRAM_2022_ID
|
||||
|| account.account.owner == TOKEN_PROGRAM_ID;
|
||||
let data = account.account.data;
|
||||
let data_len = data.len();
|
||||
let may_be_compressed_data = if is_token_program {
|
||||
lite_account_manager_common::account_data::Data::Uncompressed(data)
|
||||
} else {
|
||||
match compression.compression_type {
|
||||
quic_geyser_common::compression::CompressionType::None => {
|
||||
lite_account_manager_common::account_data::Data::Uncompressed(
|
||||
data,
|
||||
)
|
||||
}
|
||||
quic_geyser_common::compression::CompressionType::Lz4Fast(_)
|
||||
| quic_geyser_common::compression::CompressionType::Lz4(_) => {
|
||||
lite_account_manager_common::account_data::Data::Lz4 {
|
||||
binary: compression.compression_type.compress(&data),
|
||||
len: data_len,
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let account_to_save = AccountManagerAccountData {
|
||||
pubkey: account.pubkey,
|
||||
account: Arc::new(AccountManagerAccount {
|
||||
lamports: account.account.lamports,
|
||||
data: may_be_compressed_data,
|
||||
owner: account.account.owner,
|
||||
executable: account.account.executable,
|
||||
rent_epoch: account.account.rent_epoch,
|
||||
}),
|
||||
updated_slot: slot,
|
||||
write_version: 0,
|
||||
};
|
||||
if is_init {
|
||||
storage.initilize_or_update_account(account_to_save).await
|
||||
} else {
|
||||
storage
|
||||
.update_account(
|
||||
account_to_save,
|
||||
lite_account_manager_common::commitment::Commitment::Processed,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
ChannelMessage::Slot(slot, parent, commitment) => {
|
||||
storage
|
||||
.process_slot_data(
|
||||
SlotInfo {
|
||||
slot,
|
||||
parent,
|
||||
root: 0,
|
||||
},
|
||||
commitment.into(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
_ => {
|
||||
// other message are not treated
|
||||
}
|
||||
}
|
||||
}
|
||||
log::error!("snapshot creator listening thread stopped");
|
||||
panic!();
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn create_snapshot(
|
||||
&self,
|
||||
program_id: Pubkey,
|
||||
) -> Result<Vec<u8>, AccountLoadingError> {
|
||||
self.storage.create_snapshot(program_id).await
|
||||
}
|
||||
|
||||
pub async fn get_program_accounts(
|
||||
&self,
|
||||
program_pubkey: Pubkey,
|
||||
account_filters: Option<Vec<AccountFilterType>>,
|
||||
commitment: Commitment,
|
||||
) -> Result<Vec<AccountManagerAccountData>, AccountLoadingError> {
|
||||
self.storage
|
||||
.get_program_accounts(program_pubkey, account_filters, commitment)
|
||||
.await
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue