Compare commits
No commits in common. "88f8d71ba8008f4db43d9969c4a63294d14fb4fb" and "e4bb528237ab0e0027832a0c62e2ef05f01b581b" have entirely different histories.
88f8d71ba8
...
e4bb528237
|
@ -126,55 +126,6 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is-terminal",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.82"
|
||||
|
@ -744,54 +695,6 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"bitflags 1.3.2",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.58",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "3.8.1"
|
||||
|
@ -1365,7 +1268,6 @@ dependencies = [
|
|||
"async-stream",
|
||||
"base64 0.21.7",
|
||||
"bincode",
|
||||
"clap",
|
||||
"csv",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
|
@ -1461,12 +1363,6 @@ version = "0.14.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
|
@ -1689,17 +1585,6 @@ version = "2.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
|
||||
dependencies = [
|
||||
"hermit-abi 0.3.9",
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
|
@ -2331,7 +2216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"itertools 0.12.1",
|
||||
"log",
|
||||
"multimap",
|
||||
|
@ -3973,12 +3858,6 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
|
|
|
@ -35,7 +35,6 @@ dashmap = "5.5.3"
|
|||
|
||||
tonic-health = "0.10.2"
|
||||
regex = "1.10.4"
|
||||
clap = { version = "4.2", features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = "0.3.16"
|
||||
|
|
|
@ -1,156 +0,0 @@
|
|||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::io::BufRead;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use regex::Regex;
|
||||
use clap::Parser;
|
||||
|
||||
pub fn parse_log_entry_subscriber(log_entry: &str) -> (u64, u64) {
|
||||
|
||||
let re = Regex::new(r".*got account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
|
||||
let caps = re.captures(log_entry).unwrap();
|
||||
|
||||
// let mut result = HashMap::new();
|
||||
// result.insert("write_version".to_string(), caps["write_version"].to_string());
|
||||
// result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
|
||||
// result.insert("slot".to_string(), caps["slot"].to_string());
|
||||
|
||||
let write_version: u64 = caps["write_version"].parse().unwrap();
|
||||
let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
|
||||
|
||||
(write_version, timestamp_us)
|
||||
}
|
||||
|
||||
pub fn parse_log_entry_source(log_entry: &str) -> (u64, u64) {
|
||||
let re = Regex::new(r".*account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
|
||||
let caps = re.captures(log_entry).unwrap();
|
||||
|
||||
// let mut result = HashMap::new();
|
||||
// result.insert("write_version".to_string(), caps["write_version"].to_string());
|
||||
// result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
|
||||
// result.insert("slot".to_string(), caps["slot"].to_string());
|
||||
|
||||
let write_version: u64 = caps["write_version"].parse().unwrap();
|
||||
let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
|
||||
|
||||
(write_version, timestamp_us)
|
||||
}
|
||||
|
||||
|
||||
|
||||
fn read_subscriber_log(log_file: PathBuf) -> HashMap<u64, u64> {
|
||||
let mut map: HashMap<u64, u64> = HashMap::new();
|
||||
|
||||
let file = File::open(log_file).expect("file must exist");
|
||||
let reader = io::BufReader::new(file);
|
||||
for line in reader.lines().take(1000) {
|
||||
let line = line.expect("must be parsable");
|
||||
let (write_version, timestamp_us) = parse_log_entry_subscriber(&line);
|
||||
// println!("{:?}", parsed);
|
||||
map.insert(write_version, timestamp_us);
|
||||
}
|
||||
|
||||
map
|
||||
}
|
||||
|
||||
fn read_source_log(log_file: PathBuf) -> HashMap<u64, u64> {
|
||||
let mut map: HashMap<u64, u64> = HashMap::new();
|
||||
|
||||
let file = File::open(log_file).expect("file must exist");
|
||||
let reader = io::BufReader::new(file);
|
||||
for line in reader.lines().take(1000) {
|
||||
let line = line.expect("must be parsable");
|
||||
let (write_version, timestamp_us) = parse_log_entry_source(&line);
|
||||
// println!("{:?}", parsed);
|
||||
map.insert(write_version, timestamp_us);
|
||||
}
|
||||
|
||||
map
|
||||
}
|
||||
|
||||
// cat macbook.log |cut -b 111- | tr -d 'a-z_=' > macbook.log.csv
|
||||
// cat solana-validator-macbook.log | cut -b 96- | tr -d 'a-z_='
|
||||
fn read_from_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
|
||||
csv::ReaderBuilder::new()
|
||||
.delimiter(b';')
|
||||
.has_headers(false)
|
||||
.from_path(csv_file)
|
||||
.unwrap()
|
||||
.into_deserialize()
|
||||
.map(|record| {
|
||||
let record: Vec<String> = record.unwrap();
|
||||
let write_version = record[0].parse::<u64>().unwrap();
|
||||
let timestamp_us = record[1].parse::<u64>().unwrap();
|
||||
(write_version, timestamp_us)
|
||||
})
|
||||
.collect::<HashMap<u64, u64>>()
|
||||
}
|
||||
|
||||
fn read_subscriber_log_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
|
||||
csv::ReaderBuilder::new()
|
||||
.delimiter(b';')
|
||||
.has_headers(false)
|
||||
.from_path(csv_file)
|
||||
.unwrap()
|
||||
.into_deserialize()
|
||||
.map(|record| {
|
||||
let record: Vec<String> = record.unwrap();
|
||||
let write_version = record[0].parse::<u64>().unwrap();
|
||||
let timestamp_us = record[1].parse::<u64>().unwrap();
|
||||
(write_version, timestamp_us)
|
||||
})
|
||||
.collect::<HashMap<u64, u64>>()
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
pub struct Args {
|
||||
#[arg(long)]
|
||||
pub csv_file_source: String,
|
||||
#[arg(long)]
|
||||
pub csv_file_subscriber: String,
|
||||
}
|
||||
|
||||
|
||||
pub fn main() {
|
||||
let Args { csv_file_source, csv_file_subscriber } = Args::parse();
|
||||
|
||||
println!("Reading source log ...");
|
||||
let source_timestamps = read_from_csv(PathBuf::from(csv_file_source));
|
||||
|
||||
println!("Reading subscriber log ...");
|
||||
let subscriber_timestamps = read_from_csv(PathBuf::from(csv_file_subscriber));
|
||||
|
||||
for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
|
||||
if let Some(source_timestamp) = source_timestamps.get(&write_version) {
|
||||
let diff = (timestamp_us as i128) - (*source_timestamp as i128);
|
||||
println!("write_version: {}, subscriber: {}, source: {}, diff: {:.1}ms", write_version, timestamp_us, source_timestamp, diff as f64 / 1000.0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pub fn main__() {
|
||||
println!("Reading subscriber log ...");
|
||||
let subscriber_timestamps = read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
|
||||
|
||||
println!("Reading source log ...");
|
||||
let source_timestamps = read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
|
||||
|
||||
println!("Comparing ...");
|
||||
|
||||
for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
|
||||
|
||||
// println!("write_version: {}, subscriber: {}", write_version, timestamp_us);
|
||||
if let Some(source_timestamp) = source_timestamps.get(&write_version) {
|
||||
let diff = (timestamp_us as i128) - (*source_timestamp as i128);
|
||||
println!("write_version: {}, subscriber: {}, source: {}, diff: {}", write_version, timestamp_us, source_timestamp, diff);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -19,11 +19,11 @@ use tokio::time::{sleep, Duration};
|
|||
use tracing::field::debug;
|
||||
use tracing::warn;
|
||||
use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
|
||||
use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeUpdate};
|
||||
use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
|
||||
use yellowstone_grpc_proto::prost::Message as _;
|
||||
|
||||
|
||||
const ENABLE_TIMESTAMP_TAGGING: bool = false;
|
||||
const ENABLE_TIMESTAMP_TAGGING: bool = true;
|
||||
|
||||
#[tokio::main]
|
||||
pub async fn main() {
|
||||
|
@ -63,26 +63,14 @@ pub async fn main() {
|
|||
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut bytes_per_slot: HashMap<Slot, u64> = HashMap::new();
|
||||
let mut updates_per_slot: HashMap<Slot, u64> = HashMap::new();
|
||||
|
||||
let mut changing_slot = 0;
|
||||
let mut current_slot = 0;
|
||||
|
||||
|
||||
let mut green_stream = pin!(green_stream);
|
||||
while let Some(message) = green_stream.next().await {
|
||||
match message {
|
||||
Message::GeyserSubscribeUpdate(subscriber_update) => {
|
||||
match subscriber_update.update_oneof {
|
||||
Some(UpdateOneof::Slot(update)) => {
|
||||
current_slot = update.slot;
|
||||
}
|
||||
Some(UpdateOneof::Account(update)) => {
|
||||
let slot = update.slot as Slot;
|
||||
let account = update.account.unwrap();
|
||||
let account_pk = Pubkey::try_from(account.pubkey).unwrap();
|
||||
let size = account.data.len() as u64;
|
||||
trace!("got account update (green)!!! {} - {:?} - {} bytes",
|
||||
update.slot, account_pk, account.data.len());
|
||||
|
||||
|
@ -91,7 +79,6 @@ pub async fn main() {
|
|||
info!("got account update: write_version={};timestamp_us={};slot={}", account.write_version, since_the_epoch.as_micros(), update.slot);
|
||||
}
|
||||
|
||||
|
||||
match parse_token(&account.data, Some(6)) {
|
||||
Ok(TokenAccountType::Account(account_ui)) => {
|
||||
// UiTokenAccount {
|
||||
|
@ -123,30 +110,10 @@ pub async fn main() {
|
|||
if account_ui.owner.starts_with("66fEFnKy") {
|
||||
info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
|
||||
}
|
||||
// if pubkey.starts_with(b"JUP") {
|
||||
// info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
|
||||
// }
|
||||
|
||||
token_account_by_ownermint.entry(owner)
|
||||
.or_insert_with(DashMap::new)
|
||||
.insert(mint, account_ui);
|
||||
|
||||
bytes_per_slot.entry(slot)
|
||||
.and_modify(|total| *total += size).or_insert(size);
|
||||
|
||||
updates_per_slot.entry(slot)
|
||||
.and_modify(|total| *total += 1).or_insert(1);
|
||||
|
||||
info!("delta: {}", (slot as i64) - (current_slot as i64));
|
||||
|
||||
if slot != changing_slot && changing_slot != 0 {
|
||||
let total_bytes = bytes_per_slot.get(&changing_slot).unwrap();
|
||||
let updates_count = updates_per_slot.get(&changing_slot).unwrap();
|
||||
// info!("Slot {} - Total bytes: {} in {} updates", slot, total_bytes, updates_count);
|
||||
}
|
||||
changing_slot = slot;
|
||||
|
||||
|
||||
}
|
||||
Ok(TokenAccountType::Mint(mint)) => {
|
||||
// not interesting
|
||||
|
@ -201,7 +168,6 @@ pub fn token_accounts() -> SubscribeRequest {
|
|||
"client".to_string(),
|
||||
SubscribeRequestFilterAccounts {
|
||||
account: vec![],
|
||||
// vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
|
||||
owner:
|
||||
spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
|
||||
filters: vec![],
|
||||
|
@ -209,14 +175,8 @@ pub fn token_accounts() -> SubscribeRequest {
|
|||
);
|
||||
|
||||
|
||||
let mut slots_subs = HashMap::new();
|
||||
slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
|
||||
filter_by_commitment: Some(true),
|
||||
});
|
||||
|
||||
|
||||
SubscribeRequest {
|
||||
slots: slots_subs,
|
||||
slots: HashMap::new(),
|
||||
accounts: accounts_subs,
|
||||
transactions: HashMap::new(),
|
||||
entry: Default::default(),
|
||||
|
|
Loading…
Reference in New Issue