count data

Cargo.lock

@@ -126,6 +126,55 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.82"
@@ -695,6 +744,54 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "clap"
+version = "4.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+ "once_cell",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "bitflags 1.3.2",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.58",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+
 [[package]]
 name = "combine"
 version = "3.8.1"
@@ -1268,6 +1365,7 @@ dependencies = [
  "async-stream",
  "base64 0.21.7",
  "bincode",
+ "clap",
  "csv",
  "dashmap",
  "derive_more",
@@ -1363,6 +1461,12 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1585,6 +1689,17 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+dependencies = [
+ "hermit-abi 0.3.9",
+ "libc",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2216,7 +2331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1"
 dependencies = [
  "bytes",
- "heck",
+ "heck 0.5.0",
  "itertools 0.12.1",
  "log",
  "multimap",
@@ -3858,6 +3973,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "valuable"
 version = "0.1.0"

Cargo.toml

@@ -35,6 +35,7 @@ dashmap = "5.5.3"
 
 tonic-health = "0.10.2"
 regex = "1.10.4"
+clap = { version = "4.2", features = ["derive"] }
 
 [dev-dependencies]
 tracing-subscriber = "0.3.16"

examples/parse_timestamp_tagged_logs.rs

@@ -0,0 +1,156 @@
+use std::collections::HashMap;
+use std::fs::File;
+use std::io;
+use std::io::BufRead;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use regex::Regex;
+use clap::Parser;
+
+pub fn parse_log_entry_subscriber(log_entry: &str) -> (u64, u64) {
+
+    let re = Regex::new(r".*got account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
+    let caps = re.captures(log_entry).unwrap();
+
+    // let mut result = HashMap::new();
+    // result.insert("write_version".to_string(), caps["write_version"].to_string());
+    // result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
+    // result.insert("slot".to_string(), caps["slot"].to_string());
+
+    let write_version: u64 = caps["write_version"].parse().unwrap();
+    let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
+
+    (write_version, timestamp_us)
+}
+
+pub fn parse_log_entry_source(log_entry: &str) -> (u64, u64) {
+    let re = Regex::new(r".*account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
+    let caps = re.captures(log_entry).unwrap();
+
+    // let mut result = HashMap::new();
+    // result.insert("write_version".to_string(), caps["write_version"].to_string());
+    // result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
+    // result.insert("slot".to_string(), caps["slot"].to_string());
+
+    let write_version: u64 = caps["write_version"].parse().unwrap();
+    let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
+
+    (write_version, timestamp_us)
+}
+
+
+
+fn read_subscriber_log(log_file: PathBuf) -> HashMap<u64, u64> {
+    let mut map: HashMap<u64, u64> = HashMap::new();
+
+    let file = File::open(log_file).expect("file must exist");
+    let reader = io::BufReader::new(file);
+    for line in reader.lines().take(1000) {
+        let line = line.expect("must be parsable");
+        let (write_version, timestamp_us) = parse_log_entry_subscriber(&line);
+        // println!("{:?}", parsed);
+        map.insert(write_version, timestamp_us);
+    }
+
+    map
+}
+
+fn read_source_log(log_file: PathBuf) -> HashMap<u64, u64> {
+    let mut map: HashMap<u64, u64> = HashMap::new();
+
+    let file = File::open(log_file).expect("file must exist");
+    let reader = io::BufReader::new(file);
+    for line in reader.lines().take(1000) {
+        let line = line.expect("must be parsable");
+        let (write_version, timestamp_us) = parse_log_entry_source(&line);
+        // println!("{:?}", parsed);
+        map.insert(write_version, timestamp_us);
+    }
+
+    map
+}
+
+// cat macbook.log |cut -b 111- | tr -d 'a-z_=' > macbook.log.csv
+// cat solana-validator-macbook.log | cut -b 96- | tr -d 'a-z_='
+fn read_from_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
+    csv::ReaderBuilder::new()
+        .delimiter(b';')
+        .has_headers(false)
+        .from_path(csv_file)
+        .unwrap()
+        .into_deserialize()
+        .map(|record| {
+            let record: Vec<String> = record.unwrap();
+            let write_version = record[0].parse::<u64>().unwrap();
+            let timestamp_us = record[1].parse::<u64>().unwrap();
+            (write_version, timestamp_us)
+        })
+        .collect::<HashMap<u64, u64>>()
+}
+
+fn read_subscriber_log_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
+    csv::ReaderBuilder::new()
+        .delimiter(b';')
+        .has_headers(false)
+        .from_path(csv_file)
+        .unwrap()
+        .into_deserialize()
+        .map(|record| {
+            let record: Vec<String> = record.unwrap();
+            let write_version = record[0].parse::<u64>().unwrap();
+            let timestamp_us = record[1].parse::<u64>().unwrap();
+            (write_version, timestamp_us)
+        })
+        .collect::<HashMap<u64, u64>>()
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+pub struct Args {
+    #[arg(long)]
+    pub csv_file_source: String,
+    #[arg(long)]
+    pub csv_file_subscriber: String,
+}
+
+
+pub fn main() {
+    let Args { csv_file_source, csv_file_subscriber } = Args::parse();
+
+    println!("Reading source log ...");
+    let source_timestamps = read_from_csv(PathBuf::from(csv_file_source));
+
+    println!("Reading subscriber log ...");
+    let subscriber_timestamps = read_from_csv(PathBuf::from(csv_file_subscriber));
+
+    for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
+        if let Some(source_timestamp) = source_timestamps.get(&write_version) {
+            let diff = (timestamp_us as i128) - (*source_timestamp as i128);
+            println!("write_version: {}, subscriber: {}, source: {}, diff: {:.1}ms", write_version, timestamp_us, source_timestamp, diff as f64 / 1000.0);
+        }
+
+    }
+
+}
+
+pub fn main__() {
+    println!("Reading subscriber log ...");
+    let subscriber_timestamps = read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
+
+    println!("Reading source log ...");
+    let source_timestamps = read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
+
+    println!("Comparing ...");
+
+    for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
+
+        // println!("write_version: {}, subscriber: {}", write_version, timestamp_us);
+        if let Some(source_timestamp) = source_timestamps.get(&write_version) {
+            let diff = (timestamp_us as i128) - (*source_timestamp as i128);
+            println!("write_version: {}, subscriber: {}, source: {}, diff: {}", write_version, timestamp_us, source_timestamp, diff);
+        }
+
+    }
+
+
+}

examples/stream_token_accounts.rs

@@ -19,11 +19,11 @@ use tokio::time::{sleep, Duration};
 use tracing::field::debug;
 use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
-use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
 
 
-const ENABLE_TIMESTAMP_TAGGING: bool = true;
+const ENABLE_TIMESTAMP_TAGGING: bool = false;
 
 #[tokio::main]
 pub async fn main() {
@@ -63,14 +63,26 @@ pub async fn main() {
 
 
     tokio::spawn(async move {
+        let mut bytes_per_slot: HashMap<Slot, u64> = HashMap::new();
+        let mut updates_per_slot: HashMap<Slot, u64> = HashMap::new();
+
+        let mut changing_slot = 0;
+        let mut current_slot = 0;
+
+
         let mut green_stream = pin!(green_stream);
         while let Some(message) = green_stream.next().await {
             match message {
                 Message::GeyserSubscribeUpdate(subscriber_update) => {
                     match subscriber_update.update_oneof {
+                        Some(UpdateOneof::Slot(update)) => {
+                            current_slot = update.slot;
+                        }
                         Some(UpdateOneof::Account(update)) => {
+                            let slot = update.slot as Slot;
                             let account = update.account.unwrap();
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
+                            let size = account.data.len() as u64;
                             trace!("got account update (green)!!! {} - {:?} - {} bytes",
                                 update.slot, account_pk, account.data.len());
 
@@ -79,6 +91,7 @@ pub async fn main() {
                                 info!("got account update: write_version={};timestamp_us={};slot={}", account.write_version, since_the_epoch.as_micros(), update.slot);
                             }
 
+
                             match parse_token(&account.data, Some(6)) {
                                 Ok(TokenAccountType::Account(account_ui)) => {
                                     // UiTokenAccount {
@@ -110,10 +123,30 @@ pub async fn main() {
                                     if account_ui.owner.starts_with("66fEFnKy") {
                                         info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
                                     }
+                                    // if pubkey.starts_with(b"JUP") {
+                                    //     info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
+                                    // }
+
                                     token_account_by_ownermint.entry(owner)
                                         .or_insert_with(DashMap::new)
                                         .insert(mint, account_ui);
 
+                                    bytes_per_slot.entry(slot)
+                                        .and_modify(|total| *total += size).or_insert(size);
+
+                                    updates_per_slot.entry(slot)
+                                        .and_modify(|total| *total += 1).or_insert(1);
+
+                                    info!("delta: {}", (slot as i64) - (current_slot as i64));
+
+                                    if slot != changing_slot && changing_slot != 0 {
+                                        let total_bytes = bytes_per_slot.get(&changing_slot).unwrap();
+                                        let updates_count = updates_per_slot.get(&changing_slot).unwrap();
+                                        // info!("Slot {} - Total bytes: {} in {} updates", slot, total_bytes, updates_count);
+                                    }
+                                    changing_slot = slot;
+
+
                                 }
                                 Ok(TokenAccountType::Mint(mint)) => {
                                     // not interesting
@@ -168,6 +201,7 @@ pub fn token_accounts() -> SubscribeRequest {
         "client".to_string(),
         SubscribeRequestFilterAccounts {
             account: vec![],
+            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
             owner:
                 spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
@@ -175,8 +209,14 @@ pub fn token_accounts() -> SubscribeRequest {
     );
 
 
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
+        filter_by_commitment: Some(true),
+    });
+
+
     SubscribeRequest {
-        slots: HashMap::new(),
+        slots: slots_subs,
         accounts: accounts_subs,
         transactions: HashMap::new(),
         entry: Default::default(),