provide logs as args
This commit is contained in:
GroovieGermanikus
parent
b64f7002db
commit
1b5d4f458d
|
|
@ -126,6 +126,55 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is-terminal",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.82"
|
||||
|
|
@ -695,6 +744,54 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"bitflags 1.3.2",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.58",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "3.8.1"
|
||||
|
|
@ -1268,6 +1365,7 @@ dependencies = [
|
|||
"async-stream",
|
||||
"base64 0.21.7",
|
||||
"bincode",
|
||||
"clap",
|
||||
"csv",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
|
|
@ -1363,6 +1461,12 @@ version = "0.14.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
|
|
@ -1585,6 +1689,17 @@ version = "2.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
|
||||
dependencies = [
|
||||
"hermit-abi 0.3.9",
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
|
|
@ -2216,7 +2331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"itertools 0.12.1",
|
||||
"log",
|
||||
"multimap",
|
||||
|
|
@ -3858,6 +3973,12 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ dashmap = "5.5.3"
|
|||
|
||||
tonic-health = "0.10.2"
|
||||
regex = "1.10.4"
|
||||
clap = { version = "4.2", features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = "0.3.16"
|
||||
|
|
|
|||
|
|
@ -3,64 +3,154 @@ use std::fs::File;
|
|||
use std::io;
|
||||
use std::io::BufRead;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use regex::Regex;
|
||||
use clap::Parser;
|
||||
|
||||
pub fn parse_log_entry_subscriber(log_entry: &str) -> (u64, u64) {
|
||||
|
||||
pub fn parse_log_entry_subscriber(log_entry: &str) -> HashMap<String, String> {
|
||||
let re = Regex::new(r".*got account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
|
||||
let caps = re.captures(log_entry).unwrap();
|
||||
|
||||
let mut result = HashMap::new();
|
||||
result.insert("write_version".to_string(), caps["write_version"].to_string());
|
||||
result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
|
||||
result.insert("slot".to_string(), caps["slot"].to_string());
|
||||
// let mut result = HashMap::new();
|
||||
// result.insert("write_version".to_string(), caps["write_version"].to_string());
|
||||
// result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
|
||||
// result.insert("slot".to_string(), caps["slot"].to_string());
|
||||
|
||||
result
|
||||
let write_version: u64 = caps["write_version"].parse().unwrap();
|
||||
let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
|
||||
|
||||
(write_version, timestamp_us)
|
||||
}
|
||||
|
||||
pub fn parse_log_entry_source(log_entry: &str) -> HashMap<String, String> {
|
||||
pub fn parse_log_entry_source(log_entry: &str) -> (u64, u64) {
|
||||
let re = Regex::new(r".*account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
|
||||
let caps = re.captures(log_entry).unwrap();
|
||||
|
||||
let mut result = HashMap::new();
|
||||
result.insert("write_version".to_string(), caps["write_version"].to_string());
|
||||
result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
|
||||
result.insert("slot".to_string(), caps["slot"].to_string());
|
||||
// let mut result = HashMap::new();
|
||||
// result.insert("write_version".to_string(), caps["write_version"].to_string());
|
||||
// result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
|
||||
// result.insert("slot".to_string(), caps["slot"].to_string());
|
||||
|
||||
result
|
||||
let write_version: u64 = caps["write_version"].parse().unwrap();
|
||||
let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
|
||||
|
||||
(write_version, timestamp_us)
|
||||
}
|
||||
|
||||
|
||||
|
||||
fn read_subscriber_log(log_file: PathBuf) {
|
||||
fn read_subscriber_log(log_file: PathBuf) -> HashMap<u64, u64> {
|
||||
let mut map: HashMap<u64, u64> = HashMap::new();
|
||||
|
||||
let file = File::open(log_file).expect("file must exist");
|
||||
let reader = io::BufReader::new(file);
|
||||
for line in reader.lines().take(10) {
|
||||
for line in reader.lines().take(1000) {
|
||||
let line = line.expect("must be parsable");
|
||||
let parsed = parse_log_entry_subscriber(&line);
|
||||
println!("{:?}", parsed);
|
||||
let (write_version, timestamp_us) = parse_log_entry_subscriber(&line);
|
||||
// println!("{:?}", parsed);
|
||||
map.insert(write_version, timestamp_us);
|
||||
}
|
||||
|
||||
map
|
||||
}
|
||||
|
||||
fn read_source_log(log_file: PathBuf) {
|
||||
fn read_source_log(log_file: PathBuf) -> HashMap<u64, u64> {
|
||||
let mut map: HashMap<u64, u64> = HashMap::new();
|
||||
|
||||
let file = File::open(log_file).expect("file must exist");
|
||||
let reader = io::BufReader::new(file);
|
||||
for line in reader.lines().take(10) {
|
||||
for line in reader.lines().take(1000) {
|
||||
let line = line.expect("must be parsable");
|
||||
let parsed = parse_log_entry_source(&line);
|
||||
println!("{:?}", parsed);
|
||||
let (write_version, timestamp_us) = parse_log_entry_source(&line);
|
||||
// println!("{:?}", parsed);
|
||||
map.insert(write_version, timestamp_us);
|
||||
}
|
||||
|
||||
map
|
||||
}
|
||||
|
||||
// cat macbook.log |cut -b 111- | tr -d 'a-z_=' > macbook.log.csv
|
||||
// cat solana-validator-macbook.log | cut -b 96- | tr -d 'a-z_='
|
||||
fn read_from_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
|
||||
csv::ReaderBuilder::new()
|
||||
.delimiter(b';')
|
||||
.has_headers(false)
|
||||
.from_path(csv_file)
|
||||
.unwrap()
|
||||
.into_deserialize()
|
||||
.map(|record| {
|
||||
let record: Vec<String> = record.unwrap();
|
||||
let write_version = record[0].parse::<u64>().unwrap();
|
||||
let timestamp_us = record[1].parse::<u64>().unwrap();
|
||||
(write_version, timestamp_us)
|
||||
})
|
||||
.collect::<HashMap<u64, u64>>()
|
||||
}
|
||||
|
||||
fn read_subscriber_log_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
|
||||
csv::ReaderBuilder::new()
|
||||
.delimiter(b';')
|
||||
.has_headers(false)
|
||||
.from_path(csv_file)
|
||||
.unwrap()
|
||||
.into_deserialize()
|
||||
.map(|record| {
|
||||
let record: Vec<String> = record.unwrap();
|
||||
let write_version = record[0].parse::<u64>().unwrap();
|
||||
let timestamp_us = record[1].parse::<u64>().unwrap();
|
||||
(write_version, timestamp_us)
|
||||
})
|
||||
.collect::<HashMap<u64, u64>>()
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
pub struct Args {
|
||||
#[arg(long)]
|
||||
pub csv_file_source: String,
|
||||
#[arg(long)]
|
||||
pub csv_file_subscriber: String,
|
||||
}
|
||||
|
||||
|
||||
pub fn main() {
|
||||
let Args { csv_file_source, csv_file_subscriber } = Args::parse();
|
||||
|
||||
println!("Reading source log ...");
|
||||
let source_timestamps = read_from_csv(PathBuf::from(csv_file_source));
|
||||
|
||||
read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
|
||||
println!("Reading subscriber log ...");
|
||||
let subscriber_timestamps = read_from_csv(PathBuf::from(csv_file_subscriber));
|
||||
|
||||
read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
|
||||
for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
|
||||
if let Some(source_timestamp) = source_timestamps.get(&write_version) {
|
||||
let diff = (timestamp_us as i128) - (*source_timestamp as i128);
|
||||
println!("write_version: {}, subscriber: {}, source: {}, diff: {:.1}ms", write_version, timestamp_us, source_timestamp, diff as f64 / 1000.0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pub fn main__() {
|
||||
println!("Reading subscriber log ...");
|
||||
let subscriber_timestamps = read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
|
||||
|
||||
println!("Reading source log ...");
|
||||
let source_timestamps = read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
|
||||
|
||||
println!("Comparing ...");
|
||||
|
||||
for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
|
||||
|
||||
// println!("write_version: {}, subscriber: {}", write_version, timestamp_us);
|
||||
if let Some(source_timestamp) = source_timestamps.get(&write_version) {
|
||||
let diff = (timestamp_us as i128) - (*source_timestamp as i128);
|
||||
println!("write_version: {}, subscriber: {}, source: {}, diff: {}", write_version, timestamp_us, source_timestamp, diff);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue