Compare iftop logs using log-analyzer (#6684)
* Compare iftop logs using log-analyzer * fixes * fix clippy errors
This commit is contained in:
parent
385b4ce959
commit
bae83ba2b6
|
@ -3724,6 +3724,7 @@ dependencies = [
|
||||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"solana-logger 0.21.0",
|
"solana-logger 0.21.0",
|
||||||
]
|
]
|
||||||
|
|
|
@ -11,10 +11,11 @@ publish = false
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
byte-unit = "3.0.3"
|
byte-unit = "3.0.3"
|
||||||
clap = { version = "2.33.0" }
|
clap = "2.33.0"
|
||||||
log = "0.4.8"
|
log = "0.4.8"
|
||||||
semver = "0.9.0"
|
semver = "0.9.0"
|
||||||
serde = "1.0.102"
|
serde = "1.0.102"
|
||||||
|
serde_derive = "1.0.102"
|
||||||
serde_json = "1.0.41"
|
serde_json = "1.0.41"
|
||||||
solana-logger = { path = "../logger", version = "0.21.0" }
|
solana-logger = { path = "../logger", version = "0.21.0" }
|
||||||
|
|
||||||
|
|
|
@ -1,17 +1,17 @@
|
||||||
extern crate byte_unit;
|
extern crate byte_unit;
|
||||||
|
|
||||||
use byte_unit::Byte;
|
use byte_unit::Byte;
|
||||||
use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand};
|
use clap::{
|
||||||
use serde::export::fmt::Error;
|
crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, ArgMatches, SubCommand,
|
||||||
use serde::export::Formatter;
|
};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{self};
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fmt::Debug;
|
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use std::ops::Sub;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
#[derive(Deserialize, Serialize, Debug)]
|
||||||
struct LogLine {
|
struct LogLine {
|
||||||
a: String,
|
a: String,
|
||||||
b: String,
|
b: String,
|
||||||
|
@ -19,57 +19,70 @@ struct LogLine {
|
||||||
b_to_a: String,
|
b_to_a: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Debug for LogLine {
|
impl Default for LogLine {
|
||||||
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
|
fn default() -> Self {
|
||||||
let a_to_b = Byte::from_str(&self.a_to_b).expect("Failed to read a_to_b bytes");
|
Self {
|
||||||
let b_to_a = Byte::from_str(&self.b_to_a).expect("Failed to read b_to_a bytes");
|
a: String::default(),
|
||||||
write!(
|
b: String::default(),
|
||||||
f,
|
a_to_b: "0B".to_string(),
|
||||||
"{{ \"{}\", \"{}\", {}, {} }}",
|
b_to_a: "0B".to_string(),
|
||||||
self.a,
|
}
|
||||||
self.b,
|
}
|
||||||
a_to_b.get_bytes(),
|
}
|
||||||
b_to_a.get_bytes()
|
|
||||||
|
impl LogLine {
|
||||||
|
fn output(a: &str, b: &str, v1: u128, v2: u128) -> String {
|
||||||
|
format!(
|
||||||
|
"Lost {} bytes ({} - {}, {}%) from {} to {}",
|
||||||
|
v1 - v2,
|
||||||
|
v1,
|
||||||
|
v2,
|
||||||
|
((v1 - v2) * 100 / v1),
|
||||||
|
a,
|
||||||
|
b
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
impl Sub for &LogLine {
|
||||||
solana_logger::setup();
|
type Output = String;
|
||||||
|
|
||||||
let matches = App::new(crate_name!())
|
fn sub(self, rhs: Self) -> Self::Output {
|
||||||
.about(crate_description!())
|
let a_to_b = Byte::from_str(&self.a_to_b)
|
||||||
.version(crate_version!())
|
.expect("Failed to read a_to_b bytes")
|
||||||
.arg(
|
.get_bytes();
|
||||||
Arg::with_name("iftop")
|
let b_to_a = Byte::from_str(&self.b_to_a)
|
||||||
.short("i")
|
.expect("Failed to read b_to_a bytes")
|
||||||
.long("iftop")
|
.get_bytes();
|
||||||
.value_name("iftop log file")
|
let rhs_a_to_b = Byte::from_str(&rhs.a_to_b)
|
||||||
.takes_value(true)
|
.expect("Failed to read a_to_b bytes")
|
||||||
.help("Location of the log file generated by iftop"),
|
.get_bytes();
|
||||||
)
|
let rhs_b_to_a = Byte::from_str(&rhs.b_to_a)
|
||||||
.subcommand(
|
.expect("Failed to read b_to_a bytes")
|
||||||
SubCommand::with_name("map-IP")
|
.get_bytes();
|
||||||
.about("Public IP Address")
|
let mut out1 = if a_to_b > rhs_b_to_a {
|
||||||
.arg(
|
LogLine::output(&self.a, &self.b, a_to_b, rhs_b_to_a)
|
||||||
Arg::with_name("priv")
|
} else if a_to_b < rhs_b_to_a {
|
||||||
.long("priv")
|
LogLine::output(&self.b, &self.a, rhs_b_to_a, a_to_b)
|
||||||
.value_name("IP Address")
|
} else {
|
||||||
.takes_value(true)
|
String::default()
|
||||||
.required(true)
|
};
|
||||||
.help("The private IP address that should be mapped"),
|
let out2 = if rhs_a_to_b > b_to_a {
|
||||||
)
|
LogLine::output(&self.a, &self.b, rhs_a_to_b, b_to_a)
|
||||||
.arg(
|
} else if rhs_a_to_b < b_to_a {
|
||||||
Arg::with_name("pub")
|
LogLine::output(&self.b, &self.a, b_to_a, rhs_a_to_b)
|
||||||
.long("pub")
|
} else {
|
||||||
.value_name("IP Address")
|
String::default()
|
||||||
.takes_value(true)
|
};
|
||||||
.required(true)
|
if !out1.is_empty() && !out2.is_empty() {
|
||||||
.help("The public IP address"),
|
out1.push('\n');
|
||||||
),
|
}
|
||||||
)
|
out1.push_str(&out2);
|
||||||
.get_matches();
|
out1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_iftop_logs(matches: &ArgMatches) {
|
||||||
let map_address;
|
let map_address;
|
||||||
let private_address;
|
let private_address;
|
||||||
let public_address;
|
let public_address;
|
||||||
|
@ -94,7 +107,7 @@ fn main() {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let log_path = PathBuf::from(value_t_or_exit!(matches, "iftop", String));
|
let log_path = PathBuf::from(value_t_or_exit!(matches, "file", String));
|
||||||
let mut log = fs::read_to_string(&log_path).expect("Unable to read log file");
|
let mut log = fs::read_to_string(&log_path).expect("Unable to read log file");
|
||||||
log.insert(0, '[');
|
log.insert(0, '[');
|
||||||
let terminate_at = log.rfind('}').expect("Didn't find a terminating '}'") + 1;
|
let terminate_at = log.rfind('}').expect("Didn't find a terminating '}'") + 1;
|
||||||
|
@ -110,23 +123,132 @@ fn main() {
|
||||||
unique_latest_logs.entry(key).or_insert(l);
|
unique_latest_logs.entry(key).or_insert(l);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
let output: Vec<LogLine> = unique_latest_logs
|
||||||
println!(
|
.into_iter()
|
||||||
"{:#?}",
|
.map(|(_, l)| {
|
||||||
unique_latest_logs
|
if map_address {
|
||||||
.into_iter()
|
LogLine {
|
||||||
.map(|(_, l)| {
|
a: l.a.replace(private_address, public_address),
|
||||||
if map_address {
|
b: l.b.replace(private_address, public_address),
|
||||||
LogLine {
|
a_to_b: l.a_to_b,
|
||||||
a: l.a.replace(private_address, public_address),
|
b_to_a: l.b_to_a,
|
||||||
b: l.b.replace(private_address, public_address),
|
|
||||||
a_to_b: l.a_to_b,
|
|
||||||
b_to_a: l.b_to_a,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
l
|
|
||||||
}
|
}
|
||||||
})
|
} else {
|
||||||
.collect::<Vec<_>>()
|
l
|
||||||
);
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
println!("{}", serde_json::to_string(&output).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn analyze_logs(matches: &ArgMatches) {
|
||||||
|
let dir_path = PathBuf::from(value_t_or_exit!(matches, "folder", String));
|
||||||
|
if !dir_path.is_dir() {
|
||||||
|
panic!("Need a folder that contains all log files");
|
||||||
|
}
|
||||||
|
let list_all_diffs = matches.is_present("all");
|
||||||
|
let files = fs::read_dir(dir_path).expect("Failed to read log folder");
|
||||||
|
let logs: Vec<_> = files
|
||||||
|
.flat_map(|f| {
|
||||||
|
if let Ok(f) = f {
|
||||||
|
let log_str = fs::read_to_string(&f.path()).expect("Unable to read log file");
|
||||||
|
let log: Vec<LogLine> =
|
||||||
|
serde_json::from_str(log_str.as_str()).expect("Failed to deserialize log");
|
||||||
|
log
|
||||||
|
} else {
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut logs_hash = HashMap::new();
|
||||||
|
logs.iter().for_each(|l| {
|
||||||
|
let key = (l.a.clone(), l.b.clone());
|
||||||
|
logs_hash.entry(key).or_insert(l);
|
||||||
|
});
|
||||||
|
|
||||||
|
logs.iter().for_each(|l| {
|
||||||
|
let diff = logs_hash
|
||||||
|
.remove(&(l.a.clone(), l.b.clone()))
|
||||||
|
.map(|v1| {
|
||||||
|
logs_hash.remove(&(l.b.clone(), l.a.clone())).map_or(
|
||||||
|
if list_all_diffs {
|
||||||
|
v1 - &LogLine::default()
|
||||||
|
} else {
|
||||||
|
String::default()
|
||||||
|
},
|
||||||
|
|v2| v1 - v2,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
if !diff.is_empty() {
|
||||||
|
println!("{}", diff);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
solana_logger::setup();
|
||||||
|
|
||||||
|
let matches = App::new(crate_name!())
|
||||||
|
.about(crate_description!())
|
||||||
|
.version(crate_version!())
|
||||||
|
.subcommand(
|
||||||
|
SubCommand::with_name("iftop")
|
||||||
|
.about("Process iftop log file")
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("file")
|
||||||
|
.short("f")
|
||||||
|
.long("file")
|
||||||
|
.value_name("iftop log file")
|
||||||
|
.takes_value(true)
|
||||||
|
.help("Location of the log file generated by iftop"),
|
||||||
|
)
|
||||||
|
.subcommand(
|
||||||
|
SubCommand::with_name("map-IP")
|
||||||
|
.about("Public IP Address")
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("priv")
|
||||||
|
.long("priv")
|
||||||
|
.value_name("IP Address")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("The private IP address that should be mapped"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("pub")
|
||||||
|
.long("pub")
|
||||||
|
.value_name("IP Address")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("The public IP address"),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.subcommand(
|
||||||
|
SubCommand::with_name("analyze")
|
||||||
|
.about("Compare processed network log files")
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("folder")
|
||||||
|
.short("f")
|
||||||
|
.long("folder")
|
||||||
|
.value_name("DIR")
|
||||||
|
.takes_value(true)
|
||||||
|
.help("Location of processed log files"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("all")
|
||||||
|
.short("a")
|
||||||
|
.long("all")
|
||||||
|
.takes_value(false)
|
||||||
|
.help("List all differences"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.get_matches();
|
||||||
|
|
||||||
|
match matches.subcommand() {
|
||||||
|
("iftop", Some(args_matches)) => process_iftop_logs(args_matches),
|
||||||
|
("analyze", Some(args_matches)) => analyze_logs(args_matches),
|
||||||
|
_ => {}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,9 +24,9 @@ awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' <
|
||||||
| awk '{ print "{ \"a\": \""$1"\", " "\"b\": \""$3"\", \"a_to_b\": \""$2"\", \"b_to_a\": \""$4"\"}," }' > "$2"
|
| awk '{ print "{ \"a\": \""$1"\", " "\"b\": \""$3"\", \"a_to_b\": \""$2"\", \"b_to_a\": \""$4"\"}," }' > "$2"
|
||||||
|
|
||||||
if [ "$#" -ne 3 ]; then
|
if [ "$#" -ne 3 ]; then
|
||||||
solana-log-analyzer -i "$2"
|
solana-log-analyzer iftop -f "$2"
|
||||||
else
|
else
|
||||||
solana-log-analyzer -i "$2" map-IP --priv "$(hostname -i)" --pub "$3"
|
solana-log-analyzer iftop -f "$2" map-IP --priv "$(hostname -i)" --pub "$3"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exit 1
|
exit 1
|
||||||
|
|
Loading…
Reference in New Issue