New program to process `iftop` log output (#6668)
* New program to process iftop log output * fixes * fix shellcheck * address review comments * more review comments
This commit is contained in:
Pankaj Garg
GitHub
parent
d2c66c40c6
commit
2e30926ac3
|
|
@ -326,6 +326,11 @@ name = "byte-tools"
|
|||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "byte-unit"
|
||||
version = "3.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.3.2"
|
||||
|
|
@ -3710,6 +3715,19 @@ dependencies = [
|
|||
"tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-network-tool"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-logger 0.21.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-noop-program"
|
||||
version = "0.21.0"
|
||||
|
|
@ -5353,6 +5371,7 @@ dependencies = [
|
|||
"checksum bv 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6cd4ae9e585e783756cd14b0ea21863acdfbb6383664ac2f7c9ef8d180a14727"
|
||||
"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40"
|
||||
"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
|
||||
"checksum byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6894a79550807490d9f19a138a6da0f8830e70c83e83402dd23f16fd6c479056"
|
||||
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
|
||||
"checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
|
||||
"checksum bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b"
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ members = [
|
|||
"ledger-tool",
|
||||
"local_cluster",
|
||||
"logger",
|
||||
"log-analyzer",
|
||||
"merkle-tree",
|
||||
"measure",
|
||||
"metrics",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
[package]
|
||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||
edition = "2018"
|
||||
name = "solana-network-tool"
|
||||
description = "The solana cluster network analysis tool"
|
||||
version = "0.1.0"
|
||||
repository = "https://github.com/solana-labs/solana"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://solana.com/"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
byte-unit = "3.0.3"
|
||||
clap = { version = "2.33.0" }
|
||||
log = "0.4.8"
|
||||
semver = "0.9.0"
|
||||
serde = "1.0.102"
|
||||
serde_json = "1.0.41"
|
||||
solana-logger = { path = "../logger", version = "0.21.0" }
|
||||
|
||||
[[bin]]
|
||||
name = "solana-log-analyzer"
|
||||
path = "src/main.rs"
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
extern crate byte_unit;
|
||||
|
||||
use byte_unit::Byte;
|
||||
use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand};
|
||||
use serde::export::fmt::Error;
|
||||
use serde::export::Formatter;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{self};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Debug;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct LogLine {
|
||||
a: String,
|
||||
b: String,
|
||||
a_to_b: String,
|
||||
b_to_a: String,
|
||||
}
|
||||
|
||||
impl Debug for LogLine {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
|
||||
let a_to_b = Byte::from_str(&self.a_to_b).expect("Failed to read a_to_b bytes");
|
||||
let b_to_a = Byte::from_str(&self.b_to_a).expect("Failed to read b_to_a bytes");
|
||||
write!(
|
||||
f,
|
||||
"{{ \"{}\", \"{}\", {}, {} }}",
|
||||
self.a,
|
||||
self.b,
|
||||
a_to_b.get_bytes(),
|
||||
b_to_a.get_bytes()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
solana_logger::setup();
|
||||
|
||||
let matches = App::new(crate_name!())
|
||||
.about(crate_description!())
|
||||
.version(crate_version!())
|
||||
.arg(
|
||||
Arg::with_name("iftop")
|
||||
.short("i")
|
||||
.long("iftop")
|
||||
.value_name("iftop log file")
|
||||
.takes_value(true)
|
||||
.help("Location of the log file generated by iftop"),
|
||||
)
|
||||
.subcommand(
|
||||
SubCommand::with_name("map-IP")
|
||||
.about("Public IP Address")
|
||||
.arg(
|
||||
Arg::with_name("priv")
|
||||
.long("priv")
|
||||
.value_name("IP Address")
|
||||
.takes_value(true)
|
||||
.required(true)
|
||||
.help("The private IP address that should be mapped"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("pub")
|
||||
.long("pub")
|
||||
.value_name("IP Address")
|
||||
.takes_value(true)
|
||||
.required(true)
|
||||
.help("The public IP address"),
|
||||
),
|
||||
)
|
||||
.get_matches();
|
||||
|
||||
let map_address;
|
||||
let private_address;
|
||||
let public_address;
|
||||
match matches.subcommand() {
|
||||
("map-IP", Some(args_matches)) => {
|
||||
map_address = true;
|
||||
if let Some(addr) = args_matches.value_of("priv") {
|
||||
private_address = addr;
|
||||
} else {
|
||||
panic!("Private IP address must be provided");
|
||||
};
|
||||
if let Some(addr) = args_matches.value_of("pub") {
|
||||
public_address = addr;
|
||||
} else {
|
||||
panic!("Private IP address must be provided");
|
||||
};
|
||||
}
|
||||
_ => {
|
||||
map_address = false;
|
||||
private_address = "";
|
||||
public_address = "";
|
||||
}
|
||||
};
|
||||
|
||||
let log_path = PathBuf::from(value_t_or_exit!(matches, "iftop", String));
|
||||
let mut log = fs::read_to_string(&log_path).expect("Unable to read log file");
|
||||
log.insert(0, '[');
|
||||
let terminate_at = log.rfind('}').expect("Didn't find a terminating '}'") + 1;
|
||||
let _ = log.split_off(terminate_at);
|
||||
log.push(']');
|
||||
let json_log: Vec<LogLine> = serde_json::from_str(&log).expect("Failed to parse log as JSON");
|
||||
|
||||
let mut unique_latest_logs = HashMap::new();
|
||||
|
||||
json_log.into_iter().rev().for_each(|l| {
|
||||
if !l.a.is_empty() && !l.b.is_empty() && !l.a_to_b.is_empty() && !l.b_to_a.is_empty() {
|
||||
let key = (l.a.clone(), l.b.clone());
|
||||
unique_latest_logs.entry(key).or_insert(l);
|
||||
}
|
||||
});
|
||||
|
||||
println!(
|
||||
"{:#?}",
|
||||
unique_latest_logs
|
||||
.into_iter()
|
||||
.map(|(_, l)| {
|
||||
if map_address {
|
||||
LogLine {
|
||||
a: l.a.replace(private_address, public_address),
|
||||
b: l.b.replace(private_address, public_address),
|
||||
a_to_b: l.a_to_b,
|
||||
b_to_a: l.b_to_a,
|
||||
}
|
||||
} else {
|
||||
l
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
|
@ -52,6 +52,7 @@ BINS=(
|
|||
solana-install-init
|
||||
solana-keygen
|
||||
solana-ledger-tool
|
||||
solana-log-analyzer
|
||||
solana-archiver
|
||||
solana-validator
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Reports network bandwidth usage
|
||||
#
|
||||
set -e
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <iftop log file> <temp file for interediate data> [optional public IP address]"
|
||||
echo
|
||||
echo Processes iftop log file, and extracts latest bandwidth used by each connection
|
||||
echo
|
||||
echo
|
||||
}
|
||||
|
||||
if [ "$#" -lt 2 ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' < "$1" \
|
||||
| awk 'NR%2{printf "%s ",$0;next;}1' \
|
||||
| awk '{ print "{ \"a\": \""$1"\", " "\"b\": \""$3"\", \"a_to_b\": \""$2"\", \"b_to_a\": \""$4"\"}," }' > "$2"
|
||||
|
||||
if [ "$#" -ne 3 ]; then
|
||||
solana-log-analyzer -i "$2"
|
||||
else
|
||||
solana-log-analyzer -i "$2" map-IP --priv "$(hostname -i)" --pub "$3"
|
||||
fi
|
||||
|
||||
exit 1
|
||||
|
|
@ -8,9 +8,9 @@ set -e
|
|||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
sudo iftop -i "$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN { FS = ":" } ; {print $1}')" -nNbBP -t \
|
||||
| awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' \
|
||||
| awk 'NR%2{printf "%s ",$0;next;}1' \
|
||||
| awk '{ print $1, "<=>", $3, ":", $2, $4 }'
|
||||
sudo=
|
||||
if sudo true; then
|
||||
sudo="sudo -n"
|
||||
fi
|
||||
|
||||
exit 1
|
||||
$sudo iftop -i "$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN { FS = ":" } ; {print $1}')" -nNbBP -t
|
||||
|
|
|
|||
Loading…
Reference in New Issue