New program to process `iftop` log output (#6668)

* New program to process iftop log output

* fixes

* fix shellcheck

* address review comments

* more review comments

Cargo.lock

@@ -326,6 +326,11 @@ name = "byte-tools"
 version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "byte-unit"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "byteorder"
 version = "1.3.2"
@@ -3710,6 +3715,19 @@ dependencies = [
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "solana-network-tool"
+version = "0.1.0"
+dependencies = [
+ "byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-logger 0.21.0",
+]
+
 [[package]]
 name = "solana-noop-program"
 version = "0.21.0"
@@ -5353,6 +5371,7 @@ dependencies = [
 "checksum bv 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6cd4ae9e585e783756cd14b0ea21863acdfbb6383664ac2f7c9ef8d180a14727"
 "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40"
 "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
+"checksum byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6894a79550807490d9f19a138a6da0f8830e70c83e83402dd23f16fd6c479056"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
 "checksum bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b"

Cargo.toml

@@ -18,6 +18,7 @@ members = [
     "ledger-tool",
     "local_cluster",
     "logger",
+    "log-analyzer",
     "merkle-tree",
     "measure",
     "metrics",

log-analyzer/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+edition = "2018"
+name = "solana-network-tool"
+description = "The solana cluster network analysis tool"
+version = "0.1.0"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = false
+
+[dependencies]
+byte-unit = "3.0.3"
+clap = { version = "2.33.0" }
+log = "0.4.8"
+semver = "0.9.0"
+serde = "1.0.102"
+serde_json = "1.0.41"
+solana-logger = { path = "../logger", version = "0.21.0" }
+
+[[bin]]
+name = "solana-log-analyzer"
+path = "src/main.rs"

log-analyzer/src/main.rs

@@ -0,0 +1,132 @@
+extern crate byte_unit;
+
+use byte_unit::Byte;
+use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand};
+use serde::export::fmt::Error;
+use serde::export::Formatter;
+use serde::{Deserialize, Serialize};
+use serde_json::{self};
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fs;
+use std::path::PathBuf;
+
+#[derive(Serialize, Deserialize)]
+struct LogLine {
+    a: String,
+    b: String,
+    a_to_b: String,
+    b_to_a: String,
+}
+
+impl Debug for LogLine {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
+        let a_to_b = Byte::from_str(&self.a_to_b).expect("Failed to read a_to_b bytes");
+        let b_to_a = Byte::from_str(&self.b_to_a).expect("Failed to read b_to_a bytes");
+        write!(
+            f,
+            "{{ \"{}\", \"{}\", {}, {} }}",
+            self.a,
+            self.b,
+            a_to_b.get_bytes(),
+            b_to_a.get_bytes()
+        )
+    }
+}
+
+fn main() {
+    solana_logger::setup();
+
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(crate_version!())
+        .arg(
+            Arg::with_name("iftop")
+                .short("i")
+                .long("iftop")
+                .value_name("iftop log file")
+                .takes_value(true)
+                .help("Location of the log file generated by iftop"),
+        )
+        .subcommand(
+            SubCommand::with_name("map-IP")
+                .about("Public IP Address")
+                .arg(
+                    Arg::with_name("priv")
+                        .long("priv")
+                        .value_name("IP Address")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The private IP address that should be mapped"),
+                )
+                .arg(
+                    Arg::with_name("pub")
+                        .long("pub")
+                        .value_name("IP Address")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The public IP address"),
+                ),
+        )
+        .get_matches();
+
+    let map_address;
+    let private_address;
+    let public_address;
+    match matches.subcommand() {
+        ("map-IP", Some(args_matches)) => {
+            map_address = true;
+            if let Some(addr) = args_matches.value_of("priv") {
+                private_address = addr;
+            } else {
+                panic!("Private IP address must be provided");
+            };
+            if let Some(addr) = args_matches.value_of("pub") {
+                public_address = addr;
+            } else {
+                panic!("Private IP address must be provided");
+            };
+        }
+        _ => {
+            map_address = false;
+            private_address = "";
+            public_address = "";
+        }
+    };
+
+    let log_path = PathBuf::from(value_t_or_exit!(matches, "iftop", String));
+    let mut log = fs::read_to_string(&log_path).expect("Unable to read log file");
+    log.insert(0, '[');
+    let terminate_at = log.rfind('}').expect("Didn't find a terminating '}'") + 1;
+    let _ = log.split_off(terminate_at);
+    log.push(']');
+    let json_log: Vec<LogLine> = serde_json::from_str(&log).expect("Failed to parse log as JSON");
+
+    let mut unique_latest_logs = HashMap::new();
+
+    json_log.into_iter().rev().for_each(|l| {
+        if !l.a.is_empty() && !l.b.is_empty() && !l.a_to_b.is_empty() && !l.b_to_a.is_empty() {
+            let key = (l.a.clone(), l.b.clone());
+            unique_latest_logs.entry(key).or_insert(l);
+        }
+    });
+
+    println!(
+        "{:#?}",
+        unique_latest_logs
+            .into_iter()
+            .map(|(_, l)| {
+                if map_address {
+                    LogLine {
+                        a: l.a.replace(private_address, public_address),
+                        b: l.b.replace(private_address, public_address),
+                        a_to_b: l.a_to_b,
+                        b_to_a: l.b_to_a,
+                    }
+                } else {
+                    l
+                }
+            })
+            .collect::<Vec<_>>()
+    );
+}

scripts/cargo-install-all.sh

@@ -52,6 +52,7 @@ BINS=(
   solana-install-init
   solana-keygen
   solana-ledger-tool
+  solana-log-analyzer
   solana-archiver
   solana-validator
 )

scripts/iftop-postprocess.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Reports network bandwidth usage
+#
+set -e
+
+usage() {
+  echo "Usage: $0 <iftop log file> <temp file for interediate data> [optional public IP address]"
+  echo
+  echo Processes iftop log file, and extracts latest bandwidth used by each connection
+  echo
+  echo
+}
+
+if [ "$#" -lt 2 ]; then
+  usage
+  exit 1
+fi
+
+cd "$(dirname "$0")"
+
+awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' < "$1" \
+  | awk 'NR%2{printf "%s ",$0;next;}1' \
+  | awk '{ print "{ \"a\": \""$1"\", " "\"b\": \""$3"\", \"a_to_b\": \""$2"\", \"b_to_a\": \""$4"\"}," }' > "$2"
+
+if [ "$#" -ne 3 ]; then
+  solana-log-analyzer -i "$2"
+else
+  solana-log-analyzer -i "$2" map-IP --priv "$(hostname -i)" --pub "$3"
+fi
+
+exit 1

scripts/iftop.sh

@@ -8,9 +8,9 @@ set -e
 
 cd "$(dirname "$0")"
 
-sudo iftop -i "$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN { FS = ":" } ; {print $1}')" -nNbBP  -t \
+sudo=
-	| awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' \
+if sudo true; then
-	| awk 'NR%2{printf "%s ",$0;next;}1' \
+sudo="sudo -n"
-	| awk '{ print $1, "<=>", $3, ":",  $2, $4 }'
+fi
 
-exit 1
+$sudo iftop -i "$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN { FS = ":" } ; {print $1}')" -nNbBP  -t