Use iftop to collect network bandwidth usage (#6560)

* Use iftop to collect network bandwidth usage

* fix shellcheck

* more shellchecks

* review comments

net/gce.sh

@@ -714,6 +714,7 @@ $(
     add-testnet-solana-user-authorized_keys.sh \
     install-certbot.sh \
     install-earlyoom.sh \
+    install-iftop.sh \
     install-libssl-compatability.sh \
     install-nodejs.sh \
     install-redis.sh \

net/net.sh

@@ -810,7 +810,7 @@ stopNode() {
       PS4=\"$PS4\"
       set -x
       ! tmux list-sessions || tmux kill-session
-      for pid in solana/{blockexplorer,net-stats,fd-monitor,oom-monitor}.pid; do
+      for pid in solana/*.pid; do
         pgid=\$(ps opgid= \$(cat \$pid) | tr -d '[:space:]')
         if [[ -n \$pgid ]]; then
           sudo kill -- -\$pgid

net/remote/remote-node.sh

@@ -138,6 +138,8 @@ cat >> ~/solana/on-reboot <<EOF
   echo \$! > fd-monitor.pid
   scripts/net-stats.sh  > net-stats.log 2>&1 &
   echo \$! > net-stats.pid
+  scripts/iftop.sh  > iftop.log 2>&1 &
+  echo \$! > iftop.pid
 
   if ${GPU_CUDA_OK} && [[ -e /dev/nvidia0 ]]; then
     echo Selecting solana-validator-cuda

net/scripts/install-iftop.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+# Rsync setup
+#
+set -ex
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+apt-get --assume-yes install iftop

scripts/iftop.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Reports network bandwidth usage
+#
+set -e
+
+[[ $(uname) == Linux ]] || exit 0
+
+cd "$(dirname "$0")"
+
+sudo iftop -i "$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN { FS = ":" } ; {print $1}')" -nNbBP  -t \
+	| awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' \
+	| awk 'NR%2{printf "%s ",$0;next;}1' \
+	| awk '{ print $1, "<=>", $3, ":",  $2, $4 }'
+
+exit 1