Report UDP network statistics (#1176)

* Report UDP network statistics

Fixes #1093

* Address review comments

* Address additional review comments

* Fix shellcheck errors

net/net.sh

@@ -311,7 +311,7 @@ stopNode() {
         sudo snap remove solana
       fi
       ! tmux list-sessions || tmux kill-session
-      for pattern in solana- remote- oom-monitor; do
+      for pattern in solana- remote- oom-monitor net-stats; do
         pkill -9 \$pattern
       done
     "

net/remote/remote-client.sh

@@ -50,6 +50,7 @@ local)
 esac
 
 scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
+scripts/net-stats.sh  > net-stats.log 2>&1 &
 
 ! tmux list-sessions || tmux kill-session
 

net/remote/remote-node.sh

@@ -87,6 +87,7 @@ local)
   export LD_LIBRARY_PATH="$PWD/target/perf-libs:$LD_LIBRARY_PATH"
 
   scripts/oom-monitor.sh  > oom-monitor.log 2>&1 &
+  scripts/net-stats.sh  > net-stats.log 2>&1 &
 
   case $nodeType in
   leader)

scripts/net-stats.sh

@@ -0,0 +1,49 @@
+#!/bin/bash -e
+#
+# Reports network statistics
+#
+
+[[ $(uname) == Linux ]] || exit 0
+
+cd "$(dirname "$0")"
+
+# shellcheck source=scripts/configure-metrics.sh
+source configure-metrics.sh
+
+packets_received=0
+packets_received_diff=0
+receive_errors=0
+receive_errors_diff=0
+rcvbuf_errors=0
+rcvbuf_errors_diff=0
+
+update_netstat() {
+  declare net_stat
+  net_stat=$(netstat -suna)
+
+  declare stats
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /packets received/ {tmp_var = $1} END { print tmp_var }')
+  packets_received_diff=$((stats - packets_received))
+  packets_received="$stats"
+
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /packet receive errors/ {tmp_var = $1} END { print tmp_var }')
+  receive_errors_diff=$((stats - receive_errors))
+  receive_errors="$stats"
+
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /RcvbufErrors/ {tmp_var = $2} END { print tmp_var }')
+  rcvbuf_errors_diff=$((stats - rcvbuf_errors))
+  rcvbuf_errors="$stats"
+}
+
+update_netstat
+
+while true; do
+  update_netstat
+  report="packets_received=$packets_received_diff,receive_errors=$receive_errors_diff,rcvbuf_errors=$rcvbuf_errors_diff"
+
+  echo "$report"
+  ./metrics-write-datapoint.sh "net-stats,hostname=$HOSTNAME $report"
+  sleep 60
+done
+
+exit 1

snap/hooks/configure

@@ -37,3 +37,4 @@ validator)
 esac
 
 snapctl start --enable solana.daemon-oom-monitor
+snapctl start --enable solana.daemon-net-stats

snap/snapcraft.yaml

@@ -84,6 +84,11 @@ apps:
     command: scripts/snap-config-to-env.sh $SNAP/scripts/oom-monitor.sh
     plugs:
       - network
+  daemon-net-stats:
+    daemon: simple
+    command: scripts/snap-config-to-env.sh $SNAP/scripts/net-stats.sh
+    plugs:
+      - network
 
 parts:
   solana: