Added counters to track more metrics on dashboard (#1535)

- Total number of IP packets TX/RX from all nodes in the testnet
- Last consumed index on validator
- Last transmitted index on leader

scripts/net-stats.sh

@@ -18,6 +18,10 @@ receive_errors=0
 receive_errors_diff=0
 rcvbuf_errors=0
 rcvbuf_errors_diff=0
+in_octets=0
+in_octets_diff=0
+out_octets=0
+out_octets_diff=0
 
 update_netstat() {
   declare net_stat
@@ -39,13 +43,21 @@ update_netstat() {
   stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /RcvbufErrors/ {tmp_var = $2} END { print tmp_var }')
   rcvbuf_errors_diff=$((stats - rcvbuf_errors))
   rcvbuf_errors="$stats"
+
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /InOctets/ {tmp_var = $2} END { print tmp_var }')
+  in_octets_diff=$((stats - in_octets))
+  in_octets="$stats"
+
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /OutOctets/ {tmp_var = $2} END { print tmp_var }')
+  out_octets_diff=$((stats - out_octets))
+  out_octets="$stats"
 }
 
 update_netstat
 
 while true; do
   update_netstat
-  report="packets_sent=$packets_sent_diff,packets_received=$packets_received_diff,receive_errors=$receive_errors_diff,rcvbuf_errors=$rcvbuf_errors_diff"
+  report="packets_sent=$packets_sent_diff,packets_received=$packets_received_diff,receive_errors=$receive_errors_diff,rcvbuf_errors=$rcvbuf_errors_diff,in_octets=$in_octets_diff,out_octets=$out_octets_diff"
 
   echo "$report"
   ./metrics-write-datapoint.sh "net-stats,hostname=$HOSTNAME $report"

src/broadcast_stage.rs

@@ -5,9 +5,11 @@ use counter::Counter;
 use entry::Entry;
 #[cfg(feature = "erasure")]
 use erasure;
+use influx_db_client as influxdb;
 use leader_scheduler::LeaderScheduler;
 use ledger::Block;
 use log::Level;
+use metrics;
 use packet::SharedBlobs;
 use rayon::prelude::*;
 use result::{Error, Result};
@@ -151,6 +153,14 @@ fn broadcast(
         num_entries, to_blobs_elapsed, chunking_elapsed, broadcast_elapsed
     );
 
+    metrics::submit(
+        influxdb::Point::new("broadcast-stage")
+            .add_field(
+                "transmit-index",
+                influxdb::Value::Integer(transmit_index.data as i64),
+            ).to_owned(),
+    );
+
     Ok(())
 }
 

src/window_service.rs

@@ -307,6 +307,15 @@ pub fn window_service(
                     }
                 }
 
+                metrics::submit(
+                    influxdb::Point::new("window-stage")
+                        .add_field(
+                            "host",
+                            influxdb::Value::String(hostname().unwrap_or_else(|_| "?".to_string())),
+                        ).add_field("consumed", influxdb::Value::Integer(consumed as i64))
+                        .to_owned(),
+                );
+
                 if received <= consumed {
                     trace!(
                         "{} we have everything received:{} consumed:{}",