Integrated use of netem with testnet scripts (#6746)

automerge

net/net.sh

@@ -88,6 +88,8 @@ Operate a configured testnet
 
    --use-move                         - Build the move-loader-program and add it to the cluster
 
+   --netem                            - Use netem to induce packet drops/latencies/errors
+
  sanity/start-specific options:
    -F                   - Discard validator nodes that didn't bootup successfully
    -o noInstallCheck    - Skip solana-install sanity
@@ -141,6 +143,7 @@ debugBuild=false
 doBuild=true
 gpuMode=auto
 maybeUseMove=""
+netemConfig=""
 
 command=$1
 [[ -n $command ]] || usage
@@ -204,6 +207,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --use-move ]]; then
       maybeUseMove=$1
       shift 1
+    elif [[ $1 = --netem ]]; then
+      netemConfig=$2
+      shift 2
     elif [[ $1 = --gpu-mode ]]; then
       gpuMode=$2
       case "$gpuMode" in
@@ -459,6 +465,7 @@ startBootstrapLeader() {
          \"$maybeNoSnapshot $maybeSkipLedgerVerify $maybeLimitLedgerSize\" \
          \"$gpuMode\" \
          \"$GEOLOCATION_API_KEY\" \
+         \"$netemConfig\" \
       "
 
   ) >> "$logFile" 2>&1 || {
@@ -526,6 +533,7 @@ startNode() {
          \"$maybeNoSnapshot $maybeSkipLedgerVerify $maybeLimitLedgerSize\" \
          \"$gpuMode\" \
          \"$GEOLOCATION_API_KEY\" \
+         \"$netemConfig\" \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
@@ -823,6 +831,10 @@ stopNode() {
           \$sudo kill -- -\$pgid
         fi
       done
+      if [[ -f solana/netem.cfg ]]; then
+        solana/scripts/netem.sh delete < solana/netem.cfg
+        rm -f solana/netem.cfg
+      fi
       for pattern in node solana- remote-; do
         pkill -9 \$pattern
       done

net/remote/remote-node.sh

@@ -26,6 +26,7 @@ genesisOptions="${17}"
 extraNodeArgs="${18}"
 gpuMode="${19:-auto}"
 GEOLOCATION_API_KEY="${20}"
+netemConfig="${21}"
 set +x
 
 # Use a very large stake (relative to the default multinode-demo/ stake of 42)
@@ -147,6 +148,11 @@ cat >> ~/solana/on-reboot <<EOF
     echo "Expected GPU, found none!"
     export SOLANA_GPU_MISSING=1
   fi
+
+  if [[ -n "$netemConfig" ]]; then
+    scripts/netem.sh add "$netemConfig"
+    echo "$netemConfig" > netem.cfg
+  fi
 EOF
 
   case $nodeType in

scripts/netem.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Start/Stop network emulation
+#
+set -e
+
+[[ $(uname) == Linux ]] || exit 0
+
+cd "$(dirname "$0")"
+
+sudo=
+if sudo true; then
+  sudo="sudo -n"
+fi
+
+set -x
+
+iface="$(ifconfig | grep mtu | grep -iv loopback | grep -i running | awk 'BEGIN { FS = ":" } ; {print $1}')"
+# shellcheck disable=SC2086 # Do not want to quote $2. It has space separated arguments for netem
+$sudo tc qdisc "$1" dev "$iface" root netem $2