Retry rsync a couple times before failing

net/remote/remote-client.sh

@@ -27,9 +27,8 @@ fi
 
 case $deployMethod in
 snap)
-  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
+  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/solana/solana.snap" .
   sudo snap install solana.snap --devmode --dangerous
-  rm solana.snap
 
   solana_bench_tps=/snap/bin/solana.bench-tps
   solana_keygen=/snap/bin/solana.keygen
@@ -39,7 +38,7 @@ local)
   export USE_INSTALL=1
   export SOLANA_DEFAULT_METRICS_RATE=1
 
-  rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
   solana_bench_tps=solana-bench-tps
   solana_keygen=solana-keygen
   ;;

net/remote/remote-node.sh

@@ -39,7 +39,8 @@ fi
 case $deployMethod in
 snap)
   SECONDS=0
-  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
+  [[ $nodeType = leader ]] ||
+    net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/solana/solana.snap" .
   sudo snap install solana.snap --devmode --dangerous
 
   commonNodeConfig="\
@@ -94,7 +95,7 @@ local)
     ./multinode-demo/leader.sh > leader.log 2>&1 &
     ;;
   validator)
-    rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+    net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
 
     ./multinode-demo/setup.sh -t validator $setupArgs
     ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp:8001" >validator.log 2>&1 &

net/scripts/rsync-retry.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# rsync wrapper that retries a few times on failure
+#
+
+for i in $(seq 1 5); do
+  (
+    set -x
+    rsync "$@"
+  ) && exit 0
+  echo Retry "$i"...
+done