diff --git a/net/scripts/colo-node-onacquire-sh b/net/scripts/colo-node-onacquire-sh deleted file mode 100644 index c2a7f6767..000000000 --- a/net/scripts/colo-node-onacquire-sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# XXX: This file isn't *quite* a script. It is intended to be passed via stdin -# to a node to requisition logic up node creation. Currently this is done in -# colo_node_requisition using the eval-cat trick. While this gets us what we -# want, care must be taken to ensure variable expansion happens at the right -# time. Any unescaped variable references ($X) in this file will be expanded by -# eval in colo_node_requisition. Escaped variable references (\$X) will be -# expanded upon execution on the remote node. - -if [ ! -f "$SOLANA_LOCK_FILE" ]; then - exec 9>>"$SOLANA_LOCK_FILE" - flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 ) - [ -n "\$SOLANA_USER" ] && { - echo "export SOLANA_LOCK_USER=\$SOLANA_USER" - echo "export SOLANA_LOCK_INSTANCENAME=$INSTANCE_NAME" - echo "[ -v SSH_TTY -a -f \"\${HOME}/.solana-motd\" ] && cat \"\${HOME}/.solana-motd\" 1>&2" - } >&9 || ( rm "$SOLANA_LOCK_FILE" && echo "SOLANA_USER undefined" 1>&2 && false ) - 9>&- - cat > /solana-scratch/id_ecdsa < /solana-scratch/id_ecdsa.pub < /solana-scratch/authorized_keys < /dev/null) -$(cat "${SSH_PRIVATE_KEY}.pub") -EOAK - cp /solana-scratch/id_ecdsa "\${HOME}/.ssh/id_ecdsa" - cp /solana-scratch/id_ecdsa.pub "\${HOME}/.ssh/id_ecdsa.pub" - cp /solana-scratch/authorized_keys "\${HOME}/.ssh/authorized_keys" - cat > "\${HOME}/.solana-motd" <&- - echo "${INSTANCE_NAME} candidate is already \${SOLANA_LOCK_INSTANCENAME}" 1>&2 - false -fi diff --git a/net/scripts/colo-node-onacquire.sh b/net/scripts/colo-node-onacquire.sh new file mode 100644 index 000000000..4d6b4a1af --- /dev/null +++ b/net/scripts/colo-node-onacquire.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# These variable must be set before the main body is called +SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}" +INSTANCE_NAME="${INSTANCE_NAME:?}" +SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}" +SSH_PRIVATE_KEY_TEXT="${SSH_PRIVATE_KEY_TEXT:?}" +SSH_PUBLIC_KEY_TEXT="${SSH_PUBLIC_KEY_TEXT:?}" +NETWORK_INFO="${NETWORK_INFO:-"Network info unavailable"}" +CREATION_INFO="${CREATION_INFO:-"Creation info unavailable"}" + +if [[ ! -f "${SOLANA_LOCK_FILE}" ]]; then + exec 9>>"${SOLANA_LOCK_FILE}" + flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 ) + SOLANA_USER="${SOLANA_USER:?"SOLANA_USER undefined"}" + { + echo "export SOLANA_LOCK_USER=${SOLANA_USER}" + echo "export SOLANA_LOCK_INSTANCENAME=${INSTANCE_NAME}" + echo "[[ -v SSH_TTY -a -f \"${HOME}/.solana-motd\" ]] && cat \"${HOME}/.solana-motd\" 1>&2" + } >&9 + exec 9>&- + cat > /solana-scratch/id_ecdsa < /solana-scratch/id_ecdsa.pub < /solana-scratch/authorized_keys < "${HOME}/.solana-motd" <&- + echo "${INSTANCE_NAME} candidate is already ${SOLANA_LOCK_INSTANCENAME}" 1>&2 + false +fi diff --git a/net/scripts/colo-node-onfree-sh b/net/scripts/colo-node-onfree-sh deleted file mode 100644 index 726787fde..000000000 --- a/net/scripts/colo-node-onfree-sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -# XXX: This file isn't *quite* a script. It is intended to be passed via stdin -# to a node to execute cleanup logic upon deletion. Currently this is done in -# colo_node_free using the eval-cat trick. While this gets us what we want, -# care must be taken to ensure variable expansion happens at the right time. -# Any unescaped variable references ($X) in this file will be expanded by eval -# in colo_node_free. Escaped variable references (\$X) will be expanded upon -# execution on the remote node. - -RC=false -if [ -f "$SOLANA_LOCK_FILE" ]; then - exec 9<>"$SOLANA_LOCK_FILE" - flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 ) - . "$SOLANA_LOCK_FILE" - if [ "\$SOLANA_LOCK_USER" = "\$SOLANA_USER" ]; then - # Begin running process cleanup - CLEANUP_PID=\$$ - CLEANUP_PIDS=() - CLEANUP_PPIDS=() - get_pids() { - CLEANUP_PIDS=() - CLEANUP_PPIDS=() - declare line maybe_ppid maybe_pid - while read line; do - read maybe_ppid maybe_pid _ _ _ _ _ _ _ _ <<<"\$line" - CLEANUP_PIDS+=( \$maybe_pid ) - CLEANUP_PPIDS+=( \$maybe_ppid ) - done < <(ps jxh | sort -rn -k2,2) - } - - CLEANUP_PROC_CHAINS=() - resolve_chains() { - CLEANUP_PROC_CHAINS=() - declare i pid ppid handled n - for i in "\${!CLEANUP_PIDS[@]}"; do - pid=\${CLEANUP_PIDS[\$i]} - ppid=\${CLEANUP_PPIDS[\$i]} - handled=false - - for j in "\${!CLEANUP_PROC_CHAINS[@]}"; do - if grep -q "^\${ppid}\\\\b" <<<"\${CLEANUP_PROC_CHAINS[\$j]}"; then - CLEANUP_PROC_CHAINS[\$j]="\$pid \${CLEANUP_PROC_CHAINS[\$j]}" - handled=true - break - elif grep -q "\\\\b\${pid}\\$" <<<"\${CLEANUP_PROC_CHAINS[\$j]}"; then - CLEANUP_PROC_CHAINS[\$j]+=" \$ppid" - handled=true - # Don't break, we may be the parent of may proc chains - fi - done - if ! \$handled; then - n=\${#CLEANUP_PROC_CHAINS[@]} - CLEANUP_PROC_CHAINS[\$n]="\$pid \$ppid" - fi - done - } - - # Kill screen sessions - while read SID; do - screen -S "\$SID" -X quit - done < <(screen -wipe 2>&1 | sed -e 's/^\s\+\([^[:space:]]\+\)\s.*/\1/;t;d') - - # Kill tmux sessions - tmux kill-server &> /dev/null - - # Kill other processes - for SIG in INT TERM KILL; do - get_pids - if [[ \${#CLEANUP_PIDS[@]} -eq 0 ]]; then - break - else - resolve_chains - for p in "\${CLEANUP_PROC_CHAINS[@]}"; do - if ! grep -q "\b\$CLEANUP_PID\b" <<<"\$p"; then - read -a TO_KILL <<<"\$p" - N=\${#TO_KILL[@]} - ROOT_PPID="\${TO_KILL[\$((N-1))]}" - if [[ 1 -ne \$ROOT_PPID ]]; then - LAST_PID_IDX=\$((N-2)) - for I in \$(seq 0 \$LAST_PID_IDX); do - pid="\${TO_KILL[\$I]}" - kill -\$SIG \$pid &>/dev/null - done - fi - fi - done - get_pids - if [[ \${#CLEANUP_PIDS[@]} -gt 0 ]]; then - sleep 5 - fi - fi - done - # End running process cleanup - - # Begin filesystem cleanup - git clean -qxdff - rm -f /solana-scratch/* /solana-scratch/.[^.]* - cat > "\${HOME}/.ssh/authorized_keys" < /dev/null) -EOAK - EXTERNAL_CONFIG_DIR="${SECONDARY_DISK_MOUNT_POINT}/config/" - if [[ -d "\$EXTERNAL_CONFIG_DIR" ]]; then - rm -rf "\$EXTERNAL_CONFIG_DIR" - fi - # End filesystem cleanup - RC=true - else - echo "Invalid user: expected \\\"\$SOLANA_LOCK_USER\\\" got \\\"\$SOLANA_USER\\\"" - fi - 9>&- -fi -\$RC - diff --git a/net/scripts/colo-node-onfree.sh b/net/scripts/colo-node-onfree.sh new file mode 100644 index 000000000..077034566 --- /dev/null +++ b/net/scripts/colo-node-onfree.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# These variable must be set before the main body is called +SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}" +SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT:?}" +SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}" + +RC=false +if [[ -f "${SOLANA_LOCK_FILE}" ]]; then + exec 9<>"${SOLANA_LOCK_FILE}" + flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 ) + # shellcheck disable=SC1090 + . "${SOLANA_LOCK_FILE}" + if [[ "${SOLANA_LOCK_USER}" = "${SOLANA_USER}" ]]; then + # Begin running process cleanup + CLEANUP_PID=$$ + CLEANUP_PIDS=() + CLEANUP_PPIDS=() + get_pids() { + CLEANUP_PIDS=() + CLEANUP_PPIDS=() + declare line maybe_ppid maybe_pid + while read -r line; do + read -r maybe_ppid maybe_pid _ _ _ _ _ _ _ _ <<<"${line}" + CLEANUP_PIDS+=( "${maybe_pid}" ) + CLEANUP_PPIDS+=( "${maybe_ppid}" ) + done < <(ps jxh | sort -rn -k2,2) + } + + CLEANUP_PROC_CHAINS=() + resolve_chains() { + CLEANUP_PROC_CHAINS=() + declare i pid ppid handled n + for i in "${!CLEANUP_PIDS[@]}"; do + pid=${CLEANUP_PIDS[${i}]} + ppid=${CLEANUP_PPIDS[${i}]} + handled=false + + for j in "${!CLEANUP_PROC_CHAINS[@]}"; do + if grep -q "^${ppid}\b" <<<"${CLEANUP_PROC_CHAINS[${j}]}"; then + CLEANUP_PROC_CHAINS[${j}]="${pid} ${CLEANUP_PROC_CHAINS[${j}]}" + handled=true + break + elif grep -q "\b${pid}\$" <<<"${CLEANUP_PROC_CHAINS[${j}]}"; then + CLEANUP_PROC_CHAINS[${j}]+=" ${ppid}" + handled=true + # Don't break, we may be the parent of may proc chains + fi + done + if ! ${handled}; then + n=${#CLEANUP_PROC_CHAINS[@]} + CLEANUP_PROC_CHAINS[${n}]="${pid} ${ppid}" + fi + done + } + + # Kill screen sessions + while read -r SID; do + screen -S "${SID}" -X quit + done < <(screen -wipe 2>&1 | sed -e 's/^\s\+\([^[:space:]]\+\)\s.*/\1/;t;d') + + # Kill tmux sessions + tmux kill-server &> /dev/null + + # Kill other processes + for SIG in INT TERM KILL; do + get_pids + if [[ ${#CLEANUP_PIDS[@]} -eq 0 ]]; then + break + else + resolve_chains + for p in "${CLEANUP_PROC_CHAINS[@]}"; do + if ! grep -q "\b${CLEANUP_PID}\b" <<<"${p}"; then + read -ra TO_KILL <<<"${p}" + N=${#TO_KILL[@]} + ROOT_PPID="${TO_KILL[$((N-1))]}" + if [[ 1 -ne ${ROOT_PPID} ]]; then + LAST_PID_IDX=$((N-2)) + for I in $(seq 0 ${LAST_PID_IDX}); do + pid="${TO_KILL[${I}]}" + kill "-${SIG}" "${pid}" &>/dev/null + done + fi + fi + done + get_pids + if [[ ${#CLEANUP_PIDS[@]} -gt 0 ]]; then + sleep 5 + fi + fi + done + # End running process cleanup + + # Begin filesystem cleanup + git clean -qxdff + rm -f /solana-scratch/* /solana-scratch/.[^.]* + cat > "${HOME}/.ssh/authorized_keys" <&2 + fi + exec 9>&- +fi +${RC} + diff --git a/net/scripts/colo-provider.sh b/net/scripts/colo-provider.sh index 34105e7a3..818d78f14 100755 --- a/net/scripts/colo-provider.sh +++ b/net/scripts/colo-provider.sh @@ -38,26 +38,26 @@ cloud_RestartPreemptedInstances() { # __cloud_FindInstances() { declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INSTANCES_TEXT - declare filter=$1 + declare filter=${1} instances=() - if ! $COLO_PARALLELIZE; then + if ! ${COLO_PARALLELIZE}; then colo_load_resources colo_load_availability false fi INSTANCES_TEXT="$( for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do - IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"$AVAIL" - if [[ $INSTNAME =~ $filter ]]; then - printf "%-40s | publicIp=%-16s privateIp=%s zone=%s\n" "$INSTNAME" "$IP" "$PRIV_IP" "$ZONE" 1>&2 - echo -e "${INSTNAME}:${IP}:${PRIV_IP}:$ZONE" + IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${AVAIL}" + if [[ ${INSTNAME} =~ ${filter} ]]; then + printf "%-40s | publicIp=%-16s privateIp=%s zone=%s\n" "${INSTNAME}" "${IP}" "${PRIV_IP}" "${ZONE}" 1>&2 + echo -e "${INSTNAME}:${IP}:${PRIV_IP}:${ZONE}" fi done | sort -t $'\v' -k1 )" - if [[ -n "$INSTANCES_TEXT" ]]; then + if [[ -n "${INSTANCES_TEXT}" ]]; then while read -r LINE; do - instances+=( "$LINE" ) - done <<<"$INSTANCES_TEXT" + instances+=( "${LINE}" ) + done <<<"${INSTANCES_TEXT}" fi } @@ -77,7 +77,7 @@ __cloud_FindInstances() { # cloud_FindInstances() { declare filter="^${1}.*" - __cloud_FindInstances "$filter" + __cloud_FindInstances "${filter}" } # @@ -96,7 +96,7 @@ cloud_FindInstances() { # cloud_FindInstance() { declare name="^${1}$" - __cloud_FindInstances "$name" + __cloud_FindInstances "${name}" } # @@ -108,10 +108,10 @@ cloud_FindInstance() { # # This function will be called before |cloud_CreateInstances| cloud_Initialize() { - # networkName=$1 # unused - # zone=$2 #unused + # networkName=${1} # unused + # zone=${2} #unused colo_load_resources - if $COLO_PARALLELIZE; then + if ${COLO_PARALLELIZE}; then colo_load_availability fi } @@ -136,7 +136,7 @@ cloud_Initialize() { # startupScript - Optional startup script to execute when the instance boots # address - Optional name of the GCE static IP address to attach to the # instance. Requires that |numNodes| = 1 and that addressName -# has been provisioned in the GCE region that is hosting `$zone` +# has been provisioned in the GCE region that is hosting `${zone}` # bootDiskType - Optional specify SSD or HDD boot disk # additionalDiskSize - Optional specify size of additional storage volume # preemptible - Optionally request a preemptible instance ("true") @@ -144,76 +144,76 @@ cloud_Initialize() { # Tip: use cloud_FindInstances to locate the instances once this function # returns cloud_CreateInstances() { - #declare networkName="$1" # unused - declare namePrefix="$2" - declare numNodes="$3" - #declare enableGpu="$4" # unused - declare machineType="$5" - # declare zone="$6" # unused - #declare optionalBootDiskSize="$7" # unused - #declare optionalStartupScript="$8" # unused - #declare optionalAddress="$9" # unused + #declare networkName="${1}" # unused + declare namePrefix="${2}" + declare numNodes="${3}" + #declare enableGpu="${4}" # unused + declare machineType="${5}" + # declare zone="${6}" # unused + #declare optionalBootDiskSize="${7}" # unused + #declare optionalStartupScript="${8}" # unused + #declare optionalAddress="${9}" # unused #declare optionalBootDiskType="${10}" # unused #declare optionalAdditionalDiskSize="${11}" # unused #declare optionalPreemptible="${12}" # unused declare sshPrivateKey="${13}" declare -a nodes - if [[ $numNodes = 1 ]]; then - nodes=("$namePrefix") + if [[ ${numNodes} = 1 ]]; then + nodes=("${namePrefix}") else - for node in $(seq -f "${namePrefix}%0${#numNodes}g" 1 "$numNodes"); do - nodes+=("$node") + for node in $(seq -f "${namePrefix}%0${#numNodes}g" 1 "${numNodes}"); do + nodes+=("${node}") done fi - if $COLO_PARALLELIZE; then + if ${COLO_PARALLELIZE}; then declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INDEX RES LINE declare -a AVAILABLE declare AVAILABLE_TEXT AVAILABLE_TEXT="$( for RES in "${COLO_RES_AVAILABILITY[@]}"; do - IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"$RES" - if [[ "FREE" = "$STATUS" ]]; then - INDEX=$(colo_res_index_from_ip "$IP") - RES_MACH="${COLO_RES_MACHINE[$INDEX]}" - if colo_machine_types_compatible "$RES_MACH" "$machineType"; then - if ! colo_node_is_requisitioned "$INDEX" "${COLO_RES_REQUISITIONED[*]}"; then - echo -e "$RES_MACH\v$IP" + IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${RES}" + if [[ "FREE" = "${STATUS}" ]]; then + INDEX=$(colo_res_index_from_ip "${IP}") + RES_MACH="${COLO_RES_MACHINE[${INDEX}]}" + if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then + if ! colo_node_is_requisitioned "${INDEX}" "${COLO_RES_REQUISITIONED[*]}"; then + echo -e "${RES_MACH}\v${IP}" fi fi fi done | sort -nt $'\v' -k1,1 )" - if [[ -n "$AVAILABLE_TEXT" ]]; then + if [[ -n "${AVAILABLE_TEXT}" ]]; then while read -r LINE; do - AVAILABLE+=("$LINE") - done <<<"$AVAILABLE_TEXT" + AVAILABLE+=("${LINE}") + done <<<"${AVAILABLE_TEXT}" fi - if [[ ${#AVAILABLE[@]} -lt $numNodes ]]; then - echo "Insufficient resources available to allocate $numNodes $namePrefix" 1>&2 + if [[ ${#AVAILABLE[@]} -lt ${numNodes} ]]; then + echo "Insufficient resources available to allocate ${numNodes} ${namePrefix}" 1>&2 exit 1 fi declare node declare AI=0 for node in "${nodes[@]}"; do - IFS=$'\v' read -r _ IP <<<"${AVAILABLE[$AI]}" - colo_node_requisition "$IP" "$node" >/dev/null + IFS=$'\v' read -r _ IP <<<"${AVAILABLE[${AI}]}" + colo_node_requisition "${IP}" "${node}" >/dev/null AI=$((AI+1)) done else declare RES_MACH node declare RI=0 declare NI=0 - while [[ $NI -lt $numNodes && $RI -lt $COLO_RES_N ]]; do - node="${nodes[$NI]}" - RES_MACH="${COLO_RES_MACHINE[$RI]}" - IP="${COLO_RES_IP[$RI]}" - if colo_machine_types_compatible "$RES_MACH" "$machineType"; then - if colo_node_requisition "$IP" "$node" "$sshPrivateKey" >/dev/null; then + while [[ ${NI} -lt ${numNodes} && ${RI} -lt ${COLO_RES_N} ]]; do + node="${nodes[${NI}]}" + RES_MACH="${COLO_RES_MACHINE[${RI}]}" + IP="${COLO_RES_IP[${RI}]}" + if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then + if colo_node_requisition "${IP}" "${node}" "${sshPrivateKey}" >/dev/null; then NI=$((NI+1)) fi fi @@ -230,8 +230,8 @@ cloud_CreateInstances() { cloud_DeleteInstances() { declare _ IP _ _ for instance in "${instances[@]}"; do - IFS=':' read -r _ IP _ _ <<< "$instance" - colo_node_free "$IP" >/dev/null + IFS=':' read -r _ IP _ _ <<< "${instance}" + colo_node_free "${IP}" >/dev/null done } @@ -241,9 +241,9 @@ cloud_DeleteInstances() { # Return once the newly created VM instance is responding. This function is cloud-provider specific. # cloud_WaitForInstanceReady() { - #declare instanceName="$1" # unused - #declare instanceIp="$2" # unused - #declare timeout="$4" # unused + #declare instanceName="${1}" # unused + #declare instanceIp="${2}" # unused + #declare timeout="${4}" # unused true } @@ -255,28 +255,28 @@ cloud_WaitForInstanceReady() { # mechanism to fetch the file # cloud_FetchFile() { - #declare instanceName="$1" # unused - declare publicIp="$2" - declare remoteFile="$3" - declare localFile="$4" - #declare zone="$5" # unused + #declare instanceName="${1}" # unused + declare publicIp="${2}" + declare remoteFile="${3}" + declare localFile="${4}" + #declare zone="${5}" # unused scp \ -o "StrictHostKeyChecking=no" \ -o "UserKnownHostsFile=/dev/null" \ -o "User=solana" \ -o "LogLevel=ERROR" \ -F /dev/null \ - "solana@$publicIp:$remoteFile" "$localFile" + "solana@${publicIp}:${remoteFile}" "${localFile}" } cloud_StatusAll() { declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME - if ! $COLO_PARALLELIZE; then + if ! ${COLO_PARALLELIZE}; then colo_load_resources colo_load_availability false fi for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do - IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"$AVAIL" - printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s\n" "$HOST_NAME" "$IP" "$PRIV_IP" "$STATUS" "$LOCK_USER" "$ZONE" "$INSTNAME" + IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${AVAIL}" + printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s\n" "${HOST_NAME}" "${IP}" "${PRIV_IP}" "${STATUS}" "${LOCK_USER}" "${ZONE}" "${INSTNAME}" done } diff --git a/net/scripts/colo-utils.sh b/net/scripts/colo-utils.sh index 968ff8488..3e92b724f 100644 --- a/net/scripts/colo-utils.sh +++ b/net/scripts/colo-utils.sh @@ -5,7 +5,7 @@ declare -r SOLANA_LOCK_FILE="/home/solana/.solana.lock" __colo_here="$(dirname "${BASH_SOURCE[0]}")" # shellcheck source=net/common.sh -source "$__colo_here"/../common.sh +source "${__colo_here}"/../common.sh # Load colo resource specs export COLO_RES_N=0 @@ -24,20 +24,20 @@ export COLO_RESOURCES_LOADED=false colo_load_resources() { if ! ${COLO_RESOURCES_LOADED}; then while read -r LINE; do - IFS='|' read -r H I PI C M ST SC AST ASC G Z <<<"$LINE" - COLO_RES_HOSTNAME+=( "$H" ) - COLO_RES_IP+=( "$I" ) - COLO_RES_IP_PRIV+=( "$PI" ) - COLO_RES_CPU_CORES+=( "$C" ) - COLO_RES_RAM_GB+=( "$M" ) - COLO_RES_STORAGE_TYPE+=( "$ST" ) - COLO_RES_STORAGE_CAP_GB+=( "$SC" ) - COLO_RES_ADD_STORAGE_TYPE+=( "$(tr ',' $'\v' <<<"$AST")" ) - COLO_RES_ADD_STORAGE_CAP_GB+=( "$(tr ',' $'\v' <<<"$ASC")" ) - COLO_RES_MACHINE+=( "$G" ) - COLO_RES_ZONE+=( "$Z" ) + IFS='|' read -r H I PI C M ST SC AST ASC G Z <<<"${LINE}" + COLO_RES_HOSTNAME+=( "${H}" ) + COLO_RES_IP+=( "${I}" ) + COLO_RES_IP_PRIV+=( "${PI}" ) + COLO_RES_CPU_CORES+=( "${C}" ) + COLO_RES_RAM_GB+=( "${M}" ) + COLO_RES_STORAGE_TYPE+=( "${ST}" ) + COLO_RES_STORAGE_CAP_GB+=( "${SC}" ) + COLO_RES_ADD_STORAGE_TYPE+=( "$(tr ',' $'\v' <<<"${AST}")" ) + COLO_RES_ADD_STORAGE_CAP_GB+=( "$(tr ',' $'\v' <<<"${ASC}")" ) + COLO_RES_MACHINE+=( "${G}" ) + COLO_RES_ZONE+=( "${Z}" ) COLO_RES_N=$((COLO_RES_N+1)) - done < <(sort -nt'|' -k10,10 "$__colo_here"/colo_nodes) + done < <(sort -nt'|' -k10,10 "${__colo_here}"/colo_nodes) COLO_RESOURCES_LOADED=true fi } @@ -47,26 +47,26 @@ declare -ax COLO_RES_AVAILABILITY colo_load_availability() { declare USE_CACHE=${1:-${COLO_RES_AVAILABILITY_CACHED}} declare LINE PRIV_IP STATUS LOCK_USER I IP HOST_NAME ZONE INSTNAME - if ! $USE_CACHE; then + if ! ${USE_CACHE}; then COLO_RES_AVAILABILITY=() COLO_RES_REQUISITIONED=() while read -r LINE; do - IFS=$'\v' read -r IP STATUS LOCK_USER INSTNAME <<< "$LINE" - I=$(colo_res_index_from_ip "$IP") - PRIV_IP="${COLO_RES_IP_PRIV[$I]}" - HOST_NAME="${COLO_RES_HOSTNAME[$I]}" - ZONE="${COLO_RES_ZONE[$I]}" - COLO_RES_AVAILABILITY+=( "$(echo -e "$HOST_NAME\v$IP\v$PRIV_IP\v$STATUS\v$ZONE\v$LOCK_USER\v$INSTNAME")" ) + IFS=$'\v' read -r IP STATUS LOCK_USER INSTNAME <<< "${LINE}" + I=$(colo_res_index_from_ip "${IP}") + PRIV_IP="${COLO_RES_IP_PRIV[${I}]}" + HOST_NAME="${COLO_RES_HOSTNAME[${I}]}" + ZONE="${COLO_RES_ZONE[${I}]}" + COLO_RES_AVAILABILITY+=( "$(echo -e "${HOST_NAME}\v${IP}\v${PRIV_IP}\v${STATUS}\v${ZONE}\v${LOCK_USER}\v${INSTNAME}")" ) done < <(colo_node_status_all | sort -t $'\v' -k1) COLO_RES_AVAILABILITY_CACHED=true fi } colo_res_index_from_ip() { - declare IP="$1" + declare IP="${1}" for i in "${!COLO_RES_IP_PRIV[@]}"; do - if [[ "$IP" = "${COLO_RES_IP[$i]}" || "$IP" = "${COLO_RES_IP_PRIV[$i]}" ]]; then - echo "$i" + if [[ "${IP}" = "${COLO_RES_IP[${i}]}" || "${IP}" = "${COLO_RES_IP_PRIV[${i}]}" ]]; then + echo "${i}" return 0 fi done @@ -74,36 +74,36 @@ colo_res_index_from_ip() { } colo_instance_run() { - declare IP=$1 - declare CMD="$2" + declare IP=${1} + declare CMD="${2}" declare OUT set +e - OUT=$(ssh -l solana -o "StrictHostKeyChecking=no" -o "ConnectTimeout=3" -n "$IP" "$CMD" 2>&1) + OUT=$(ssh -l solana -o "StrictHostKeyChecking=no" -o "ConnectTimeout=3" -n "${IP}" "${CMD}" 2>&1) declare RC=$? set -e while read -r LINE; do - echo -e "$IP\v$RC\v$LINE" - if [[ "$RC" -ne 0 ]]; then + echo -e "${IP}\v${RC}\v${LINE}" + if [[ "${RC}" -ne 0 ]]; then echo "IP(${IP}) Err(${RC}) LINE(${LINE})" 1>&2 fi - done < <(tr -d $'\r' <<<"$OUT") - return $RC + done < <(tr -d $'\r' <<<"${OUT}") + return ${RC} } colo_instance_run_foreach() { declare CMD if test 1 -eq $#; then - CMD="$1" + CMD="${1}" declare IPS=() for I in $(seq 0 $((COLO_RES_N-1))); do - IPS+=( "${COLO_RES_IP[$I]}" ) + IPS+=( "${COLO_RES_IP[${I}]}" ) done - set "${IPS[@]}" "$CMD" + set "${IPS[@]}" "${CMD}" fi CMD="${*: -1}" for I in $(seq 0 $(($#-2))); do - declare IP="$1" - colo_instance_run "$IP" "$CMD" & + declare IP="${1}" + colo_instance_run "${IP}" "${CMD}" & shift done @@ -114,35 +114,35 @@ colo_whoami() { declare ME LINE SOL_USER EOL while read -r LINE; do declare IP RC - IFS=$'\v' read -r IP RC SOL_USER EOL <<< "$LINE" - if [ "$RC" -eq 0 ]; then - [[ "$EOL" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"$LINE\"" 1>&2 - if [ -z "$ME" ] || [ "$ME" = "$SOL_USER" ]; then - ME="$SOL_USER" + IFS=$'\v' read -r IP RC SOL_USER EOL <<< "${LINE}" + if [ "${RC}" -eq 0 ]; then + [[ "${EOL}" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"${LINE}\"" 1>&2 + if [ -z "${ME}" ] || [ "${ME}" = "${SOL_USER}" ]; then + ME="${SOL_USER}" else - echo "Found conflicting username \"$SOL_USER\" on $IP, expected \"$ME\"" 1>&2 + echo "Found conflicting username \"${SOL_USER}\" on ${IP}, expected \"${ME}\"" 1>&2 fi fi - done < <(colo_instance_run_foreach "[ -n \"\$SOLANA_USER\" ] && echo -e \"\$SOLANA_USER\\vEOL\"") - echo "$ME" + done < <(colo_instance_run_foreach "[ -n \"\${SOLANA_USER}\" ] && echo -e \"\${SOLANA_USER}\\vEOL\"") + echo "${ME}" } COLO_SOLANA_USER="" colo_get_solana_user() { - if [ -z "$COLO_SOLANA_USER" ]; then + if [ -z "${COLO_SOLANA_USER}" ]; then COLO_SOLANA_USER=$(colo_whoami) fi - echo "$COLO_SOLANA_USER" + echo "${COLO_SOLANA_USER}" } __colo_node_status_script() { cat <&2 exec 2>/dev/null # Suppress stderr as the next call to exec fails most of - # the time due to $SOLANA_LOCK_FILE not existing and is running from a + # the time due to ${SOLANA_LOCK_FILE} not existing and is running from a # subshell where normal redirection doesn't work - exec 9<"$SOLANA_LOCK_FILE" && flock -s 9 && . "$SOLANA_LOCK_FILE" && exec 9>&- - echo -e "\$SOLANA_LOCK_USER\\v\$SOLANA_LOCK_INSTANCENAME\\vEOL" + exec 9<"${SOLANA_LOCK_FILE}" && flock -s 9 && . "${SOLANA_LOCK_FILE}" && exec 9>&- + echo -e "\${SOLANA_LOCK_USER}\\v\${SOLANA_LOCK_INSTANCENAME}\\vEOL" exec 2>&3 # Restore stderr EOF } @@ -150,31 +150,31 @@ EOF __colo_node_status_result_normalize() { declare IP RC US BY INSTNAME EOL declare ST="DOWN" - IFS=$'\v' read -r IP RC US INSTNAME EOL <<< "$1" - if [ "$RC" -eq 0 ]; then - [[ "$EOL" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"$1\"" 1>&2 - if [ -n "$US" ]; then - BY="$US" + IFS=$'\v' read -r IP RC US INSTNAME EOL <<< "${1}" + if [ "${RC}" -eq 0 ]; then + [[ "${EOL}" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"${1}\"" 1>&2 + if [ -n "${US}" ]; then + BY="${US}" ST="HELD" - if [[ -z "$INSTNAME" ]]; then + if [[ -z "${INSTNAME}" ]]; then return fi else ST="FREE" fi fi - echo -e $"$IP\v$ST\v$BY\v$INSTNAME" + echo -e $"${IP}\v${ST}\v${BY}\v${INSTNAME}" } colo_node_status() { - declare IP="$1" - __colo_node_status_result_normalize "$(colo_instance_run "$IP" "$(__colo_node_status_script)")" + declare IP="${1}" + __colo_node_status_result_normalize "$(colo_instance_run "${IP}" "$(__colo_node_status_script)")" } colo_node_status_all() { declare LINE while read -r LINE; do - __colo_node_status_result_normalize "$LINE" + __colo_node_status_result_normalize "${LINE}" done < <(colo_instance_run_foreach "$(__colo_node_status_script)") } @@ -183,58 +183,68 @@ colo_node_status_all() { # for validators export COLO_RES_REQUISITIONED=() colo_node_requisition() { - declare IP=$1 + declare IP=${1} # shellcheck disable=SC2034 - declare INSTANCE_NAME=$2 + declare INSTANCE_NAME=${2} # shellcheck disable=SC2034 - declare SSH_PRIVATE_KEY="$3" + declare SSH_PRIVATE_KEY="${3}" declare INDEX - INDEX=$(colo_res_index_from_ip "$IP") + INDEX=$(colo_res_index_from_ip "${IP}") declare RC=false - colo_instance_run "$IP" "$(eval "cat <