#!/usr/bin/env bash # |source| this file # # Utilities for working with Colo instances # declare COLO_TODO_PARALLELIZE=false __cloud_colo_here="$(dirname "${BASH_SOURCE[0]}")" # shellcheck source=net/scripts/colo-utils.sh source "${__cloud_colo_here}/colo-utils.sh" # Default zone cloud_DefaultZone() { echo "Denver" } # # __cloud_FindInstances # # Find instances matching the specified pattern. # # For each matching instance, an entry in the `instances` array will be added with the # following information about the instance: # "name:zone:public IP:private IP" # # filter - The instances to filter on # # examples: # $ __cloud_FindInstances "name=exact-machine-name" # $ __cloud_FindInstances "name~^all-machines-with-a-common-machine-prefix" # __cloud_FindInstances() { declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INSTANCES_TEXT declare filter=$1 instances=() if ! $COLO_TODO_PARALLELIZE; then colo_load_resources colo_load_availability false fi INSTANCES_TEXT="$( for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"$AVAIL" if [[ $INSTNAME =~ $filter ]]; then printf "%-40s | publicIp=%-16s privateIp=%s zone=%s\n" "$INSTNAME" "$IP" "$PRIV_IP" "$ZONE" 1>&2 echo -e "${INSTNAME}:${IP}:${PRIV_IP}:$ZONE" fi done | sort -t $'\v' -k1 )" if [[ -n "$INSTANCES_TEXT" ]]; then while read -r LINE; do instances+=( "$LINE" ) done <<<"$INSTANCES_TEXT" fi } # # cloud_FindInstances [namePrefix] # # Find instances with names matching the specified prefix # # For each matching instance, an entry in the `instances` array will be added with the # following information about the instance: # "name:public IP:private IP" # # namePrefix - The instance name prefix to look for # # examples: # $ cloud_FindInstances all-machines-with-a-common-machine-prefix # cloud_FindInstances() { declare filter="^${1}.*" __cloud_FindInstances "$filter" } # # cloud_FindInstance [name] # # Find an instance with a name matching the exact pattern. # # For each matching instance, an entry in the `instances` array will be added with the # following information about the instance: # "name:public IP:private IP" # # name - The instance name to look for # # examples: # $ cloud_FindInstance exact-machine-name # cloud_FindInstance() { declare name="^${1}$" __cloud_FindInstances "$name" } # # cloud_Initialize [networkName] # # Perform one-time initialization that may be required for the given testnet. # # networkName - unique name of this testnet # # This function will be called before |cloud_CreateInstances| cloud_Initialize() { # networkName=$1 # unused # zone=$2 #unused colo_load_resources if $COLO_TODO_PARALLELIZE; then colo_load_availability fi } # # cloud_CreateInstances [networkName] [namePrefix] [numNodes] [imageName] # [machineType] [bootDiskSize] [enableGpu] # [startupScript] [address] # # Creates one more identical instances. # # networkName - unique name of this testnet # namePrefix - unique string to prefix all the instance names with # numNodes - number of instances to create # imageName - Disk image for the instances # machineType - GCE machine type. Note that this may also include an # `--accelerator=` or other |gcloud compute instances create| # options # bootDiskSize - Optional size of the boot disk in GB # enableGpu - Optionally enable GPU, use the value "true" to enable # eg, request 4 K80 GPUs with "count=4,type=nvidia-tesla-k80" # startupScript - Optional startup script to execute when the instance boots # address - Optional name of the GCE static IP address to attach to the # instance. Requires that |numNodes| = 1 and that addressName # has been provisioned in the GCE region that is hosting `$zone` # bootDiskType - Optional specify SSD or HDD boot disk # additionalDiskSize - Optional specify size of additional storage volume # # Tip: use cloud_FindInstances to locate the instances once this function # returns cloud_CreateInstances() { #declare networkName="$1" # unused declare namePrefix="$2" declare numNodes="$3" #declare enableGpu="$4" # unused declare machineType="$5" # declare zone="$6" # unused #declare optionalBootDiskSize="$7" # unused #declare optionalStartupScript="$8" # unused #declare optionalAddress="$9" # unused #declare optionalBootDiskType="${10}" # unused #declare optionalAdditionalDiskSize="${11}" # unused declare sshPrivateKey="${12}" declare -a nodes if [[ $numNodes = 1 ]]; then nodes=("$namePrefix") else for node in $(seq -f "${namePrefix}%0${#numNodes}g" 1 "$numNodes"); do nodes+=("$node") done fi if $COLO_TODO_PARALLELIZE; then declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INDEX RES LINE declare -a AVAILABLE declare AVAILABLE_TEXT AVAILABLE_TEXT="$( for RES in "${COLO_RES_AVAILABILITY[@]}"; do IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"$RES" if [[ "FREE" = "$STATUS" ]]; then INDEX=$(colo_res_index_from_ip "$IP") RES_MACH="${COLO_RES_MACHINE[$INDEX]}" if colo_machine_types_compatible "$RES_MACH" "$machineType"; then if ! colo_node_is_requisitioned "$INDEX" "${COLO_RES_REQUISITIONED[*]}"; then echo -e "$RES_MACH\v$IP" fi fi fi done | sort -nt $'\v' -k1,1 )" if [[ -n "$AVAILABLE_TEXT" ]]; then while read -r LINE; do AVAILABLE+=("$LINE") done <<<"$AVAILABLE_TEXT" fi if [[ ${#AVAILABLE[@]} -lt $numNodes ]]; then echo "Insufficient resources available to allocate $numNodes $namePrefix" 1>&2 exit 1 fi declare node declare AI=0 for node in "${nodes[@]}"; do IFS=$'\v' read -r _ IP <<<"${AVAILABLE[$AI]}" colo_node_requisition "$IP" "$node" >/dev/null AI=$((AI+1)) done else declare RES_MACH node declare RI=0 declare NI=0 while [[ $NI -lt $numNodes && $RI -lt $COLO_RES_N ]]; do node="${nodes[$NI]}" RES_MACH="${COLO_RES_MACHINE[$RI]}" IP="${COLO_RES_IP_PRIV[$RI]}" if colo_machine_types_compatible "$RES_MACH" "$machineType"; then if colo_node_requisition "$IP" "$node" "$sshPrivateKey" >/dev/null; then NI=$((NI+1)) fi fi RI=$((RI+1)) done fi } # # cloud_DeleteInstances # # Deletes all the instances listed in the `instances` array # cloud_DeleteInstances() { declare _ IP _ _ for instance in "${instances[@]}"; do IFS=':' read -r _ IP _ _ <<< "$instance" colo_node_free "$IP" >/dev/null done } # # cloud_WaitForInstanceReady [instanceName] [instanceIp] [instanceZone] [timeout] # # Return once the newly created VM instance is responding. This function is cloud-provider specific. # cloud_WaitForInstanceReady() { #declare instanceName="$1" # unused declare instanceIp="$2" declare timeout="$4" timeout "${timeout}"s bash -c "set -o pipefail; until ping -c 3 $instanceIp | tr - _; do echo .; done" } # # cloud_FetchFile [instanceName] [publicIp] [remoteFile] [localFile] # # Fetch a file from the given instance. This function uses a cloud-specific # mechanism to fetch the file # cloud_FetchFile() { #declare instanceName="$1" # unused declare publicIp="$2" declare remoteFile="$3" declare localFile="$4" #declare zone="$5" # unused scp \ -o "StrictHostKeyChecking=no" \ -o "UserKnownHostsFile=/dev/null" \ -o "User=solana" \ -o "LogLevel=ERROR" \ -F /dev/null \ "solana@$publicIp:$remoteFile" "$localFile" } cloud_StatusAll() { declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME if ! $COLO_TODO_PARALLELIZE; then colo_load_resources colo_load_availability false fi for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"$AVAIL" printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s\n" "$HOST_NAME" "$IP" "$PRIV_IP" "$STATUS" "$LOCK_USER" "$ZONE" "$INSTNAME" done }