solana/net/scripts/colo-provider.sh

#!/usr/bin/env bash

# |source| this file
#
# Utilities for working with Colo instances
#

# COLO_PARALLELIZE is not ready for use, disable it
declare COLO_PARALLELIZE=false

__cloud_colo_here="$(dirname "${BASH_SOURCE[0]}")"
# shellcheck source=net/scripts/colo-utils.sh
source "${__cloud_colo_here}/colo-utils.sh"

# Default zone
cloud_DefaultZone() {
  echo "Denver"
}

cloud_DefaultCustomMemoryGB() {
  : # Not implemented
}

cloud_RestartPreemptedInstances() {
  : # Not implemented
}

#
# __cloud_FindInstances
#
# Find instances matching the specified pattern.
#
# For each matching instance, an entry in the `instances` array will be added with the
# following information about the instance:
#   "name:zone:public IP:private IP"
#
# filter   - The instances to filter on
#
# examples:
#   $ __cloud_FindInstances "name=exact-machine-name"
#   $ __cloud_FindInstances "name~^all-machines-with-a-common-machine-prefix"
#
__cloud_FindInstances() {
  declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INSTANCES_TEXT
  declare filter=${1}
  declare onlyPreemptible=${2}
  instances=()

  if ! ${COLO_PARALLELIZE}; then
    colo_load_resources
    colo_load_availability false
  fi
  INSTANCES_TEXT="$(
    for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do
      IFS=$'\x1f' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE <<<"${AVAIL}"
      if [[ ${INSTNAME} =~ ${filter} ]]; then
        if [[ -n $onlyPreemptible && $PREEMPTIBLE == "false" ]]; then
          continue
        else
          printf "%-40s | publicIp=%-16s privateIp=%s zone=%s preemptible=%s\n" "${INSTNAME}" "${IP}" "${PRIV_IP}" "${ZONE}" "${PREEMPTIBLE}" 1>&2
          echo -e "${INSTNAME}:${IP}:${PRIV_IP}:${ZONE}"
        fi
      fi
    done | sort -t $'\x1f' -k1
  )"
  if [[ -n "${INSTANCES_TEXT}" ]]; then
    while read -r LINE; do
      instances+=( "${LINE}" )
    done <<<"${INSTANCES_TEXT}"
  fi
}

#
# cloud_FindInstances [namePrefix]
#
# Find instances with names matching the specified prefix
#
# For each matching instance, an entry in the `instances` array will be added with the
# following information about the instance:
#   "name:public IP:private IP"
#
# namePrefix - The instance name prefix to look for
#
# examples:
#   $ cloud_FindInstances all-machines-with-a-common-machine-prefix
#
cloud_FindInstances() {
  declare filter="^${1}.*"
  declare onlyPreemptible="${2}"
  __cloud_FindInstances "${filter}" "${onlyPreemptible}"
}

#
# cloud_FindInstance [name]
#
# Find an instance with a name matching the exact pattern.
#
# For each matching instance, an entry in the `instances` array will be added with the
# following information about the instance:
#   "name:public IP:private IP"
#
# name - The instance name to look for
#
# examples:
#   $ cloud_FindInstance exact-machine-name
#
cloud_FindInstance() {
  declare name="^${1}$"
  declare onlyPreemptible="${2}"
  __cloud_FindInstances "${name}" "${onlyPreemptible}"
}

#
# cloud_Initialize [networkName]
#
# Perform one-time initialization that may be required for the given testnet.
#
# networkName   - unique name of this testnet
#
# This function will be called before |cloud_CreateInstances|
cloud_Initialize() {
  # networkName=${1} # unused
  # zone=${2} #unused
  colo_load_resources
  if ${COLO_PARALLELIZE}; then
    colo_load_availability
  fi
}

#
# cloud_CreateInstances [networkName] [namePrefix] [numNodes]
#                       [enableGpu] [machineType] [zone]
#                       [bootDiskSize] [startupScript] [address]
#                       [bootDiskType] [additionalDiskSize] [preemptible]
#
# Creates one more identical instances.
#
# networkName   - unique name of this testnet
# namePrefix    - unique string to prefix all the instance names with
# numNodes      - number of instances to create
# enableGpu     - Optionally enable GPU, use the value "true" to enable
#                 eg, request 4 K80 GPUs with "count=4,type=nvidia-tesla-k80"
# machineType   - GCE machine type.  Note that this may also include an
#                 `--accelerator=` or other |gcloud compute instances create|
#                 options
# zone          - cloud zone
# bootDiskSize  - Optional size of the boot disk in GB
# startupScript - Optional startup script to execute when the instance boots
# address       - Optional name of the GCE static IP address to attach to the
#                 instance.  Requires that |numNodes| = 1 and that addressName
#                 has been provisioned in the GCE region that is hosting `$zone`
# bootDiskType  - Optional specify SSD or HDD boot disk
# additionalDiskSize - Optional specify size of additional storage volume
# preemptible   - Optionally request a preemptible instance ("true")
#
# Tip: use cloud_FindInstances to locate the instances once this function
#      returns
cloud_CreateInstances() {
  #declare networkName="${1}" # unused
  declare namePrefix="${2}"
  declare numNodes="${3}"
  #declare enableGpu="${4}" # unused
  declare machineType="${5}"
  # declare zone="${6}" # unused
  #declare optionalBootDiskSize="${7}" # unused
  #declare optionalStartupScript="${8}" # unused
  #declare optionalAddress="${9}" # unused
  #declare optionalBootDiskType="${10}" # unused
  #declare optionalAdditionalDiskSize="${11}" # unused
  declare optionalPreemptible="${12}"
  declare sshPrivateKey="${13}"

  declare -a nodes
  if [[ ${numNodes} = 1 ]]; then
    nodes=("${namePrefix}")
  else
    for node in $(seq -f "${namePrefix}%0${#numNodes}g" 1 "${numNodes}"); do
      nodes+=("${node}")
    done
  fi

  if ${COLO_PARALLELIZE}; then
    declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INDEX RES LINE
    declare -a AVAILABLE
    declare AVAILABLE_TEXT
    AVAILABLE_TEXT="$(
      for RES in "${COLO_RES_AVAILABILITY[@]}"; do
        IFS=$'\x1f' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${RES}"
        if [[ "FREE" = "${STATUS}" ]]; then
          INDEX=$(colo_res_index_from_ip "${IP}")
          RES_MACH="${COLO_RES_MACHINE[${INDEX}]}"
          if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then
            if ! colo_node_is_requisitioned "${INDEX}" "${COLO_RES_REQUISITIONED[*]}"; then
              echo -e "${RES_MACH}\x1f${IP}"
            fi
          fi
        fi
      done | sort -nt $'\x1f' -k1,1
    )"

    if [[ -n "${AVAILABLE_TEXT}" ]]; then
      while read -r LINE; do
        AVAILABLE+=("${LINE}")
      done <<<"${AVAILABLE_TEXT}"
    fi

    if [[ ${#AVAILABLE[@]} -lt ${numNodes} ]]; then
      echo "Insufficient resources available to allocate ${numNodes} ${namePrefix}" 1>&2
      exit 1
    fi

    declare node
    declare AI=0
    for node in "${nodes[@]}"; do
      IFS=$'\x1f' read -r _ IP <<<"${AVAILABLE[${AI}]}"
      colo_node_requisition "${IP}" "${node}" >/dev/null
      AI=$((AI+1))
    done
  else
    declare RES_MACH node
    declare RI=0
    declare NI=0
    while [[ ${NI} -lt ${numNodes} && ${RI} -lt ${COLO_RES_N} ]]; do
      node="${nodes[${NI}]}"
      RES_MACH="${COLO_RES_MACHINE[${RI}]}"
      IP="${COLO_RES_IP[${RI}]}"
      if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then
        if colo_node_requisition "${IP}" "${node}" "${sshPrivateKey}" "${optionalPreemptible}" >/dev/null; then
          NI=$((NI+1))
        fi
      fi
      RI=$((RI+1))
    done
  fi
}

#
# cloud_DeleteInstances
#
# Deletes all the instances listed in the `instances` array
#
cloud_DeleteInstances() {
  declare forceDelete="${1}"
  declare _ IP _ _
  for instance in "${instances[@]}"; do
    IFS=':' read -r _ IP _ _ <<< "${instance}"
    colo_node_free "${IP}" "${forceDelete}" >/dev/null
  done
}

#
# cloud_WaitForInstanceReady [instanceName] [instanceIp] [instanceZone] [timeout]
#
# Return once the newly created VM instance is responding.  This function is cloud-provider specific.
#
cloud_WaitForInstanceReady() {
  #declare instanceName="${1}" # unused
  #declare instanceIp="${2}" # unused
  #declare timeout="${4}" # unused

  true
}

#
# cloud_FetchFile [instanceName] [publicIp] [remoteFile] [localFile]
#
# Fetch a file from the given instance.  This function uses a cloud-specific
# mechanism to fetch the file
#
cloud_FetchFile() {
  #declare instanceName="${1}" # unused
  declare publicIp="${2}"
  declare remoteFile="${3}"
  declare localFile="${4}"
  #declare zone="${5}" # unused
  scp \
    -o "StrictHostKeyChecking=no" \
    -o "UserKnownHostsFile=/dev/null" \
    -o "User=solana" \
    -o "LogLevel=ERROR" \
    -F /dev/null \
    "solana@${publicIp}:${remoteFile}" "${localFile}"
}

cloud_StatusAll() {
  declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE
  if ! ${COLO_PARALLELIZE}; then
    colo_load_resources
    colo_load_availability false
  fi
  for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do
    IFS=$'\x1f' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE <<<"${AVAIL}"
    printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s preemptible=%s\n" "${HOST_NAME}" "${IP}" "${PRIV_IP}" "${STATUS}" "${LOCK_USER}" "${ZONE}" "${INSTNAME}" "${PREEMPTIBLE}"
  done
}