talos-bootstrap

#!/bin/sh
#
# Copyright 2023 Andrei Kvapil. All rights reserved.
# SPDX-License-Identifier: Apache2.0
#
# This code should (try to) follow Google's Shell Style Guide
# - https://google.github.io/styleguide/shell.xml
#

node=
OP=
while [ $# -gt 0 ]; do
  key="$1"
  case $key in
  -n | --node)
    node="$2"
    shift
    shift
    ;;
  -*)
    echo "flag provided but not defined: ${1}" >&2
    exit 1
    ;;
  *)
    if [ -z "$OP" ]; then
      OP="${1}"
      shift
    else
      echo "exactly one action required" >&2
      exit 1
    fi
    ;;
  esac
done

case "$OP" in
  install)
    OPTS="-i"
    ;;
  upgrade|dashboard|reboot|shutdown|reset)
    OPTS="--talosconfig=talosconfig"
    ;;
  *)
    printf "USAGE:\n\t%s\n" "talos-bootstrap ACTION [OPTIONS]"
    printf "\nACTIONS:\n"
    printf "\t%s\t\t%s\n" "help" "Show this help message."
    printf "\t%s\t\t%s\n" "install" "Setup a node for a new or existing cluster."
    printf "\t%s\t\t%s\n" "upgrade" "Upgrade a node in an existing cluster."
    printf "\t%s\t\t%s\n" "reset" "Reset and remove a node from an existing cluster."
    printf "\t%s\t\t%s\n" "reboot" "Reboot a node."
    printf "\t%s\t%s\n" "shutdown" "Shutdown a node."
    printf "\t%s\t%s\n" "dashboard" "Open dashboard for a node."
    printf "\nOPTIONS:\n"
    printf "\t%s <%s>\t%s\n" "-n, --node" "address" "Node address"
    exit 1
    ;;
esac

# Version check
command -V dialog || exit 1
command -V talosctl || exit 1
command -V awk || exit 1
command -V nmap || exit 1

VERSION_REQUIRED=v1.8
if [ $(echo "${VERSION_REQUIRED}\n$(talosctl version --client | awk '$1 == "Tag:" {print $2}')" | sort -V | tail -n 1) = "${VERSION_REQUIRED}" ]; then
  echo "talos-bootstrap requires talosctl version ${VERSION_REQUIRED} or higher. Please check 'talosctl version --client'" >&2
  exit 1
fi

# Load cluster configuration
if [ -f cluster.conf ]; then
  for key in BOOTSTRAP_ETCD CLUSTER_NAME KUBERNETES_API_ENDPOINT VIP_ADDRESS; do
    val=$(awk "/^${key}=/ "'{sub(/^[^=]+="?/, ""); sub(/"$/, ""); print}' cluster.conf)
    export "CONFIG_${key}=${val}"
  done
fi

if [ "$OP" != install ]; then
  if [ ! -f cluster.conf ] || [ ! -f secrets.yaml ]; then
    echo "Error: ./cluster.conf and ./secrets.yaml are required for $OP operation" >&2
    exit 1
  fi
  # Generate talosconfig
  talosctl gen config "$CONFIG_CLUSTER_NAME" "$CONFIG_KUBERNETES_API_ENDPOINT" --with-secrets=secrets.yaml --force -t talosconfig || exit $?
fi

# Screen: Enter cluster name
if [ -n "${CONFIG_CLUSTER_NAME}" ]; then
  cluster_name="${CONFIG_CLUSTER_NAME}"
else
  should_bootstrap=1
  default_cluster_name="$(basename "${PWD}")"
  if [ -z "${default_cluster_name}" ] || [ "${default_cluster_name}" = "/" ]; then
    default_cluster_name=talos
  fi
  cluster_name=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter cluster name:" 8 40 "${default_cluster_name}" 3>&1 1>&2 2>&3) || exit 0
fi

if [ -n "${node}" ]; then
  talosctl -e "${node}" -n "${node}" get machinestatus ${OPTS} >/dev/null || exit $?
else
  # Screen: Enter networks to scan
  default_scan_networks=$(ip -o route | awk '$3 !~ /^(docker|cni)/ && $2 == "dev" {print $1}' | awk '$1=$1' RS=" " OFS=" ")
  scan_networks=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter networks to scan:" 8 80 "${default_scan_networks}" 3>&1 1>&2 2>&3) || exit 0
  scan_networks=$(echo "${scan_networks}" | awk -F, '{$1=$1}1' OFS=' ')

  node_list_file=$(mktemp)

  # Screen: Seatching Talos nodes
  {
    printf "%s\nXXX\n%s\nXXX\n" "10" "Searching Talos nodes in ${scan_networks}..."
    candidate_nodes=$(nmap -Pn -n -p 50000 ${scan_networks} -vv | awk '/Discovered open port/ {print $NF}')

    #echo found:
    #printf "  - %s\n" $candidate_nodes

    if [ "$OP" != install ]; then
      printf "%s\nXXX\n%s\nXXX\n" "40" "Filtering nodes in the cluster..."
    else
      printf "%s\nXXX\n%s\nXXX\n" "40" "Filtering nodes in maintenance mode..."
    fi
    nodes=
    for node in ${candidate_nodes}; do
      if talosctl -e "${node}" -n "${node}" get machinestatus ${OPTS} >/dev/null 2>/dev/null; then
        nodes="${nodes} ${node}"
      fi
    done

    #echo filtered:
    #printf "  - %s\n" $nodes

    printf "%s\nXXX\n%s\nXXX\n" "60" "Collecting information about the nodes..."
    seen=
    for node in ${nodes}; do
      mac=$(talosctl -e "${node}" -n "${node}" get hardwareaddresses.net.talos.dev first ${OPTS} -o jsonpath='{.spec.hardwareAddr}')
      case " ${seen} " in *" ${mac} "*) continue ;; esac # remove duplicated nodes
      seen="${seen} ${mac}"
      name="${node}"
      hostname=$(talosctl -e "${node}" -n "${node}" get hostname ${OPTS} -o jsonpath='{.spec.hostname}')
      if [ -n "${hostname}" ]; then
        name="${name} (${hostname})"
      fi
      manufacturer=$(talosctl -e "${node}" -n "${node}" get cpu ${OPTS} -o jsonpath='{.spec.manufacturer}' | head -n1)
      cpu=$(talosctl -e "${node}" -n "${node}" get cpu ${OPTS} -o jsonpath='{.spec.threadCount}' | awk '{sum+=$1;} END{print sum "-core";}')
      ram=$(talosctl -e "${node}" -n "${node}" get ram -o json ${OPTS} | awk '/"sizeMiB":/ {sub(",", ""); sum+=$2} END{print sum/1024 "GB"}')
      disks=$(talosctl -e "${node}" -n "${node}" get disks ${OPTS} | awk 'sub(/^.*Disk +/, "", $0) {print $0}' | awk '$1 !~ "^(zd|drbd|loop|sr)" {print $1 ":" $3$4}' | awk '$1=$1' RS="," OFS=",")
      echo "\"${name}\"" "\"${mac}, ${cpu} ${manufacturer:-CPU}, RAM: ${ram}, Disks: [${disks}]\"" >> "${node_list_file}"
    done

  } | dialog --keep-tite --title talos-bootstrap --gauge "Please wait" 10 70 0 3>&1 1>&2 2>&3 || exit 0

  node_list=$(cat "${node_list_file}")

  if [ -z "${node_list}" ]; then
    if [ "$OP" != install ]; then
      message="No Talos nodes found in the cluster!"
    else
      message="No Talos nodes in maintenance mode found!"
    fi
    dialog --keep-tite --title talos-bootstrap --msgbox "$message

  Searched networks: ${scan_networks}" 10 60
    exit 1
  fi

  # Screen: Node list
  node=$(echo "${node_list}" | dialog --keep-tite --title talos-bootstrap --menu "Select node to $OP" 0 0 0 --file /dev/stdin 3>&1 1>&2 2>&3) || exit 0
  # cut hostname
  node=$(echo "${node}" | awk '{print $1}')
fi

# --- Management flows beginning

# Run dashboard flow
if [ "$OP" = dashboard ]; then
  talosctl -e "${node}" -n "${node}" ${OPTS} dashboard
  exit $?
fi
# Run reset flow
if [ "$OP" = reset ]; then
  reset_option=$(dialog --keep-tite --title talos-bootstrap --menu "Select reset option" 0 0 0 \
    1 "graceful reset and reboot" \
    2 "graceful reset and shutdown" \
    3 "force reset and reboot" \
    4 "force reset and shutdown" 3>&1 1>&2 2>&3) || exit 0
  reset_opt=
  case ${reset_option} in
     1) reset_opt="--graceful=true --reboot=true" ;;
     2) reset_opt="--graceful=true --reboot=false" ;;
     3) reset_opt="--graceful=false --reboot=true" ;;
     4) reset_opt="--graceful=false --reboot=false" ;;
  esac
  wipe_mode=$(dialog --keep-tite --title talos-bootstrap --menu "Select wipe option" 15 60 4 \
    "all" "Wipe all disks" \
    "system-disk" "Wipe system disk" \
    "user-disks" "Wipe user disks" 3>&1 1>&2 2>&3) || exit 0
  talosctl -e "${node}" -n "${node}" ${OPTS} ${reset_opt} --wipe-mode="${wipe_mode}" reset
  exit $?
fi
# Run reboot flow
if [ "$OP" = reboot ]; then
  reboot_mode=$(dialog --keep-tite --title talos-bootstrap --menu "Select reboot option" 15 60 4 \
    "default" "Default mode" \
    "powercycle" "Skips kexec" 3>&1 1>&2 2>&3) || exit 0
  talosctl -e "${node}" -n "${node}" ${OPTS} ${reboot_opt} --mode="${reboot_mode}" reboot
  exit $?
fi
# Run shutdown flow
if [ "$OP" = shutdown ]; then
  shutdown_option=$(dialog --keep-tite --title talos-bootstrap --menu "Select shutdown option" 15 60 4 \
    "1" "Default mode" \
    "2" "Force a node to shutdown without a cordon/drain" 3>&1 1>&2 2>&3) || exit 0
  case ${shutdown_option} in
     1) shutdown_opt="--force=false" ;;
     2) shutdown_opt="--force=true" ;;
  esac
  talosctl -e "${node}" -n "${node}" ${OPTS} ${shutdown_opt} shutdown
  exit $?
fi

# --- Management flows end

# Screen: Select role
default_role=$(talosctl -e "${node}" -n "${node}" get machinetype machine-type ${OPTS} -o jsonpath={.spec})
role=$(dialog --keep-tite --title talos-bootstrap --default-item "${default_role}" --menu "Select role" 0 0 0 \
  "controlplane" "Responsible for running cluster components" \
  "worker" "Responsible for running workloads" 3>&1 1>&2 2>&3) || exit 0

# Screen: Select hostname
default_hostname=$(talosctl -e "${node}" -n "${node}" get hostname ${OPTS} -o jsonpath='{.spec.hostname}')
hostname=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter hostname:" 8 40 "${default_hostname}" 3>&1 1>&2 2>&3) || exit 0

# Screen: Select disk to install
disks_list=$(talosctl -e "${node}" -n "${node}" get disks ${OPTS} | awk 'sub(/^.*Disk +/, "", $0) {print $0}' | awk '$1 !~ "^(zd|drbd|loop|sr)" {f=$1;$2=$1=""; sub(/^ +/, "", $0); print "\"" f "\" \"" $0 "\""}')
default_disk=$(talosctl get systemdisk -n ${node} -e ${node} system-disk ${OPTS} -o jsonpath={.spec.diskID})
disk=$(echo "${disks_list}" | dialog --keep-tite --title talos-bootstrap --default-item "${default_disk}" --menu "Select disk to install" 0 0 0 --file /dev/stdin 3>&1 1>&2 2>&3) || exit 0

# Screen: Select interface
link_list=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} | awk -F'  +' 'NR>1 && $4 ~ /^(ID|eno|eth|enp|enx|ens|bond)/ {print $4 "|" $(NF-2)}')
address_list=$(talosctl -e "${node}" -n "${node}" get addresses ${OPTS} | awk 'NR>1 {print $NF " " $(NF-1)}')

default_addresses=$(talosctl -e "${node}" -n "${node}" get nodeaddress default ${OPTS} -o jsonpath='{.spec.addresses[*]}' | awk '$1=$1' RS=, OFS=,)
default_interface=$(talosctl -e "${node}" -n "${node}" get addresses ${OPTS} | awk "BEGIN { split(\"${default_addresses}\", arr, \",\"); } {for (i in arr) if (\$(NF-1) == arr[i]) print \$NF}" | awk 'NR==1')

interface_list=$(
  for link_mac in ${link_list}; do
    link="${link_mac%%|*}"
    mac="${link_mac#*|}"
    ips=$(echo "${address_list}" | awk "\$1 == \"${link}\" {print \$2}" | awk '$1=$1' RS=, OFS=,)
    details="${mac}"
    if [ -n "${ips}" ]; then
      details="${mac} (${ips})"
    fi
    echo "${link} \"${details}\""
  done
)

interface=$(echo "${interface_list}" | dialog --keep-tite --title talos-bootstrap --default-item "${default_interface}" --menu "Select interface:" 0 0 0 --file /dev/stdin 3>&1 1>&2 2>&3) || exit 0

interface_kind=$(talosctl -e "${node}" -n "${node}" get link "${interface}" ${OPTS} -o jsonpath='{.spec.kind}') || exit $?

if [ "$interface_kind" = bond ]; then
  # Screen: Select slave interfaces for bonding
  interface_index=$(talosctl -e "${node}" -n "${node}" get link "${interface}" ${OPTS} -o jsonpath='{.spec.index}') || exit $?
  default_slave_interfaces=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.metadata.id}{.spec}' | \
    awk '{sub("}", "\n"); print}' ORS=""| awk -F '{' '/masterIndex/ {sub("{.*masterIndex", ""); print}' | awk '{sub(",.*", ""); sub("\":", ""); if ($2 == '"${interface_index}"') print $1}')
  slave_interfaces_list=$(echo "$interface_list" | awk '$1 !~ /^bond/' | while read link details; do
    echo "$link" "$details" "$(echo "$default_slave_interfaces" | awk "\$1 == \"$link\" {found=1; exit} END {print found ? \"on\" : \"off\"}")"
  done)
  slave_interfaces=$(echo "${slave_interfaces_list}" | dialog --keep-tite --title talos-bootstrap --checklist "Select interfaces for bonding:" 0 0 0 --file /dev/stdin 3>&1 1>&2 2>&3) || exit 0
  slave_interfaces=$(echo "${slave_interfaces}" | awk '{$1=$1}1' OFS=",")

  if [ -z "$slave_interfaces" ]; then
    echo "Error: at least one slave must be specified for bonding interface!" >&2
    exit 1
  fi

  # Screen: Select bond mode
  default_bond_mode=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.spec.bondMaster.mode}' "${interface}")
  bond_mode=$(dialog --keep-tite --title talos-bootstrap --default-item "${default_bond_mode}" --menu "Select bond mode" 0 0 0 \
    "balance-rr" "Balance Round-Robin" \
    "active-backup" "Active-Backup" \
    "balance-xor" "Balance-xor" \
    "broadcast" "Broadcast" \
    "802.3ad" "Dynamic link aggregation" \
    "balance-tlb" "Adaptive transmit load balancing" \
    "balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3) || exit 0

  if [ "${bond_mode}" = "802.3ad" ]; then
    # Screen: Select bond lacp rate
    default_bond_lacp_rate=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.spec.bondMaster.lacpRate}' "${interface}")
    bond_lacp_rate=$(dialog --keep-tite --title talos-bootstrap --default-item "${default_bond_lacp_rate}" --menu "Select bond lacp rate" 0 0 0 \
      "slow" "transmit LACPDUs every 30 seconds" \
      "fast" "transmit LACPDUs every 1 second" 3>&1 1>&2 2>&3) || exit 0
  fi

  # Screen: Select bond xmitHashPolicy
  default_bond_xmit_hash_policy=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.spec.bondMaster.xmitHashPolicy}' "${interface}")
  bond_lacp_xmit_hash_policy=$(dialog --keep-tite --title talos-bootstrap --default-item "${default_bond_xmit_hash_policy}" --menu "Select bond xmit_hash_policy" 0 0 0 \
    "layer2" "uses XOR of hardware MAC addresses and packet type ID field to generate the hash." \
    "layer2+3" "uses a combination of layer2 and layer3 protocol information to generate the hash." \
    "layer3+4" "uses upper layer protocol information, when available, to generate the hash." 3>&1 1>&2 2>&3) || exit 0

  # Screen: Enter bond miimon
  default_bond_miimon=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.spec.bondMaster.miimon}' "${interface}")
  bond_miimon=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter bond miimon:" 8 40 "${default_bond_miimon:-0}" 3>&1 1>&2 2>&3) || exit 0

  # Screen: Enter bond updelay
  default_bond_updelay=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.spec.bondMaster.updelay}' "${interface}")
  bond_updelay=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter bond updelay:" 8 40 "${default_bond_updelay:-0}" 3>&1 1>&2 2>&3) || exit 0

  # Screen: Enter bond downdelay
  default_bond_downdelay=$(talosctl -e "${node}" -n "${node}" get link ${OPTS} -o jsonpath='{.spec.bondMaster.downdelay}' "${interface}")
  bond_downdelay=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter bond downdelay:" 8 40 "${default_bond_downdelay:-0}" 3>&1 1>&2 2>&3) || exit 0
fi

# Screen: Configure networks
default_addresses=$(talosctl -e "${node}" -n "${node}" get nodeaddress default ${OPTS} -o jsonpath='{.spec.addresses[*]}' | awk '$1=$1' RS=, OFS=,)
addresses=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter addresses:" 8 40 "${default_addresses}" 3>&1 1>&2 2>&3) || exit 0
addresses=$(echo "${addresses}" | awk '{$1=$1}1' OFS=",")
# select first address
address=$(echo "${addresses}" | awk -F/ '{print $1}')

# Screen: Configure default gateway
default_gateway=$(talosctl -e "${node}" -n "${node}" get routes ${OPTS} -o json | awk '{gsub(/[{}]/, "\n"); printf $0}' | awk -F', *' '$0 ~ /"dst": *""/ && $0 !~ /"gateway": *""/ {for(i=1;i<=NF;i++) if ($i ~ /"gateway":/) {split($i,a,"\""); print a[4]; exit}}')
gateway=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter gateway:" 8 40 "${default_gateway}" 3>&1 1>&2 2>&3) || exit 0

# Screen: Configure DNS servers
default_dns_servers=$(talosctl -e "${node}" -n "${node}" get resolvers resolvers ${OPTS} -o jsonpath='{.spec.dnsServers[*]}' | awk '$1=$1' RS=" " OFS=" ")
dns_servers=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter DNS servers:" 8 80 "${default_dns_servers}" 3>&1 1>&2 2>&3) || exit 0
dns_servers=$(echo "${dns_servers}" | awk '{$1=$1}1' OFS=",")

# Screen: Configure VIP
vip_address="${CONFIG_VIP_ADDRESS}"
if [ "${role}" = controlplane ]; then
  vip_address=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter virtual shared IP address, or leave blank to skip:" 8 40 "${vip_address}" 3>&1 1>&2 2>&3) || exit 0
fi

# Screen: Configure Kubernetes endpoint
default_k8s_endpoint="${CONFIG_KUBERNETES_API_ENDPOINT:-https://${vip_address:-${address}}:6443}"
k8s_endpoint=$(dialog --keep-tite --title talos-bootstrap --inputbox "Enter Kubernetes endpoint:" 8 40 "${default_k8s_endpoint}" 3>&1 1>&2 2>&3) || exit 0

# Generating config patch...
machine_config=$(cat <<EOT
machine:
  type: ${role}
  install:
    disk: /dev/${disk}
  network:
    hostname: ${hostname}
    nameservers: [${dns_servers}]
    interfaces:
    - interface: ${interface}
EOT
  if [ "${role}" = controlplane ] && [ -n "${vip_address}" ]; then
    cat <<EOT
      vip:
        ip: ${vip_address}
EOT
  fi
  if [ "${interface_kind}" = bond ]; then
  cat <<EOT
      bond:
        mode: ${bond_mode}
EOT
    if [ "${bond_mode}" = "802.3ad" ]; then
      cat <<EOT
        lacpRate: ${bond_lacp_rate}
EOT
    fi
  cat <<EOT
        xmitHashPolicy: ${bond_lacp_xmit_hash_policy}
        miimon: ${bond_miimon}
        updelay: ${bond_updelay}
        downdelay: ${bond_downdelay}
        interfaces: [$slave_interfaces]
EOT
  fi
  cat <<EOT
      addresses: [${addresses}]
      routes:
        - network: 0.0.0.0/0
          gateway: ${gateway}
EOT
)

# Screen: Confirm configuration
file=$(mktemp)
# shellcheck disable=SC2064
trap "rm -f '${file}'" EXIT
echo "Please confirm your configuration:

${machine_config}" > "${file}"

dialog --keep-tite --title talos-bootstrap --ok-label "OK" --extra-button --extra-label "Cancel" --textbox "${file}" 0 0 || exit 0
rm -f "${file}"
trap '' EXIT

# Generating configuration...
if [ ! -f secrets.yaml ]; then
  talosctl gen secrets || exit $?
fi
patches=""
if [ -f patch.yaml ]; then
  patches="${patches} --config-patch=@patch.yaml"
fi
if [ -f patch-controlplane.yaml ]; then
  patches="${patches} --config-patch-control-plane=@patch-controlplane.yaml"
fi
if [ -f patch-worker.yaml ]; then
  patches="${patches} --config-patch-worker=@patch-worker.yaml"
fi
talosctl gen config "${cluster_name}" "${k8s_endpoint}" --with-secrets=secrets.yaml ${patches} --config-patch="${machine_config}" --force || exit $?

# Screen: Select upgrade option
if [ "$OP" = upgrade ]; then
  upgrade_option=$(dialog --keep-tite --title talos-bootstrap --menu "Select upgrade option" 0 0 0 \
    1 "apply config on live and skip upgrade" \
    2 "apply config and perform upgrade (mode: default)" \
    3 "apply config and perform upgrade (mode: powercycle)" \
    4 "apply config and perform upgrade with force (mode: default)" \
    5 "apply config and perform upgrade with force (mode: powercycle)" \
    6 "apply config and schedule upgrade to perform it after a reboot" 3>&1 1>&2 2>&3) || exit 0

  should_upgrade=1
  upgrade_opt=""
  case ${upgrade_option} in
     1) should_upgrade=0 ;;
     2) upgrade_opt="--reboot-mode default" ;;
     3) upgrade_opt="--reboot-mode powercycle" ;;
     4) upgrade_opt="--force --reboot-mode default" ;;
     5) upgrade_opt="--force --reboot-mode powercycle" ;;
     6) upgrade_opt="--stage" ;;
  esac
fi

# Swap IP addresses
bootstrap_ip=${node}
node="${address}"

# Try applying config before install
talosctl apply -e "${bootstrap_ip}" -n "${bootstrap_ip}" -f "${role}.yaml" ${OPTS} --dry-run || exit $?

# Screen: Installation process
{
  printf "%s\nXXX\n%s\nXXX\n" "1" "Applying configuration..."
  talosctl apply -e "${bootstrap_ip}" -n "${bootstrap_ip}" -f "${role}.yaml" ${OPTS} >/dev/null 2>&1

  if [ "$OP" = upgrade ] && [ "${should_upgrade}" = 1 ]; then
    image=$(talosctl -e "${node}" -n "${node}" get machineconfig -o jsonpath="{.spec.machine.install.image}" ${OPTS})
    printf "%s\nXXX\n%s\n%s\nXXX\n" "10" "Scheduling upgrade..." "(image: $image)"
    talosctl -e "${node}" -n "${node}" upgrade ${upgrade_opt} -i "${image}" --wait=false ${OPTS} || exit $?
  fi

  if [ "$OP" = upgrade ]; then
    printf "%s\nXXX\n%s\nXXX\n" "10" "Installing..."
  else
    printf "%s\nXXX\n%s\n%s\nXXX\n" "20" "Upgrading..." "(this will take a while)"
  fi

  old_is_up=1
  old_is_pingable=1
  new_is_pingable=0
  new_is_up=0
  until [ "${new_is_up}" = 1 ] ; do
    sleep 0.2
    if [ "${new_is_pingable}" = 0 ]; then
      if [ "${old_is_up}" = 1 ]; then
        status=$(timeout 3 talosctl --talosconfig=talosconfig -e "${node}" -n "${node}" get machinestatus ${OPTS} -o jsonpath={.spec.stage} 2>/dev/null)
        if [ $? = 124 ]; then
          old_is_up=0
        fi
        if [ "$status" = upgrading ] || [ "$status" = rebooting ]; then
          continue
        fi
      else
        if ! ping -W1 -c1 "${node}" >/dev/null 2>&1; then
          if ! ping -W1 -c1 "${node}" >/dev/null 2>&1; then # TODO dirty hack
          old_is_pingable=0
          fi
        fi
      fi
    fi

    if [ "${old_is_pingable}" = 0 ]; then
      if ping -W1 -c1 "${node}" >/dev/null 2>&1; then
        new_is_pingable=1
      fi
    fi

    if timeout 3 talosctl --talosconfig=talosconfig -e "${node}" -n "${node}" get machinestatus >/dev/null 2>&1; then
      new_is_up=1
    fi

    case ${old_is_up}${old_is_pingable}${new_is_pingable} in
      110) printf "%s\nXXX\n%s\n%s\nXXX\n" "20" "Installing..." "(port is open at ${node})" ;;
      010) printf "%s\nXXX\n%s\n%s\nXXX\n" "50" "Rebooting..." "(node is pingable at ${node})" ;;
      000) printf "%s\nXXX\n%s\n%s\nXXX\n" "70" "Rebooting..." "(node is not pingable at ${node})" ;;
      001) printf "%s\nXXX\n%s\n%s\nXXX\n" "80" "Rebooting..." "(node is pingable again at ${node})" ;;
    esac
  done
} | dialog --keep-tite --title talos-bootstrap --gauge "Please wait" 10 70 0 3>&1 1>&2 2>&3 || exit 1

# Screen: Should we bootstrap etcd?
if [ "${CONFIG_BOOTSTRAP_ETCD}" = true ]; then
  should_bootstrap=1
elif [ "${CONFIG_BOOTSTRAP_ETCD}" = false ]; then
  should_bootstrap=0
else
  dialog --stdout --keep-tite --title talos-bootstrap \
    --yesno "It seems this is a first node in a cluster. Should we bootstrap etcd on it?" 7 60 3>&1 1>&2 2>&3
  response=$?
  case ${response} in
     0) should_bootstrap=1 ;;
     1) should_bootstrap=0 ;;
     *) exit 0 ;;
  esac
fi

if [ "${should_bootstrap}" = 1 ]; then
  count=0
  max_retries=20
  while ! nmap -Pn ${vip_address} -p 50000 | grep -q 'open' && [ ${count} -lt ${max_retries} ]; do
    count=$((count+1))
    sleep 5
    talosctl --talosconfig=talosconfig -e "${node}" -n "${node}" bootstrap
  done
  if [ ${count} -ge ${max_retries} ]; then
    dialog --keep-tite --title "talos-bootstrap" --msgbox "Port 50000 closed, ETCD is not installed!" 5 26
    exit 1
  fi
fi


# Saving cluster configuration
cat > cluster.conf <<EOT
BOOTSTRAP_ETCD=false
CLUSTER_NAME="${cluster_name}"
KUBERNETES_API_ENDPOINT="${k8s_endpoint}"
VIP_ADDRESS="${vip_address}"
EOT

# Save kubeconfig
if [ ! -f kubeconfig ]; then
  KUBECONFIG=kubeconfig talosctl --talosconfig=talosconfig -e "${node}" -n "${node}" kubeconfig -f
fi

# Screen: Complete installation
dialog --keep-tite --title talos-bootstrap --msgbox "Installation finished!" 5 26