Newer
Older
openstack-caracal-ipv4 / scripts / reenroll-hosts.sh
#!/usr/bin/env bash
# scripts/reenroll-hosts.sh
#
# Re-enroll the four OpenStack KVM hosts into MAAS as virsh-power machines after
# they were deleted/decomposed from MAAS. The libvirt domains must still EXIST --
# this does NOT recreate VMs, only their MAAS machine objects. MAAS auto-commissions
# on create, so each host goes New -> Commissioning -> Ready, PXE-booting off its
# 2_metal boot NIC.
#
# Modes:
#   (default)  create any of the four that are MISSING, then poll all four to Ready
#   --check    read-only: report current status of openstack0-3 (no mutation)
#
# Discover-assert-pin: never creates a host that already exists. Idempotent --
# a re-run after a partial run only creates the still-missing hosts.
#
# The libvirt SSH password is READ INTERACTIVELY (never a CLI arg, never echoed,
# never logged, never in the repo). SECURITY NOTE: that credential was exposed in
# plaintext on 2026-06-26 (`maas machine power-parameters` echoes power_pass) --
# ROTATE the libvirt SSH credential after this rebuild.
#
# Pinned values come from scripts/lib-hosts.sh (host identity) + lib-net.sh.
# Exit codes: 0 all four Ready | 1 fatal | 2 warning/timeout

set -euo pipefail
shopt -s inherit_errexit 2>/dev/null || true
IFS=$'\n\t'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/lib-net.sh
. "$SCRIPT_DIR/lib-net.sh"
# shellcheck source=scripts/lib-hosts.sh
. "$SCRIPT_DIR/lib-hosts.sh"

MAAS_PROFILE="${MAAS_PROFILE:-admin}"
READY_TIMEOUT="${READY_TIMEOUT:-1200}"   # seconds (~20 min)

FATAL=0; WARN=0
fail() { echo "FAIL: $*" >&2; FATAL=$((FATAL+1)); }
warn() { echo "WARN: $*" >&2; WARN=$((WARN+1)); }
pass() { echo "PASS: $*"; }
note() { echo "NOTE: $*"; }
hdr()  { echo; echo "=== $* ==="; }
finish() {
  echo; echo "Summary: ${FATAL} fatal, ${WARN} warning"
  if   [ "$FATAL" -gt 0 ]; then exit 1
  elif [ "$WARN"  -gt 0 ]; then exit 2
  fi
  exit 0
}

need_jq || exit 1

MODE="create"
[ "${1:-}" = "--check" ] && MODE="check"

# mread <sysid> -> machine JSON or empty (set -e safe)
mread() { maas "$MAAS_PROFILE" machine read "$1" 2>/dev/null || true; }

# report: read-only status of the four hosts (system_id resolved live by hostname)
report() {
  local hn sid j st pwr fab
  for hn in "${HOSTS[@]}"; do
    sid="$(host_sysid "$hn" || true)"
    if [ -z "$sid" ]; then printf "  %-11s NOT-ENROLLED\n" "$hn"; continue; fi
    j="$(mread "$sid")"
    st="$(printf '%s' "$j"  | jq -r '.status_name // "?"' 2>/dev/null || echo '?')"
    pwr="$(printf '%s' "$j" | jq -r '.power_state // "?"' 2>/dev/null || echo '?')"
    fab="$(maas "$MAAS_PROFILE" interfaces read "$sid" 2>/dev/null \
            | jq -r --arg m "${HOST_BOOT_MAC[$hn]}" '.[]|select(.mac_address==$m)|.vlan.fabric' \
            | head -1 || true)"
    printf "  %-11s sid=%-8s %-12s power=%-4s bootnic_fabric=%s\n" \
      "$hn" "$sid" "$st" "$pwr" "${fab:-?}"
  done
}

hdr "Current host status (by hostname; system_id resolved live)"
report

if [ "$MODE" = "check" ]; then
  note "read-only check mode; no changes made"
  finish
fi

# ---------------------------------------------------------------------------
# CREATE mode: build the MISSING set (discover-assert-pin); create only those.
MISSING=()
for hn in "${HOSTS[@]}"; do
  [ -z "$(host_sysid "$hn" || true)" ] && MISSING+=("$hn")
done

if [ "${#MISSING[@]}" -eq 0 ]; then
  pass "all four hosts already enrolled; nothing to create"
else
  note "to create: ${MISSING[*]}"
  # Secret read WITHOUT echo. Never placed on a command line; unset after use.
  read -rsp "libvirt SSH password for ${VIRSH_POWER_ADDRESS}: " PPASS; echo
  [ -n "${PPASS:-}" ] || { fail "empty password; aborting (nothing created)"; finish; }
  for hn in "${MISSING[@]}"; do
    echo "  creating $hn (boot mac ${HOST_BOOT_MAC[$hn]})"
    if maas "$MAAS_PROFILE" machines create \
         hostname="$hn" \
         architecture="$HOST_ARCH" \
         mac_addresses="${HOST_BOOT_MAC[$hn]}" \
         power_type=virsh \
         power_parameters_power_id="$hn" \
         power_parameters_power_address="$VIRSH_POWER_ADDRESS" \
         power_parameters_power_pass="$PPASS" \
         | jq '{system_id, hostname, status_name, power_type}'; then
      pass "create accepted: $hn"
    else
      fail "create failed: $hn"
    fi
  done
  unset PPASS
fi

# ---------------------------------------------------------------------------
hdr "Waiting for all four Ready (timeout ${READY_TIMEOUT}s)"
deadline=$(( $(date +%s) + READY_TIMEOUT ))
while :; do
  allready=1
  for hn in "${HOSTS[@]}"; do
    sid="$(host_sysid "$hn" || true)"
    st="MISSING"
    [ -n "$sid" ] && st="$(printf '%s' "$(mread "$sid")" | jq -r '.status_name // "?"' 2>/dev/null || echo '?')"
    [ "$st" = "Ready" ] || allready=0
  done
  if [ "$allready" = 1 ]; then pass "all four Ready"; break; fi
  if [ "$(date +%s)" -ge "$deadline" ]; then warn "timeout waiting for Ready"; break; fi
  sleep 20
done

hdr "Final status"
report

# Verify each boot NIC landed on the 2_metal (PXE/admin) fabric.
hdr "Boot-NIC fabric check (expect 2_metal)"
for hn in "${HOSTS[@]}"; do
  sid="$(host_sysid "$hn" || true)"
  [ -n "$sid" ] || { fail "$hn not enrolled"; continue; }
  fab="$(maas "$MAAS_PROFILE" interfaces read "$sid" 2>/dev/null \
          | jq -r --arg m "${HOST_BOOT_MAC[$hn]}" '.[]|select(.mac_address==$m)|.vlan.fabric' \
          | head -1 || true)"
  if [ "$fab" = "2_metal" ]; then pass "$hn boot NIC on 2_metal"; else fail "$hn boot NIC on '${fab:-?}' (want 2_metal)"; fi
done

note "next: re-tag '${HOST_TAG}' on all four, then the Strategy-B interface carve"
finish