Newer
Older
openstack-caracal-dc-dc / scripts / pre-flight-checks.sh
#!/usr/bin/env bash
# scripts/pre-flight-checks.sh
#
# Pre-deploy sanity gate. Read-only; no state changes. Run from inside the repo
# (or set REPO=) BEFORE 'juju add-model' / 'juju deploy'. Surfaces issues that
# would cause the deploy to fail or mis-bind during settle.
#
# DC-AWARE since 2026-07-29 (F5 / readiness item 3.7). Set DC to pick the datacenter;
# it defaults to vr1-dc0, and scripts/preflight.sh EXPORTS its own validated selection
# so P4 always measures the same DC the rest of preflight did.
#   DC=vr1-dc1 bash scripts/pre-flight-checks.sh
# Before this, every check below was frozen on VR0: it probed hosts openstack0..3 that
# exist in no VR1 DC, asserted VR0's VLAN-103 metal fabric that D-133 abolished for VR1,
# and read a single unscoped octavia-pki overlay. On a VR1 DC it produced fatals that
# described the CHECKER, not the cloud.
#
# Covers (every item resolved for the SELECTED DC):
#   - repo HEAD / cleanliness (informational)
#   - per-DC octavia-pki overlay sanity (5 keys + ASCII; no key material printed)
#   - VIP validator over the per-DC overlay (provider/admin/internal columns, aligned,
#     .50-.99; a v4 triple or an R2 dual-family sextet). Deep v6 checking is
#     provider-bundle-check's job.
#   - MAAS six-plane layout resolved BY CIDR (id/vid/gw/dns), incl. the metal-internal
#     tagging invariant: VR0's VID 103, or D-133's untagged carve on a VR1 DC
#   - per-node data/storage NIC links resolved BY CIDR, incl. D-133's flat-carve
#     invariant (VR0: br-internal; VR1: a PHYSICAL interface, no bridge/VLAN stack)
#   - the DC's OpenStack role nodes Ready / power state
#
# NOT covered here (by design):
#   - juju per-model SPACE names  -> scripts/juju-spaces-check.sh (runs AFTER add-model)
#   - YAML / deploy-plan validity  -> 'juju deploy --dry-run' (phase-01 Step 1.2)
#   - OSD secondary-disk blank     -> scripts/osd-blank-check.sh (needs sudo)
#
# Pinned values come from scripts/lib-net.sh (single source of truth).
# Exit codes: 0 all checks pass | 1 fatal (do NOT deploy) | 2 warning (review then decide)

set -euo pipefail
shopt -s inherit_errexit 2>/dev/null || true
IFS=$'\n\t'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/lib-net.sh
. "$SCRIPT_DIR/lib-net.sh"
# shellcheck source=scripts/lib-hosts.sh
. "$SCRIPT_DIR/lib-hosts.sh"

FATAL=0
WARN=0
fail() { echo "FAIL: $*" >&2; FATAL=$((FATAL+1)); }
warn() { echo "WARN: $*" >&2; WARN=$((WARN+1)); }
pass() { echo "PASS: $*"; }
note() { echo "NOTE: $*"; }
hdr()  { echo; echo "=== $* ==="; }

finish() {
  echo
  echo "Summary: ${FATAL} fatal, ${WARN} warning"
  if   [ "$FATAL" -gt 0 ]; then exit 1
  elif [ "$WARN"  -gt 0 ]; then exit 2
  fi
  exit 0
}

MAAS_PROFILE="${MAAS_PROFILE:-admin}"
REPO="${REPO:-$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || dirname "$SCRIPT_DIR")}"
cd "$REPO" || { fail "cannot cd to REPO=$REPO"; finish; }

# ---------------------------------------------------------------------------
# $DC selector (DOCFIX-166 wiring pattern, as used by carve-host-interfaces.sh:48-67).
# Called ONCE, immediately after sourcing both libs and before ANY flat variable is
# read -- that is the libs' own documented contract, and both refuse a second,
# different selection in the same shell.
#
# The default matches scripts/preflight.sh's so the two agree when P4 is run
# standalone. An unknown or RETIRED token is REFUSED by the selectors themselves; we
# turn that into a fatal + summary rather than letting `set -e` kill the run with no
# verdict, because "could not evaluate" must never look like "nothing found".
DC="${DC:-vr1-dc0}"
hdr "DC selection"
if ! lib_net_select_dc "$DC"; then
  fail "lib-net refused DC='$DC' -- nothing below can be measured against an unknown DC"
  finish
fi
if ! lib_hosts_select_dc "$DC"; then
  fail "lib-hosts refused DC='$DC' -- nothing below can be measured against an unknown DC"
  finish
fi
pass "gating DC=$DC (planes ${PLANE_CIDRS[0]} .. ${PLANE_CIDRS[5]}; ${#HOSTS[@]} node(s))"

# Which nodes does the DEPLOY actually bind to? Not all of them.
# D-134 AMENDMENT (2026-07-29) rules octets .4-.49 the per-DC UTILITY band -- "per-DC
# INFRASTRUCTURE the OpenStack nodes consume, NOT OpenStack role nodes" -- which is
# where the D-104 dedicated Juju controller sits at .5. It carries the
# `juju-controller-<dc>` tag and NONE of the role tags precisely so no OpenStack
# constraint can grab it (D-104 amendment), so bundle placement, the six-plane node
# carve and "Ready before deploy" are all statements about the ROLE nodes only.
# scripts/maas-role-tags.sh:70-98 classifies by exactly this band; reusing its rule
# keeps one definition of "role node" in the repo instead of two.
# VR0 is EXEMPT: openstack0..3 sit at .40-.43 under the D-052 index, which predates
# the D-134 bands entirely -- classifying them by band would silently drop the whole
# VR0 fleet into "utility" and leave this gate asserting nothing.
#
# Classified ONCE, here, into NODE_CLASS -- not re-derived per check. CHECK 2 and
# CHECK 4 both need the answer, and a per-check helper emitted the refusal TWICE for
# an unclassifiable node and then printed a "utility-band" note contradicting the
# refusal that had just fired.
D134_UTILITY_LO=4
D134_UTILITY_HI=49
declare -A NODE_CLASS=()
for _h in "${HOSTS[@]}"; do
  _o="${HOST_OCTET[$_h]}"
  case "$DC" in
    vr0-dc0) NODE_CLASS[$_h]="role" ;;
    *)
      if   [ "$_o" -ge "$D134_UTILITY_LO" ] && [ "$_o" -le "$D134_UTILITY_HI" ]; then NODE_CLASS[$_h]="utility"
      elif [ "$_o" -ge 100 ]; then NODE_CLASS[$_h]="role"
      else
        # Neither band. REFUSE rather than guess which side it falls on, and say so
        # once. The node is then asserted on by nothing, which is why this is fatal.
        NODE_CLASS[$_h]="unclassified"
        fail "$_h octet .$_o is in NO ruled D-134 band (utility .$D134_UTILITY_LO-.$D134_UTILITY_HI, role .100+) -- cannot classify, refusing to skip or to assert"
      fi ;;
  esac
done

# system_id resolution differs BY REGION and getting it wrong is what produced eight
# of P4's eleven fatals. lib-hosts.sh:54-57 / :64-67: VR0 hostnames are stable
# (hostname == libvirt domain == power_id) so host_sysid() is correct there; VR1 MAAS
# hostnames are RANDOM and RE-MINTED on every re-enlistment (measured: the 2026-07-21
# pod-delete recovery renamed all 9), so the only stable key is the config-pinned boot
# MAC. `|| true` is deliberate: under `set -e` + inherit_errexit a failing pipeline
# inside $( ) aborts the script instead of reaching the refusal branch below.
sysid_for() {
  local h="$1"
  case "$DC" in
    vr0-dc0) host_sysid "$h" || true ;;
    *)       host_sysid_by_bootmac "${HOST_BOOT_MAC[$h]}" || true ;;
  esac
}

# ---------------------------------------------------------------------------
hdr "Repo (informational)"
note "REPO=$REPO"
note "HEAD: $(git --no-pager log --oneline -1 2>/dev/null || echo '(not a git tree)')"
DIRTY="$( { git status --porcelain 2>/dev/null || true; } | wc -l | tr -d ' ')"
if [ "$DIRTY" = "0" ]; then note "working tree clean"; else warn "working tree has $DIRTY modified file(s)"; fi

# ---------------------------------------------------------------------------
hdr "CHECK 0: per-DC octavia-pki overlay (no key material printed)"
# PER-DC since F1 (2026-07-29). The old fixed `overlays/octavia-pki.yaml` was a single
# unscoped path holding what the D-109 R7 amendment rules must be per-DC CA material
# ("each DC gets its own Octavia CA; no cross-DC amphora root-of-trust"): generating
# dc1's PKI overwrote dc0's, and every existing check asked only *does a file exist*,
# never *whose CA is it*. Naming the file per DC makes the existence check a DC check.
OVL="overlays/${DC}-octavia-pki.yaml"
if [ -f "$OVL" ]; then
  KEYS="$(grep -cE 'lb-mgmt-' "$OVL" || true)"
  if [ "$KEYS" -eq 5 ]; then pass "overlay present with 5 lb-mgmt-* keys"; else fail "overlay has $KEYS lb-mgmt-* keys (want 5)"; fi
  if LC_ALL=C grep -qP '[^\x00-\x7F]' "$OVL"; then fail "overlay contains non-ASCII bytes"; else pass "overlay ASCII clean"; fi
else
  fail "MISSING $OVL (gitignored secret; place it or regenerate via runbook Step 1.0-GEN before deploy)"
fi

# ---------------------------------------------------------------------------
hdr "CHECK 1: bundle VIPs -- v4 triple or R2 dual-family sextet, .${VIP_OCTET_MIN}-.${VIP_OCTET_MAX} (provider/admin/internal)"
if [ ! -f bundle.yaml ]; then
  fail "bundle.yaml not found in $REPO"
else
  # BLOCKER-1 (ruling 3, 2026-07-25): bundle.yaml is now VIP-FREE -- every VIP
  # arrives via a per-DC overlay, dc0 INCLUDED. Reading the base alone would see
  # ZERO VIPs and hard-fail, so this reads the MERGED input. The ruled fix is
  # exactly this, and explicitly NOT bumping the expected count to paper over it.
  #
  # DC-AWARE since 2026-07-29. The previous note here said this check was "still
  # dc0-only -- it never calls lib_net_select_dc, and lib-net.sh UNSETS VIP_PREFIX_*
  # for dc1 by design, so it cannot evaluate dc1 at all". Both halves are now closed:
  # the selector runs at the top of this file, and lib-net's dc1 arm carries the R9
  # group. The overlay read is the SELECTED DC's, so a dc1 run validates dc1's numbers
  # against dc1's bands instead of silently re-validating dc0.
  # An ARRAY, not a space-joined string: under `set -euo pipefail` an unsplit
  # "a b" reaches grep as ONE filename, grep exits 2, pipefail propagates it and
  # the whole gate dies silently mid-check. Measured here, not theorised.
  VIP_OVL="overlays/${DC}-vips.yaml"
  if [ ! -f "$VIP_OVL" ]; then
    # REFUSE, do not fall through. Under ruling 3 bundle.yaml is VIP-FREE, so without
    # the per-DC overlay there is nothing to validate -- and the old code path would
    # have read zero VIPs and reported a count mismatch, which describes the wrong
    # problem. `vr0-dc0` is a legal selector with no overlay in this tree and lands
    # here by design.
    fail "no VIP overlay for DC=$DC ($VIP_OVL). bundle.yaml is VIP-FREE (ruling 3, 2026-07-25), so this DC has NO VIPs to validate -- render it or select a DC that has one"
  else
  VIP_SRC=(bundle.yaml "$VIP_OVL")
  # `|| true` on the grep: it exits 1 when a file has no matches, pipefail propagates
  # that, and under inherit_errexit the command substitution ABORTS the whole script
  # instead of reaching any branch. bundle.yaml is VIP-free, so the zero-match case is
  # the NORMAL one here, not an edge case.
  VIPLINES="$( { grep -hcE '^[[:space:]]+vip:' "${VIP_SRC[@]}" 2>/dev/null || true; } | awk '{s+=$1} END{print s+0}')"
  if [ "$VIPLINES" -eq "$VIP_COUNT_EXPECT" ]; then pass "vip: line count = $VIPLINES (from $VIP_OVL)"; else warn "vip: line count = $VIPLINES (want $VIP_COUNT_EXPECT, from $VIP_OVL)"; fi
  VIPOUT="$(awk -v pp="$VIP_PREFIX_PROVIDER" -v pa="$VIP_PREFIX_ADMIN" -v pi="$VIP_PREFIX_INTERNAL" \
                -v lo="$VIP_OCTET_MIN" -v hi="$VIP_OCTET_MAX" '
    BEGIN{ok=0;bad=0}
    /^[[:space:]]+vip:/{
      v=$0; sub(/^[^"]*"/,"",v); sub(/".*/,"",v); n=split(v,a," ");
      # R2 (RULED 2026-07-27): a vip is the v4 TRIPLE, or a dual-family SEXTET
      # appending the three v6 legs. Anything else is malformed.
      if(n!=3 && n!=6){print "    MALFORMED(not 3 or 6 IPs): " $0; bad++; next}
      # For a sextet the last three must be v6 literals. Their PREFIX and octet-mirror
      # correctness is asserted by provider-bundle-check against the NetBox apex -- this
      # gate deliberately does not duplicate that, it only rejects a v4/v6 mix-up here.
      if(n==6){
        for(i=4;i<=6;i++){ if(index(a[i],":")==0){print "    NOT-IPv6 in v6 leg " i-3 ": " $0; bad++; next6=1} }
        if(next6){next6=0; next}
      }
      if(index(a[1],pp".")!=1){print "    WRONG provider col: " $0; bad++; next}
      if(index(a[2],pa".")!=1){print "    WRONG admin col: " $0; bad++; next}
      if(index(a[3],pi".")!=1){print "    WRONG internal col: " $0; bad++; next}
      split(a[1],x,"."); split(a[2],y,"."); split(a[3],z,".");
      if(x[4]!=y[4]||y[4]!=z[4]){print "    UNALIGNED last octet: " $0; bad++; next}
      if(x[4]+0<lo||x[4]+0>hi){print "    OUT-OF-RANGE octet: " $0; bad++; next}
      ok++
    }
    END{print "RESULT " ok " " bad}
  ' "${VIP_SRC[@]}")"
  echo "$VIPOUT" | grep -v '^RESULT ' || true
  VOK="$(echo "$VIPOUT" | awk '/^RESULT /{print $2}')"
  VBAD="$(echo "$VIPOUT" | awk '/^RESULT /{print $3}')"
  if [ "${VOK:-0}" -eq "$VIP_COUNT_EXPECT" ] && [ "${VBAD:-1}" -eq 0 ]; then
    pass "aligned VIPs OK=$VOK bad=$VBAD (DC=$DC bands $VIP_PREFIX_PROVIDER/$VIP_PREFIX_ADMIN/$VIP_PREFIX_INTERNAL)"
  else
    fail "VIP validation OK=${VOK:-0} bad=${VBAD:-?} (want OK=$VIP_COUNT_EXPECT bad=0)"
  fi
  fi
fi

# ---------------------------------------------------------------------------
hdr "MAAS reachability gate (read-only)"
need_jq || finish
SUBJSON="$(maas "$MAAS_PROFILE" subnets read 2>/dev/null)" || { fail "MAAS unreachable: 'maas $MAAS_PROFILE subnets read' failed"; finish; }
pass "MAAS reachable (profile=$MAAS_PROFILE)"

# ---------------------------------------------------------------------------
hdr "CHECK 3: six planes resolved BY CIDR (id/vid/gw/dns)"
present=0
for c in "${PLANE_CIDRS[@]}"; do
  row="$(printf '%s' "$SUBJSON" | jq -r --arg c "$c" '.[] | select(.cidr==$c) | "id=\(.id) vid=\(.vlan.vid // 0) gw=\(.gateway_ip // "none") dns=\(.dns_servers|tostring)"')"
  if [ -z "$row" ]; then fail "plane ${PLANE_NAME[$c]} ($c) NOT FOUND"; continue; fi
  present=$((present+1))
  printf "    %-15s %-16s %s\n" "${PLANE_NAME[$c]}" "$c" "$row"
  gw="$(printf '%s' "$SUBJSON" | jq -r --arg c "$c" '.[]|select(.cidr==$c)|.gateway_ip // "none"')"
  exp="${PLANE_GW[$c]:-none}"
  if [ "$gw" != "$exp" ]; then
    if [ "$exp" = "none" ]; then warn "${PLANE_NAME[$c]} has gateway $gw (want none -- spurious-gw defect class, D-052)"; else fail "${PLANE_NAME[$c]} gateway=$gw (want $exp)"; fi
  fi
done
if [ "$present" -eq 6 ]; then pass "all six planes present (by CIDR)"; else fail "only $present/6 planes present"; fi

# --- metal-internal TAGGING (readiness item 3.7) ------------------------------------
# RETIRED EXPECTATION: this asserted `vid == METAL_INTERNAL_VID` unconditionally, i.e.
# VLAN 103. D-133 (ADOPTED 2026-07-23) ABOLISHED that stack for VR1 -- "VR1 DCs carve
# every plane FLAT on its dedicated node NIC. Metal-internal is an untagged subnet on
# the enp3s0 plane network ... VID 103 is a VR0-only fact." The assertion could
# therefore NEVER pass on either VR1 DC; the live `vid 0` it kept failing on is the
# CORRECT carve. lib-net's VR1 arms now UNSET the pair, so the guard below is a real
# branch, not a paper one -- and under `set -u` an unguarded read would abort the run.
#
# The guard does NOT weaken the gate: it swaps VR0's expectation for D-133's, which is
# the stronger of the two on a VR1 DC. Both branches can fail, and an unreadable VLAN
# refuses instead of defaulting to success -- `.vlan.vid // 0` used to render a MISSING
# vlan object and a genuine untagged VLAN identically as "0", so a subnet whose VLAN
# could not be read would have silently satisfied the untagged branch.
miraw="$(printf '%s' "$SUBJSON" | jq -r --arg c "$METAL_INTERNAL_CIDR" '.[]|select(.cidr==$c)|.vlan.vid')"
if [ -z "$miraw" ] || [ "$miraw" = "null" ]; then
  fail "metal-internal ($METAL_INTERNAL_CIDR) carries NO readable VLAN in MAAS -- cannot evaluate its tagging, refusing to treat that as untagged"
elif [ -n "${METAL_INTERNAL_VID:-}" ]; then
  # VR0 (vr0-dc0): the tagged VLAN-103 stack is this DC's real, validated as-built.
  if [ "$miraw" = "$METAL_INTERNAL_VID" ]; then pass "metal-internal is VID $METAL_INTERNAL_VID (VR0 Pattern A)"
  else fail "metal-internal VID=$miraw (want $METAL_INTERNAL_VID)"; fi
else
  # VR1 (D-133): untagged. MAAS represents the untagged/default VLAN as vid 0.
  if [ "$miraw" = "0" ]; then pass "metal-internal is UNTAGGED (vid 0) -- D-133 flat carve"
  else fail "metal-internal VID=$miraw but D-133 rules it UNTAGGED for VR1 -- a VLAN tag here means the abolished VLAN stack came back"; fi
fi
note "stale-NAME check is juju-side (run scripts/juju-spaces-check.sh after add-model)"

# ---------------------------------------------------------------------------
hdr "CHECK 2: data/storage NIC links BY CIDR (DC=$DC role nodes; octet per D-134 band)"
ROLE_SEEN=0
for h in "${HOSTS[@]}"; do
  if [ "${NODE_CLASS[$h]}" = "unclassified" ]; then continue; fi   # already refused above
  if [ "${NODE_CLASS[$h]}" = "utility" ]; then
    # Not an OpenStack role node (D-134 utility band). It is still a pinned node of
    # this DC, so its ENROLLMENT is asserted -- but the six-plane carve and the
    # bundle's binding are statements about role nodes only.
    usid="$(sysid_for "$h")"
    if [ -z "$usid" ]; then fail "$h (utility band .${HOST_OCTET[$h]}) has NO MAAS record for its pinned boot MAC"
    else note "$h ($usid, octet .${HOST_OCTET[$h]}) is D-134 utility-band infrastructure, not an OpenStack role node -- data-plane carve not asserted"; fi
    continue
  fi
  ROLE_SEEN=$((ROLE_SEEN+1))
  sid="$(sysid_for "$h")"
  if [ -z "$sid" ]; then fail "$h not enrolled in MAAS"; continue; fi
  echo "  == $h ($sid, octet .${HOST_OCTET[$h]}) =="
  IFJSON="$(maas "$MAAS_PROFILE" interfaces read "$sid" 2>/dev/null)" || { fail "cannot read interfaces for $h"; continue; }
  for c in "${DATA_PLANE_CIDRS[@]}"; do
    # Tab-separated on purpose: IFS is $'\n\t' at the top of this file, so a
    # space-separated payload does NOT word-split and a third field would be lost.
    line="$(printf '%s' "$IFJSON" | jq -r --arg c "$c" '.[] as $if | $if.links[]? | select(.subnet.cidr==$c) | "\($if.name)\t\(.ip_address // "(no-ip)")\t\($if.type // "unknown")"' | head -1)"
    if [ -z "$line" ]; then fail "$h missing link on ${PLANE_NAME[$c]} ($c)"; continue; fi
    IFS=$'\t' read -r ifname ip iftype <<<"$line"
    printf "    %-14s -> %-16s %-16s type=%s\n" "$ifname" "$c" "$ip" "$iftype"
    if [ "$c" = "$METAL_INTERNAL_CIDR" ]; then
      # RETIRED EXPECTATION (readiness item 3.7): this asserted the link name equals
      # METAL_INTERNAL_IFACE, i.e. `br-internal`. D-133 ABOLISHED that bridge for VR1
      # ("no br-metal.103 / br-internal stack exists in VR1"), and lib-net's VR1 arms
      # UNSET the constant, so the assertion is guarded to the DC that still has it.
      # The VR1 branch is NOT a skip -- it asserts D-133's own invariant, that the
      # plane lands FLAT on a dedicated PHYSICAL NIC. That is falsifiable in the exact
      # way that matters: rebuild a bridge or a VLAN stack over metal-internal and
      # MAAS reports type `bridge` / `vlan` and this fails. It is also stronger than
      # pinning the NIC NAME would be, and needs no new literal in a gate.
      # Measured basis: docs/audit/stage4-carve-verify-20260723.txt shows
      # `enp3s0 physical <dc>-metal-internal static:...` on all 18 role nodes, both DCs.
      if [ -n "${METAL_INTERNAL_IFACE:-}" ]; then
        [ "$ifname" = "$METAL_INTERNAL_IFACE" ] || fail "$h metal-internal on '$ifname' (want $METAL_INTERNAL_IFACE)"
      else
        [ "$iftype" = "physical" ] || fail "$h metal-internal on '$ifname' is type '$iftype' (D-133 rules a FLAT carve on a dedicated PHYSICAL NIC for VR1 -- a bridge/vlan here is the abolished VLAN-103 stack returning)"
      fi
    fi
    o="$(fourth_octet "$ip")"
    if [ "$o" != "${HOST_OCTET[$h]}" ]; then warn "$h ${PLANE_NAME[$c]} ip=$ip (want last octet .${HOST_OCTET[$h]})"; fi
  done
done
# Zero-floor: a gate that asserted nothing must not read as a gate that found nothing.
if [ "$ROLE_SEEN" -eq 0 ]; then fail "DC=$DC yielded ZERO OpenStack role nodes -- nothing was carve-checked; a check over an empty set is not a check"; fi

# ---------------------------------------------------------------------------
hdr "CHECK 4: DC=$DC OpenStack role nodes -- status / power"
MJSON="$(maas "$MAAS_PROFILE" machines read 2>/dev/null)" || { fail "cannot read machines"; finish; }
# Resolved BY THE SAME KEY as CHECK 2, not by hostname. Selecting on `.hostname` is
# what produced four of P4's eleven fatals on a VR1 DC: lib-hosts' VR1 HOSTS entries
# are TOFU NODE NAMES (config-of-record), while the MAAS hostname is random and
# re-minted on every re-enlistment (lib-hosts.sh:83-88).
for h in "${HOSTS[@]}"; do
  # The D-104 per-DC Juju controller is DELIBERATELY not asserted Ready here: it is a
  # `juju bootstrap` precondition, not a deploy one, and once bootstrapped it is
  # legitimately `Deployed`. Its presence is asserted in CHECK 2 above. An
  # unclassified node was already refused in the DC-selection block.
  [ "${NODE_CLASS[$h]}" = "role" ] || continue
  sid="$(sysid_for "$h")"
  if [ -z "$sid" ]; then fail "host $h not found in MAAS"; continue; fi
  row="$(printf '%s' "$MJSON" | jq -r --arg s "$sid" '.[]|select(.system_id==$s)|"\(.hostname) \(.status_name) power=\(.power_state)"')"
  if [ -z "$row" ]; then fail "host $h ($sid) resolved but absent from the machines list"; continue; fi
  echo "    $h -> $row"
  st="$(printf '%s' "$MJSON" | jq -r --arg s "$sid" '.[]|select(.system_id==$s)|.status_name')"
  # DOCFIX-200: Stage 4 hands off READY nodes, not Deployed ones -- Juju provisions at
  # Stage 5, so Ready is the correct pre-deploy state for a role node.
  if [ "$st" = "Ready" ]; then pass "$h Ready"; else fail "$h status=$st (want Ready)"; fi
done

finish