Newer
Older
openstack-caracal-dc-dc / scripts / maas-role-tags.sh
#!/usr/bin/env bash
# scripts/maas-role-tags.sh <check|apply> <site> [--commit]
#
# Fork 3 (D-121 Option C placement): create the per-role MAAS tags the bundle's machines
# block CONSTRAINS ON, and apply them to this site's nodes.
#
# WHY THIS EXISTS. Every machine in `bundle.yaml` constrains on TWO tags, e.g.
# `tags=openstack-vr1-dc0,control`. Measured 2026-07-29 (D-136 chain audit, finding 20):
# `grep -rn "tags create" scripts/ runbooks/` finds ONLY the site tag
# (`reenroll-hosts.sh`), and the three ROLE tags -- control / compute / storage -- are
# created NOWHERE. Juju passes `tags=` to MAAS as an allocation constraint, so an
# unmatched tag yields no candidate machine and the deploy aborts at allocation. That is
# the GOOD failure mode (loud, not silent), but it blocks BOTH DCs.
#
# HOW THE ROLE IS DERIVED -- two independent signals that must AGREE (hard rule 2: no
# inferred values):
#   1. the D-134-AMENDED octet band, RULED: control .100-.119 / compute .120-.149 /
#      storage .150-.200, read from lib-hosts' HOST_OCTET
#   2. the role token in the host's own name (vr1-dcN-<role>-NN), read from HOST_OCTET's
#      key set
# If they disagree the script REFUSES for that node rather than picking one. A node whose
# octet falls in no ruled band also refuses -- an unrecognised state is never a default.
#
# Nodes are matched to MAAS records by PINNED BOOT MAC, never by hostname: MAAS renames
# machines at enlistment (2026-07-21 incident) and the current names are random.
#
# DRY BY DEFAULT. `apply` plans and prints; nothing is written without --commit, and every
# write is READ BACK. Precedent: opnsense-plugins.sh apply ALWAYS silently dry-ran, which
# voided every prior "applied" claim from it.
#
# Exit: 0 ok/clean | 1 findings or write error | 2 could-not-evaluate.  ASCII + LF.
set -uo pipefail
IFS=$'\n\t'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$SCRIPT_DIR/.." && pwd)"
MAAS_PROFILE="${MAAS_PROFILE:-admin}"
ROLES=(control compute storage)

usage() { echo "usage: $(basename "$0") <check|apply> <vr1-dc0|vr1-dc1> [--commit]" >&2; exit 2; }
refuse() { echo "REFUSE: $*" >&2; exit 2; }

ACTION="${1:-}"; SITE="${2:-}"; COMMIT=0
[ "${3:-}" = "--commit" ] && COMMIT=1
case "$ACTION" in check|apply) ;; *) usage ;; esac
case "$SITE" in vr1-dc0|vr1-dc1) ;; *) usage ;; esac

# An ABSENT client is not an unreachable service. This deployment has misdiagnosed that
# three times (preflight's "MAAS unreachable" was a missing binary), so they are split.
command -v maas >/dev/null 2>&1 \
  || refuse "the 'maas' CLI is ABSENT on $(hostname) -- this is not 'MAAS unreachable'. Run this where maas lives (the D-128 Plane-2 headend)."

MACHINES="$(maas "$MAAS_PROFILE" machines read 2>/dev/null)" \
  || refuse "'maas $MAAS_PROFILE machines read' failed -- MAAS unreachable or the CLI is not logged in (profile=$MAAS_PROFILE)"
TMPD="$(mktemp -d)"; trap 'rm -rf "$TMPD"' EXIT
printf '%s' "$MACHINES" > "$TMPD/machines.json"
export MJSON="$TMPD/machines.json"

# ---- expected roles, derived from lib-hosts + the D-134 bands ------------------------
EXPECT="$(bash -c '
  source "'"$REPO"'/scripts/lib-hosts.sh" >/dev/null 2>&1 || exit 3
  lib_hosts_select_dc "'"$SITE"'" >/dev/null 2>&1 || exit 3
  for h in "${HOSTS[@]}"; do
    printf "%s\t%s\t%s\n" "$h" "${HOST_OCTET[$h]}" "${HOST_BOOT_MAC[$h]}"
  done' 2>/dev/null)" || refuse "lib-hosts did not yield hosts for $SITE"
[ -n "$EXPECT" ] || refuse "lib-hosts produced no hosts for $SITE"

PLAN="$(printf '%s\n' "$EXPECT" | python3 -c '
import sys
BANDS = (("control", 100, 119), ("compute", 120, 149), ("storage", 150, 200))
# D-134 AMENDMENT 2026-07-29: .4-.49 is the UTILITY band -- per-DC INFRASTRUCTURE the
# OpenStack nodes consume (the artifact service at .4, the Juju controller at .5), NOT
# OpenStack role nodes. A host there with no role token in its name is not a role node
# and is SKIPPED, not refused. The refusal keeps its teeth for genuine DISAGREEMENT:
# a utility-band octet on a role-named host, or a role-band octet whose name disagrees.
UTILITY = (4, 49)
rc = 0
for line in sys.stdin.read().splitlines():
    if not line.strip():
        continue
    host, octet, mac = line.split("\t")
    o = int(octet)
    by_band = next((r for r, lo, hi in BANDS if lo <= o <= hi), None)
    by_name = next((r for r, _, _ in BANDS if "-%s-" % r in host), None)
    if UTILITY[0] <= o <= UTILITY[1]:
        if by_name is not None:
            print("REFUSE\t%s\t%s\tutility-band octet .%d but name says %s" % (host, mac, o, by_name)); rc = 1
        else:
            print("SKIP\t%s\t%s\tutility band .%d -- per-DC infrastructure, not an OpenStack role node" % (host, mac, o))
        continue
    if by_band is None:
        print("REFUSE\t%s\t%s\toctet .%d is in NO ruled D-134 band" % (host, mac, o)); rc = 1
    elif by_name is None:
        print("REFUSE\t%s\t%s\tname carries no role token" % (host, mac)); rc = 1
    elif by_band != by_name:
        print("REFUSE\t%s\t%s\tband says %s but name says %s" % (host, mac, by_band, by_name)); rc = 1
    else:
        print("OK\t%s\t%s\t%s" % (host, mac.lower(), by_band))
sys.exit(rc)
')" || { printf '%s\n' "${PLAN:-}" | grep '^REFUSE' | while IFS=$'\t' read -r _ h m why; do
        echo "  [REFUSE] $h ($m): $why" >&2; done
      refuse "role derivation disagreed with itself for the node(s) above -- refusing rather than picking one"; }

echo "== maas-role-tags $ACTION $SITE $([ "$COMMIT" -eq 1 ] && echo '--commit' || echo '(DRY RUN)') =="
echo "   role derived from the D-134 band AND the host name; both must agree"
printf '%s\n' "$PLAN" | grep '^SKIP' | while IFS=$'\t' read -r _ h m why; do
  printf "  [skip]  %-22s %s\n" "$h" "$why"
done
echo

# ---- which role tags exist in MAAS? --------------------------------------------------
TAGS="$(maas "$MAAS_PROFILE" tags read 2>/dev/null | python3 -c '
import json,sys
try: print(" ".join(t.get("name","") for t in json.load(sys.stdin)))
except Exception: print("")')"
missing_tags=""
for r in "${ROLES[@]}"; do
  case " $TAGS " in *" $r "*) echo "  [ok]   tag '$r' exists" ;;
                    *) echo "  [MISS] tag '$r' does NOT exist -- every machine constraining on it would fail allocation"
                       missing_tags="${missing_tags}${r}\n" ;; esac
done
echo

# ---- per-node current vs expected ----------------------------------------------------
REPORT="$(printf '%s\n' "$PLAN" | python3 -c '
import json, sys, os
plan = [l.split("\t") for l in sys.stdin.read().splitlines() if l.startswith("OK\t")]
machines = json.load(open(os.environ["MJSON"]))
by_mac = {}
for m in machines:
    bi = m.get("boot_interface") or {}
    mac = (bi.get("mac_address") or "").lower()
    if mac:
        by_mac[mac] = m
for _, host, mac, role in plan:
    m = by_mac.get(mac)
    if not m:
        print("ABSENT\t%s\t%s\t%s\t-" % (host, mac, role)); continue
    have = set(m.get("tag_names") or [])
    print("%s\t%s\t%s\t%s\t%s" % ("HAS" if role in have else "NEED", host, mac, role, m["system_id"]))
' 2>/dev/null)" || refuse "could not correlate lib-hosts nodes with MAAS machines"

printf '%s\n' "$REPORT" | while IFS=$'\t' read -r st host mac role sid; do
  case "$st" in
    HAS)    printf "  [ok]   %-22s %s -> tag '%s' present\n" "$host" "$sid" "$role" ;;
    NEED)   printf "  [NEED] %-22s %s -> tag '%s' MISSING\n" "$host" "$sid" "$role" ;;
    ABSENT) printf "  [MISS] %-22s pinned MAC %s not found in MAAS\n" "$host" "$mac" ;;
  esac
done

n_need="$(printf '%s\n' "$REPORT" | grep -c '^NEED' || true)"
n_absent="$(printf '%s\n' "$REPORT" | grep -c '^ABSENT' || true)"
n_tagmiss="$(printf '%b' "${missing_tags:-}" | grep -c . || true)"
echo
echo "  summary: $n_tagmiss role tag(s) missing, $n_need node(s) needing a tag, $n_absent node(s) not in MAAS"

if [ "$ACTION" = "check" ]; then
  if [ "$n_tagmiss" -eq 0 ] && [ "$n_need" -eq 0 ] && [ "$n_absent" -eq 0 ]; then
    echo "PASS: every ruled role tag exists and every node carries its role"; exit 0
  fi
  echo "FAIL: role tagging incomplete -- 'apply $SITE --commit' after review"; exit 1
fi

# ---- apply ---------------------------------------------------------------------------
[ "$n_absent" -eq 0 ] || refuse "$n_absent node(s) have no MAAS record for their pinned MAC -- resolve before tagging"
if [ "$COMMIT" -ne 1 ]; then
  echo
  echo "DRY RUN -- would create $n_tagmiss tag(s) and tag $n_need node(s). Re-run with --commit."
  exit 0
fi

rc=0
while IFS= read -r r; do
  [ -n "$r" ] || continue
  if maas "$MAAS_PROFILE" tags create name="$r" comment="D-121 Option C role tag (Fork 3); consumed by bundle.yaml machines constraints" >/dev/null 2>&1; then
    # read back -- a create that is not verified is a claim, not a fact
    if maas "$MAAS_PROFILE" tags read 2>/dev/null | grep -q "\"name\": \"$r\""; then
      echo "  [applied] tag '$r' created (read back)"
    else
      echo "  [ERROR]   tag '$r' create returned ok but is ABSENT on read-back"; rc=1
    fi
  else
    echo "  [ERROR]   tag '$r' create failed"; rc=1
  fi
done < <(printf '%b' "${missing_tags:-}")

printf '%s\n' "$REPORT" | grep '^NEED' | while IFS=$'\t' read -r _ host mac role sid; do
  if maas "$MAAS_PROFILE" tag update-nodes "$role" add="$sid" >/dev/null 2>&1; then
    if maas "$MAAS_PROFILE" machine read "$sid" 2>/dev/null | grep -q "\"$role\""; then
      echo "  [applied] $host ($sid) tagged '$role' (read back)"
    else
      echo "  [ERROR]   $host ($sid) tag '$role' not present on read-back"
    fi
  else
    echo "  [ERROR]   $host ($sid) tag '$role' failed"
  fi
done

echo
[ "$rc" -eq 0 ] && echo "apply complete -- re-run 'check $SITE' to confirm convergence"
exit "$rc"