diff --git a/scripts/dc-region-topology.sh b/scripts/dc-region-topology.sh index 949852d..5aba03e 100644 --- a/scripts/dc-region-topology.sh +++ b/scripts/dc-region-topology.sh @@ -179,6 +179,30 @@ echo "${SITE%%-*}-${DCLABEL}-$1" } +# All subnet CIDRs currently sitting on a given fabric NAME. +subs_on_fabric() { printf '%s' "$SUBJSON" | python3 -c ' +import json,sys +n=sys.argv[1] +print(" ".join(s["cidr"] for s in json.load(sys.stdin) if s["vlan"].get("fabric")==n)) +' "$1" 2>/dev/null; } + +# Is this an auto-created MAAS fabric name (fabric-)? +is_auto_fabric() { case "$1" in fabric-[0-9]*) return 0 ;; *) return 1 ;; esac; } + +# PREFER RENAME OVER MOVE -- learned the hard way, 2026-07-30. +# The first version of this script CREATED a named fabric and then MOVED the +# existing provider-public subnet onto it. MAAS did NOT bring the region VM's +# own `enp2s0` along: the interface stayed on the old auto fabric's VLAN while +# its subnet left, so `hot-kid` was left holding a static 10.12.4.6 on a VLAN +# with no subnet -- the region's provider-public leg, which it needs for image +# sync, and the identical under-carve class that cost three bootstrap attempts +# on the Juju controller. Live connectivity was unaffected (the OS netplan is +# independent of MAAS's model) so nothing alerted; it was found by reading the +# interface links back, which the check did not do at the time. +# RENAMING the auto fabric that ALREADY carries the subnet moves nothing and +# leaves every interface link intact. A MOVE is kept only as a last resort for +# the case where the subnet sits on a fabric that carries OTHER planes too. + # ---------------------------------------------------------------- check ---- do_check() { reload @@ -249,6 +273,47 @@ tags="$(m tags read 2>/dev/null | python3 -c 'import json,sys; print(" ".join(t["name"] for t in json.load(sys.stdin)))' 2>/dev/null)" case " $tags " in *" openstack-$SITE "*) ok "tag 'openstack-$SITE' exists" ;; *) bad "tag 'openstack-$SITE' exists (tags: ${tags:-none})" ;; esac + # 6. NO STRANDED INTERFACE LINKS. + # THIS IS THE ASSERTION THAT WOULD HAVE CAUGHT THE 2026-07-30 DEFECT. Moving a + # subnet between fabrics does NOT bring interface links along: the region VM's + # enp2s0 was left holding a static 10.12.4.6 while its subnet had moved to a + # different VLAN. Live connectivity was UNAFFECTED (the OS netplan is + # independent of MAAS's model), so nothing else noticed -- every other + # assertion in this script passed 39/39 with the model inconsistent. + # Invariant: an interface holding a link to subnet S must be on S's VLAN. + STRAND="$( + { m rack-controllers read 2>/dev/null; echo '---'; m machines read 2>/dev/null; } | python3 -c ' +import json,sys +raw=sys.stdin.read().split("---") +sub_vlan={} +try: subs=json.loads(sys.argv[1]) +except Exception: subs=[] +for s in subs: sub_vlan[s["cidr"]]=s["vlan"]["id"] +bad=[] +for chunk in raw: + chunk=chunk.strip() + if not chunk: continue + try: nodes=json.loads(chunk) + except Exception: continue + for n in nodes: + for i in n.get("interface_set",[]): + ivlan=(i.get("vlan") or {}).get("id") + for l in i.get("links",[]): + sn=(l.get("subnet") or {}).get("cidr") + if not sn or sn not in sub_vlan: continue + if ivlan != sub_vlan[sn]: + bad.append("%s/%s link=%s(vlan %s) iface-vlan=%s" % ( + n.get("hostname"), i.get("name"), sn, sub_vlan[sn], ivlan)) +print("\n".join(bad)) +' "$SUBJSON" 2>/dev/null + )" + if [ -z "$STRAND" ]; then + ok "no stranded interface links (every link is on its subnet's VLAN)" + else + bad "no stranded interface links -- found:" + printf '%s\n' "$STRAND" | sed 's/^/ /' + fi + echo "dc-region-topology: $P passed, $F failed" [ "$F" -eq 0 ] || return 1 return 0 @@ -260,11 +325,26 @@ reload echo "dc-region-topology apply: site=$SITE profile=$PROFILE commit=$COMMIT" - # 1. named plane fabrics + # 1. named plane fabrics -- RENAME an auto fabric that already carries this + # plane's subnet (moves nothing); only CREATE when there is nothing to rename. for cidr in "${PLANE_CIDRS[@]}"; do plane="${PLANE_NAME[$cidr]}"; fn="$(fabric_name_for "$plane")" [ -z "$fn" ] && continue if [ -n "$(fab_id_byname "$fn")" ]; then echo " [idem] fabric '$fn' present"; continue; fi + + cur_fab="$(sub_fabric_bycidr "$cidr")" + if [ -n "$cur_fab" ] && is_auto_fabric "$cur_fab" && [ "$(subs_on_fabric "$cur_fab")" = "$cidr" ]; then + # The auto fabric carries ONLY this plane's subnet -> renaming is safe and + # keeps every interface link on it intact. + cur_id="$(fab_id_byname "$cur_fab")" + say "fabric update $cur_id name=$fn # RENAME '$cur_fab' (keeps interface links; no move)" + if [ "$COMMIT" -eq 1 ]; then + m fabric update "$cur_id" name="$fn" >/dev/null 2>&1 || refuse "could not rename fabric $cur_fab -> $fn" + reload; [ -n "$(fab_id_byname "$fn")" ] || refuse "renamed $cur_fab but $fn did not read back" + fi + continue + fi + say "fabrics create name=$fn" [ "$COMMIT" -eq 1 ] && { m fabrics create name="$fn" >/dev/null 2>&1 || refuse "could not create fabric $fn"; reload [ -n "$(fab_id_byname "$fn")" ] || refuse "created fabric $fn but it did not read back"; } @@ -305,6 +385,9 @@ # placeholder rather than an empty value -- an empty `vlan=` reads like a # defect and would hide a real one. say "subnet update $sid vlan=${vid:-} # MOVE $cidr off '$actual_fab' onto '$fn'" + echo " WARN a MOVE does NOT bring interface links along -- any NIC still on" + echo " '$actual_fab' will be left holding a static on a subnet-less VLAN." + echo " Read the affected machines' interfaces back after this apply." if [ "$COMMIT" -eq 1 ]; then [ -n "$vid" ] || refuse "fabric $fn has no untagged VLAN to move $cidr onto" m subnet update "$sid" vlan="$vid" >/dev/null 2>&1 || refuse "could not move subnet $cidr" diff --git a/tests/dc-region-topology/run-tests.sh b/tests/dc-region-topology/run-tests.sh index d050074..e8bd570 100644 --- a/tests/dc-region-topology/run-tests.sh +++ b/tests/dc-region-topology/run-tests.sh @@ -102,8 +102,14 @@ case "$sub" in rack-controllers) case "${STUB_RACK:-hot-kid}" in none) echo '[]' ;; - *) printf '[{"hostname":"%s"}]\n' "${STUB_RACK:-hot-kid}" ;; + *) # interface_set drives the stranded-link assertion. + # STUB_STRAND=1 puts enp2s0 on vlan 5002 while its + # subnet 10.12.4.0/22 lives on vlan 5189 -- exactly the + # 2026-07-30 defect. + if [ "${STUB_STRAND:-0}" = "1" ]; then IV=5002; else IV=5189; fi + printf '[{"hostname":"%s","interface_set":[{"name":"enp1s0","vlan":{"id":5001},"links":[{"mode":"static","ip_address":"10.12.8.6","subnet":{"cidr":"10.12.8.0/22"}}]},{"name":"enp2s0","vlan":{"id":%s},"links":[{"mode":"static","ip_address":"10.12.4.6","subnet":{"cidr":"10.12.4.0/22"}}]}]}]\n' "${STUB_RACK:-hot-kid}" "$IV" ;; esac ;; + machines) echo '[]' ;; fabrics) case "${1:-read}" in read) emit_fabrics ;; create) echo '{"id":99}' ;; esac ;; spaces) case "${1:-read}" in read) emit_spaces ;; create) echo '{"id":99}' ;; esac ;; subnets) case "${1:-read}" in read) emit_subnets ;; create) echo '{"id":99}' ;; esac ;; @@ -184,7 +190,14 @@ OUT="$(STUB_STATE=empty bash "$SCRIPT" apply vr1-dc0 --profile vr1-dc0-region --expect-rack hot-kid 2>&1)"; RC=$? chk "apply without --commit exits 0" "$RC" 0 case "$OUT" in *"DRY RUN complete"*) ok "dry run says it wrote nothing" ;; *) no "dry run says it wrote nothing (got '$OUT')" ;; esac -case "$OUT" in *"DRY fabrics create name=vr1-dc0-provider-public"*) ok "dry run plans the named fabrics" ;; *) no "dry run plans the named fabrics (got '$OUT')" ;; esac +# RE-POINTED 2026-07-30: this used to assert that provider-public's fabric was +# CREATED. That pinned the DEFECTIVE behaviour -- creating a fabric and moving +# the existing subnet onto it is what stranded the region VM's enp2s0. The +# invariant is now: a plane whose subnet already sits alone on an auto fabric is +# RENAMED (asserted below), and only a plane with NO existing subnet is created. +# Replaced rather than deleted, per the standing rule that remediating a finding +# must not be done by removing the assertion that catches it. +case "$OUT" in *"DRY fabrics create name=vr1-dc0-metal-internal"*) ok "dry run plans a create for a plane with no existing subnet" ;; *) no "dry run plans a create for a plane with no existing subnet (got '$OUT')" ;; esac case "$OUT" in *"DRY spaces create name=metal-admin"*) ok "dry run plans the spaces" ;; *) no "dry run plans the spaces" ;; esac case "$OUT" in *APPLY*) no "dry run emitted APPLY lines" ;; *) ok "dry run emitted no APPLY lines" ;; esac @@ -208,6 +221,27 @@ case "$OUT" in *""*) ok "MOVE line uses an explicit vlan placeholder" ;; *) no "MOVE line uses an explicit vlan placeholder (got '$OUT')" ;; esac case "$OUT" in *""*) ok "space-bind line uses an explicit fabric placeholder" ;; *) no "space-bind line uses an explicit fabric placeholder" ;; esac +# --- STRANDED INTERFACE LINKS ---------------------------------------------- +# The 2026-07-30 defect: a subnet MOVE leaves interface links behind on the old +# VLAN. Every other assertion passed 39/39 while the model was inconsistent, and +# live connectivity was unaffected, so only an interface read catches it. +OUT="$(STUB_STATE=complete bash "$SCRIPT" check vr1-dc0 --profile vr1-dc0-region --expect-rack hot-kid 2>&1)"; RC=$? +case "$OUT" in *"[ok] no stranded interface links"*) ok "consistent links pass the stranded check" ;; *) no "consistent links pass the stranded check (got '$OUT')" ;; esac +chk "consistent region still exits 0" "$RC" 0 + +OUT="$(STUB_STATE=complete STUB_STRAND=1 bash "$SCRIPT" check vr1-dc0 --profile vr1-dc0-region --expect-rack hot-kid 2>&1)"; RC=$? +case "$OUT" in *"[FAIL] no stranded interface links"*) ok "a STRANDED link FAILS the check" ;; *) no "a stranded link FAILS the check (got '$OUT')" ;; esac +case "$OUT" in *"enp2s0"*) ok "the stranded failure names the interface" ;; *) no "the stranded failure names the interface" ;; esac +chk "stranded region exits 1" "$RC" 1 + +# --- RENAME IS PREFERRED OVER MOVE ----------------------------------------- +# Renaming the auto fabric that already carries the subnet moves nothing and +# leaves interface links intact; creating + moving is what stranded enp2s0. +OUT="$(STUB_STATE=empty bash "$SCRIPT" apply vr1-dc0 --profile vr1-dc0-region --expect-rack hot-kid 2>&1)" +case "$OUT" in *"RENAME 'fabric-1'"*) ok "an auto fabric carrying only this plane is RENAMED" ;; *) no "auto fabric is RENAMED (got '$OUT')" ;; esac +case "$OUT" in *"fabrics create name=vr1-dc0-provider-public"*) no "provider-public was CREATED instead of renamed (the stranding path)" ;; *) ok "provider-public was not created-and-moved" ;; esac +case "$OUT" in *"fabrics create name=vr1-dc0-storage"*) ok "a plane with no existing subnet is still CREATED" ;; *) no "a plane with no existing subnet is still created" ;; esac + echo echo "dc-region-topology: $P passed, $F failed" [ "$F" -eq 0 ] || exit 1