diff --git a/docs/v1-redeploy-changelog.md b/docs/v1-redeploy-changelog.md index f741f75..470f003 100644 --- a/docs/v1-redeploy-changelog.md +++ b/docs/v1-redeploy-changelog.md @@ -240,3 +240,35 @@ the normal release-to-Ready path PRESERVES host interfaces, so that block only ran on a normal teardown; the full carve (this script) is needed only after a decompose, which is why the bridges were never scripted before. + +### Carve hardening: self-discovered metal IP blocks br-metal static (KI) + +Root cause (cost several diagnostic rounds): after re-enrollment each host PXE-leases +its own metal IP (10.12.8.4N) at commission. MAAS records this as a StaticIPAddress +of **alloc_type 6 (DISCOVERED)** tied to the node via its boot NIC. This is a +SEPARATE object from the network-discovery table (`discoveries clear-by-mac-and-ip` +does NOT clear it) and from user allocations (`ipaddresses read` user-scope does NOT +show it). It causes `link-subnet ... ip_address=10.12.8.4N` to fail with the +misleading "IP address is already in use". + +Authoritative read (the lesson): `maas admin subnet ip-addresses ` reports +every in-use IP WITH its alloc_type and owning node -- this is the single correct +"who holds this IP and why" query. Lead with it; do not probe ipaddresses/discovery/ +leases piecemeal. + +Release: `maas admin ipaddresses release ip= force=true discovered=true` (BOTH +flags required; force alone returns "does not exist" for a discovered address). + +Script fix (carve-host-interfaces.sh): `release_self_discovered()` runs before every +STATIC link -- releases an alloc_type-6 record for the target IP ONLY when its owning +node == this host (node_summary.system_id), and REFUSES (fatal) if a different node +discovered it (a real conflict). Plus `emit` now captures and prints the MAAS error on +a failed mutation instead of discarding it to /dev/null (the discard hid the real +message and prolonged diagnosis). Only the metal plane (dhcp_on=true) is affected; +the no-DHCP planes never produced a self-lease. Verified: mock self-release path + +foreign-node refuse gate. + +NOTE (design consistency, not a blocker): host statics .40-.43 sit inside the +metal-admin/provider/internal VIP+mgmt reserve band (.2-.100). A reserved range blocks +AUTO assignment, not explicit STATIC, so it did not break the carve -- but host octets +arguably belong outside the VIP band. Log for the reserve-layout review. diff --git a/runbooks/appendix-A-troubleshooting.md b/runbooks/appendix-A-troubleshooting.md index 2d7ef3a..9d43136 100644 --- a/runbooks/appendix-A-troubleshooting.md +++ b/runbooks/appendix-A-troubleshooting.md @@ -520,3 +520,24 @@ Warning about the asymmetry: the Container Infra panel lists clusters cross-project under admin policy, which makes the strictly-scoped Nova panel look broken when it is not. + +-------------------------------------------------------------------------------- +SYMPTOM: link-subnet fails "IP address is already in use" but the IP is in no + visible table (ipaddresses read empty, discovery cleared, not in DHCP + dynamic range, interface on the correct VLAN). +-------------------------------------------------------------------------------- +CAUSE: A freshly re-enrolled host PXE-leases its own metal IP (10.12.8.4N) at + commission; MAAS keeps it as a StaticIPAddress of alloc_type 6 + (DISCOVERED), tied to the node. Distinct from the network-discovery + table AND from user allocations -- neither `discoveries + clear-by-mac-and-ip` nor a plain `ipaddresses release` clears it. +AUTHORITATIVE READ (use FIRST, before guessing): + maas admin subnet ip-addresses + -> lists every in-use IP with .alloc_type and .node_summary. alloc_type 6 + = DISCOVERED. This is the definitive "who holds this IP and why". +FIX: maas admin ipaddresses release ip= force=true discovered=true + (BOTH flags; force alone -> "does not exist"). Only release when the + discovered record's node is the SAME host -- a different node means a + real address conflict; stop and investigate. +NOW AUTOMATED: scripts/carve-host-interfaces.sh release_self_discovered() does + this, gated to self-owned records only. diff --git a/scripts/carve-host-interfaces.sh b/scripts/carve-host-interfaces.sh index f001cd9..ed99192 100644 --- a/scripts/carve-host-interfaces.sh +++ b/scripts/carve-host-interfaces.sh @@ -93,13 +93,37 @@ local desc="$1"; shift if [ "$MODE" = "apply" ]; then echo " DO: $desc" - if ! maas "$MAAS_PROFILE" "$@" >/dev/null; then fail "$desc"; return 1; fi + local out + if ! out="$(maas "$MAAS_PROFILE" "$@" 2>&1)"; then + fail "$desc" + echo " MAAS said: $(printf '%s' "$out" | grep -viE '^(Success|Machine-readable)' | head -3 | tr '\n' ' ')" >&2 + return 1 + fi else echo " WOULD: $desc" echo " maas $MAAS_PROFILE $*" fi } +# release_self_discovered : if MAAS holds as a DISCOVERED (alloc_type 6) +# address observed from THIS host (node==SID), release it so a STATIC can take it. +# Gated: only releases when the discovered record belongs to this host -- never +# touches an address discovered on another node (that would be a real conflict). +# (Re-enrolled hosts PXE-lease their own metal IP at commission; that self-lease +# otherwise blocks the br-metal static. See troubleshooting appendix.) +release_self_discovered() { + local ip="$1" subid="$2" owner + owner="$(maas_q subnet ip-addresses "$subid" 2>/dev/null \ + | jq -r --arg ip "$ip" '.[]|select(.ip==$ip and .alloc_type==6)|.node_summary.system_id // empty' | head -1)" + [ -z "$owner" ] && return 0 # not discovered -> nothing to do + if [ "$owner" != "$SID" ]; then + fail "$ip is DISCOVERED by a DIFFERENT node ($owner), not $HN -- refusing to release (possible real conflict)" + return 1 + fi + emit "release self-discovered $ip (alloc_type 6, node=$HN)" \ + ipaddresses release ip="$ip" force=true discovered=true +} + hdr "$HN ($SID) octet=.$OCTET mode=$MODE" echo "resolved subnet/vlan ids (by CIDR):" printf " provider %s sub=%s vlan=%s\n" "$C_PROV" "$(subid_of "$C_PROV")" "$(vlanid_of "$C_PROV")" @@ -118,6 +142,7 @@ if linked_to "$nic" "$cidr"; then note "$nic already STATIC on $cidr -- SKIP"; return 0; fi vlan="$(vlanid_of "$cidr")"; sub="$(subid_of "$cidr")" emit "$nic(id=$id) -> VLAN $vlan ($cidr)" interface update "$SID" "$id" vlan="$vlan" + release_self_discovered "$ip" "$sub" || return 1 emit "$nic(id=$id) -> STATIC $ip on subnet $sub" interface link-subnet "$SID" "$id" mode=STATIC subnet="$sub" ip_address="$ip" } @@ -138,6 +163,7 @@ else note "br-metal exists -- SKIP create"; fi [ "$MODE" = apply ] && BMID="$(ifid_of br-metal)" || BMID="" if ! linked_to br-metal "$C_METAL"; then + release_self_discovered "10.12.8.$OCTET" "$(subid_of "$C_METAL")" || true emit "br-metal(id=$BMID) -> STATIC 10.12.8.$OCTET on subnet $(subid_of "$C_METAL")" \ interface link-subnet "$SID" "$BMID" mode=STATIC subnet="$(subid_of "$C_METAL")" ip_address="10.12.8.$OCTET" else note "br-metal already on $C_METAL -- SKIP"; fi