Newer
Older
openstack-caracal-dc-dc / scripts / dc-region-topology.sh
#!/usr/bin/env bash
# scripts/dc-region-topology.sh <check|apply> <site> --profile <maas-profile>
#                               --expect-rack <name>[,<name>...] [--commit]
#
# Build (or verify) a per-DC MAAS region's PLANE TOPOLOGY: named plane fabrics,
# the six spaces, the six v4 plane subnets on the right VLANs, the metal-admin
# service config (dynamic range + node DNS), and the site placement tag.
#
#   check   READ-ONLY. Exit 0 only if every assertion passes.
#   apply   Idempotent, DRY unless --commit. Reads back every write.
#
# WHY THIS EXISTS (2026-07-30, D-132 q1 per-DC MAAS regions).
# A read-only survey of the dc0 carve found that THREE of its components had no
# repo tool at all -- the named plane fabrics, the six v4 plane subnets, and the
# site tag were all created ad-hoc during the Stage-4 window, with the mutations
# landing only in `~/as-executed/2026-07-23-stage4-carve.log`, which is not in
# the repo. When D-132 q1 required rebuilding that carve on a new region there
# was nothing to re-run. This script closes that gap so the SECOND DC and every
# Roosevelt DC standup rebuild from a tool rather than from a transcript.
#
# WHAT THIS SCRIPT DOES NOT DO, and what owns each piece instead:
#   v6 plane subnets      scripts/dc-plane-ipam.sh carve-v6 <site>
#   D-134 reserved bands  scripts/dc-plane-ipam.sh reserve  <site>
#   role tags on nodes    scripts/maas-role-tags.sh apply   <site>
#   per-node v4 NIC carve scripts/dc-node-carve.sh          <site>
#   per-node v6 addresses scripts/dc-node-v6-carve.py --profile <p>
#   per-machine power     scripts/maas-node-power.sh
# Sequencing matters: carve-v6 refuses to guess a fabric, so the v4 twins this
# script creates must exist first.
#
# REGION SAFETY IS NOT OPTIONAL HERE. Every mutating call is preceded by
# scripts/maas-profile-assert.sh, which proves the profile resolves to the
# intended region by RACK-CONTROLLER IDENTITY. Two regions hold separate
# databases, so against the wrong profile this script's `apply` is an
# idempotent no-op that prints PASS while the real region stays untouched --
# the failure mode that motivated the assert tool in the first place. A machine
# COUNT is not proof; a rack registers to exactly one region.
#
# NUMERIC MAAS IDS ARE REGION-LOCAL AND ARE NEVER PERSISTED. Fabric, VLAN,
# subnet and space ids are resolved per-run, by NAME or CIDR, under the same
# profile (lib-net.sh:8-10 -- "CIDR is the stable key throughout"). The
# pre-migration capture's `id=`/`vlan_id=` values are DATABASE ROW IDS and are
# deliberately not read by anything here.
#
# Exit: 0 all assertions pass | 1 assertion(s) failed | 2 could not evaluate.
# ASCII + LF.
set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/.." && pwd)"

ACTION="${1:-}"; SITE="${2:-}"
if [ "$#" -gt 2 ]; then shift 2; else set --; fi
PROFILE=""; EXPECT_RACK=""; COMMIT=0
while [ "$#" -gt 0 ]; do
  case "$1" in
    --profile)     PROFILE="${2:-}"; shift 2 ;;
    --expect-rack) EXPECT_RACK="${2:-}"; shift 2 ;;
    --commit)      COMMIT=1; shift ;;
    *) echo "FAIL: unknown option '$1'" >&2; exit 2 ;;
  esac
done

usage() {
  cat >&2 <<'U'
usage: dc-region-topology.sh <check|apply> <site> --profile <p> --expect-rack <r>[,<r>] [--commit]
  e.g. dc-region-topology.sh check vr1-dc0 --profile vr1-dc0-region --expect-rack hot-kid
       dc-region-topology.sh apply vr1-dc0 --profile vr1-dc0-region --expect-rack hot-kid --commit
U
  exit 2
}
[ -n "$ACTION" ] && [ -n "$SITE" ] || usage
case "$ACTION" in check|apply) ;; *) usage ;; esac
[ -n "$PROFILE" ] || { echo "FAIL: --profile is REQUIRED. There is no default: the repo-wide default 'admin' resolves to the OFFICE1 region, and a topology build against the wrong region is a silent no-op." >&2; exit 2; }
[ -n "$EXPECT_RACK" ] || { echo "FAIL: --expect-rack is REQUIRED -- it is what proves the profile points at the region you mean." >&2; exit 2; }

command -v maas >/dev/null 2>&1 || { echo "REFUSE: no 'maas' CLI on this host" >&2; exit 2; }

# ---- region identity gate: run BEFORE reading or writing anything ----------
ASSERT="$HERE/maas-profile-assert.sh"
[ -f "$ASSERT" ] || { echo "REFUSE: $ASSERT missing -- refusing to touch a region whose identity cannot be proven" >&2; exit 2; }
if ! bash "$ASSERT" "$PROFILE" "$EXPECT_RACK"; then
  echo "REFUSE: profile '$PROFILE' does not resolve to the expected region -- nothing was read or written" >&2
  exit 2
fi

# ---- expected state: DERIVED from lib-net, never typed ---------------------
# shellcheck source=/dev/null
. "$REPO/scripts/lib-net.sh"
lib_net_select_dc "$SITE" >/dev/null 2>&1 || { echo "FAIL: lib_net_select_dc rejected site '$SITE'" >&2; exit 2; }

# The DC label ("dc0"/"dc1") drives fabric naming. DERIVE it, never type it.
case "$SITE" in *-*) DCLABEL="${SITE#*-}" ;; *) echo "FAIL: site '$SITE' is not <region>-<dc>" >&2; exit 2 ;; esac

# metal-admin is DELIBERATELY NOT a named fabric. In the dc0 as-built it sits on
# the auto-created fabric that owns the rack's PXE interface (`fabric-4` in the
# Office1 region), because that fabric is created by MAAS when the rack
# registers -- before anything in this repo can name it. Renaming it would fight
# MAAS for ownership of its own discovery artifact for no functional gain:
# Juju binds SPACES, not fabrics. So five planes get named fabrics and
# metal-admin keeps whatever MAAS made.
METAL_ADMIN_CIDR="10.12.8.0/22"
case "$SITE" in
  vr1-dc1) METAL_ADMIN_CIDR="10.12.68.0/22" ;;
esac

P=0; F=0
ok(){ echo "  [ok]   $1"; P=$((P+1)); }
bad(){ echo "  [FAIL] $1"; F=$((F+1)); }
ck(){ if [ "$1" = 0 ]; then ok "$2"; else bad "$2"; fi; }
refuse(){ echo "REFUSE: $1" >&2; exit 2; }

m(){ maas "$PROFILE" "$@"; }

# ---- per-run id resolution (never persisted) ------------------------------
FABJSON=""; SUBJSON=""; SPCJSON=""
reload() {
  FABJSON="$(m fabrics read 2>/dev/null)" || refuse "could not read fabrics"
  SUBJSON="$(m subnets read 2>/dev/null)" || refuse "could not read subnets"
  SPCJSON="$(m spaces  read 2>/dev/null)" || refuse "could not read spaces"
  printf '%s' "$FABJSON" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null || refuse "fabrics payload is not JSON"
}
fab_id_byname() { printf '%s' "$FABJSON" | python3 -c '
import json,sys
n=sys.argv[1]
for f in json.load(sys.stdin):
    if f.get("name")==n: print(f["id"]); break
' "$1" 2>/dev/null; }
# The untagged (vid 0) VLAN of a fabric -- the one a flat plane rides.
fab_vlan_vid0() { printf '%s' "$FABJSON" | python3 -c '
import json,sys
n=sys.argv[1]
for f in json.load(sys.stdin):
    if f.get("name")==n:
        for v in f.get("vlans",[]):
            if v.get("vid")==0: print(v["id"]); break
        break
' "$1" 2>/dev/null; }
sub_id_bycidr() { printf '%s' "$SUBJSON" | python3 -c '
import json,sys
c=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get("cidr")==c: print(s["id"]); break
' "$1" 2>/dev/null; }
sub_vlan_bycidr() { printf '%s' "$SUBJSON" | python3 -c '
import json,sys
c=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get("cidr")==c: print(s["vlan"]["id"]); break
' "$1" 2>/dev/null; }
sub_fabric_bycidr() { printf '%s' "$SUBJSON" | python3 -c '
import json,sys
c=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get("cidr")==c: print(s["vlan"]["fabric"]); break
' "$1" 2>/dev/null; }
sub_gw_bycidr() { printf '%s' "$SUBJSON" | python3 -c '
import json,sys
c=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get("cidr")==c: print(s.get("gateway_ip") or ""); break
' "$1" 2>/dev/null; }
space_id_byname() { printf '%s' "$SPCJSON" | python3 -c '
import json,sys
n=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get("name")==n: print(s["id"]); break
' "$1" 2>/dev/null; }
vlan_space_of_sub() { printf '%s' "$SUBJSON" | python3 -c '
import json,sys
c=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get("cidr")==c:
        print(s["vlan"].get("space") or ""); break
' "$1" 2>/dev/null; }

fabric_name_for() { # $1 = plane name; "" means metal-admin keeps MAAS's own fabric
  [ "$1" = "metal-admin" ] && { echo ""; return; }
  echo "${SITE%%-*}-${DCLABEL}-$1"
}

# ---------------------------------------------------------------- check ----
do_check() {
  reload
  echo "dc-region-topology check: site=$SITE profile=$PROFILE"

  # 1. named plane fabrics (five; metal-admin excluded by design)
  for cidr in "${PLANE_CIDRS[@]}"; do
    plane="${PLANE_NAME[$cidr]}"
    fn="$(fabric_name_for "$plane")"
    [ -z "$fn" ] && continue
    id="$(fab_id_byname "$fn")"
    [ -n "$id" ]; ck $? "fabric '$fn' exists"
  done

  # 2. six spaces
  for sp in "${SPACES6[@]}"; do
    id="$(space_id_byname "$sp")"
    [ -n "$id" ]; ck $? "space '$sp' exists"
  done

  # 3. v4 plane subnets present, on the right fabric, bound to the right space
  for cidr in "${PLANE_CIDRS[@]}"; do
    plane="${PLANE_NAME[$cidr]}"
    sid="$(sub_id_bycidr "$cidr")"
    if [ -z "$sid" ]; then bad "subnet $cidr ($plane) exists"; continue; fi
    ok "subnet $cidr ($plane) exists"
    fn="$(fabric_name_for "$plane")"
    actual_fab="$(sub_fabric_bycidr "$cidr")"
    if [ -n "$fn" ]; then
      [ "$actual_fab" = "$fn" ]; ck $? "subnet $cidr is on fabric '$fn' (got '${actual_fab:-none}')"
    else
      # metal-admin: assert only that it is NOT sharing a fabric with a named
      # plane, which would collapse two L2 domains into one.
      case "$actual_fab" in
        "${SITE%%-*}-${DCLABEL}-"*) bad "metal-admin $cidr must NOT sit on a named plane fabric (got '$actual_fab')" ;;
        "") bad "metal-admin $cidr has no fabric" ;;
        *) ok "metal-admin $cidr is on MAAS's own fabric '$actual_fab' (by design)" ;;
      esac
    fi
    # space binding -- this is what Juju actually consumes
    sp="$(vlan_space_of_sub "$cidr")"
    [ "$sp" = "$plane" ]; ck $? "subnet $cidr VLAN is bound to space '$plane' (got '${sp:-none}')"
    # gateway: exactly the planes lib-net rules a gateway for, and no others
    want_gw="${PLANE_GW[$cidr]:-}"
    got_gw="$(sub_gw_bycidr "$cidr")"
    [ "$got_gw" = "$want_gw" ]; ck $? "subnet $cidr gateway '${want_gw:-none}' (got '${got_gw:-none}')"
  done

  # 4. metal-admin service config -- the piece that was runbook prose only
  masid="$(sub_id_bycidr "$METAL_ADMIN_CIDR")"
  if [ -z "$masid" ]; then
    bad "metal-admin subnet $METAL_ADMIN_CIDR exists (cannot check its service config)"
  else
    MAJSON="$(m subnet read "$masid" 2>/dev/null)"
    dns="$(printf '%s' "$MAJSON" | python3 -c 'import json,sys; print(",".join(json.load(sys.stdin).get("dns_servers") or []))' 2>/dev/null)"
    [ -n "$dns" ]; ck $? "metal-admin carries node-facing dns_servers (got '${dns:-none}') -- D-131"
    ad="$(printf '%s' "$MAJSON" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("allow_dns"))' 2>/dev/null)"
    [ "$ad" = "False" ]; ck $? "metal-admin allow_dns is False (got '$ad') -- D-131"
    dyn="$(m subnet reserved-ip-ranges "$masid" 2>/dev/null | python3 -c '
import json,sys
try: d=json.load(sys.stdin)
except Exception: sys.exit(1)
print(sum(1 for r in d if "dynamic" in (r.get("purpose") or [])))' 2>/dev/null)"
    [ "${dyn:-0}" -ge 1 ]; ck $? "metal-admin has a DHCP dynamic range (got ${dyn:-0})"
  fi

  # 5. the site placement tag the bundle and the v6 carve both depend on
  tags="$(m tags read 2>/dev/null | python3 -c 'import json,sys; print(" ".join(t["name"] for t in json.load(sys.stdin)))' 2>/dev/null)"
  case " $tags " in *" openstack-$SITE "*) ok "tag 'openstack-$SITE' exists" ;; *) bad "tag 'openstack-$SITE' exists (tags: ${tags:-none})" ;; esac

  echo "dc-region-topology: $P passed, $F failed"
  [ "$F" -eq 0 ] || return 1
  return 0
}

# ---------------------------------------------------------------- apply ----
say(){ if [ "$COMMIT" -eq 1 ]; then echo "  APPLY  $1"; else echo "  DRY    $1"; fi; }
do_apply() {
  reload
  echo "dc-region-topology apply: site=$SITE profile=$PROFILE commit=$COMMIT"

  # 1. named plane fabrics
  for cidr in "${PLANE_CIDRS[@]}"; do
    plane="${PLANE_NAME[$cidr]}"; fn="$(fabric_name_for "$plane")"
    [ -z "$fn" ] && continue
    if [ -n "$(fab_id_byname "$fn")" ]; then echo "  [idem] fabric '$fn' present"; continue; fi
    say "fabrics create name=$fn"
    [ "$COMMIT" -eq 1 ] && { m fabrics create name="$fn" >/dev/null 2>&1 || refuse "could not create fabric $fn"; reload
      [ -n "$(fab_id_byname "$fn")" ] || refuse "created fabric $fn but it did not read back"; }
  done

  # 2. spaces
  for sp in "${SPACES6[@]}"; do
    if [ -n "$(space_id_byname "$sp")" ]; then echo "  [idem] space '$sp' present"; continue; fi
    say "spaces create name=$sp"
    [ "$COMMIT" -eq 1 ] && { m spaces create name="$sp" >/dev/null 2>&1 || refuse "could not create space $sp"; reload
      [ -n "$(space_id_byname "$sp")" ] || refuse "created space $sp but it did not read back"; }
  done

  # 3. subnets: create missing, MOVE any that sit on the wrong fabric, set gw
  for cidr in "${PLANE_CIDRS[@]}"; do
    plane="${PLANE_NAME[$cidr]}"; fn="$(fabric_name_for "$plane")"
    want_gw="${PLANE_GW[$cidr]:-}"
    sid="$(sub_id_bycidr "$cidr")"
    if [ -z "$sid" ]; then
      [ -n "$fn" ] || refuse "metal-admin subnet $cidr is absent -- it is created by the rack's own registration, not by this script; commission the rack first"
      vid="$(fab_vlan_vid0 "$fn")"
      [ -n "$vid" ] || { [ "$COMMIT" -eq 1 ] && refuse "fabric $fn has no untagged VLAN"; vid="<vlan-of-$fn>"; }
      say "subnets create cidr=$cidr vlan=$vid ${want_gw:+gateway_ip=$want_gw}"
      if [ "$COMMIT" -eq 1 ]; then
        if [ -n "$want_gw" ]; then m subnets create cidr="$cidr" vlan="$vid" gateway_ip="$want_gw" >/dev/null 2>&1
        else m subnets create cidr="$cidr" vlan="$vid" >/dev/null 2>&1; fi
        [ $? -eq 0 ] || refuse "could not create subnet $cidr"
        reload; [ -n "$(sub_id_bycidr "$cidr")" ] || refuse "created subnet $cidr but it did not read back"
      fi
    else
      # present: is it on the right fabric?
      if [ -n "$fn" ]; then
        actual_fab="$(sub_fabric_bycidr "$cidr")"
        if [ "$actual_fab" != "$fn" ]; then
          vid="$(fab_vlan_vid0 "$fn")"
          # In a DRY run the target fabric may not exist yet (it is created earlier
          # in this same apply), so its VLAN id cannot resolve. Print an explicit
          # placeholder rather than an empty value -- an empty `vlan=` reads like a
          # defect and would hide a real one.
          say "subnet update $sid vlan=${vid:-<vlan-of-$fn>}   # MOVE $cidr off '$actual_fab' onto '$fn'"
          if [ "$COMMIT" -eq 1 ]; then
            [ -n "$vid" ] || refuse "fabric $fn has no untagged VLAN to move $cidr onto"
            m subnet update "$sid" vlan="$vid" >/dev/null 2>&1 || refuse "could not move subnet $cidr"
            reload
            [ "$(sub_fabric_bycidr "$cidr")" = "$fn" ] || refuse "moved subnet $cidr but it did not read back on $fn"
          fi
        else
          echo "  [idem] subnet $cidr already on '$fn'"
        fi
      fi
      got_gw="$(sub_gw_bycidr "$cidr")"
      if [ "$got_gw" != "$want_gw" ]; then
        say "subnet update $sid gateway_ip='${want_gw:-}'   # was '${got_gw:-none}'"
        if [ "$COMMIT" -eq 1 ]; then
          m subnet update "$sid" gateway_ip="$want_gw" >/dev/null 2>&1 || refuse "could not set gateway on $cidr"
          reload
          [ "$(sub_gw_bycidr "$cidr")" = "$want_gw" ] || refuse "set gateway on $cidr but it did not read back"
        fi
      fi
    fi
  done

  # 4. bind each plane's VLAN to its space
  reload
  for cidr in "${PLANE_CIDRS[@]}"; do
    plane="${PLANE_NAME[$cidr]}"
    cur="$(vlan_space_of_sub "$cidr")"
    [ "$cur" = "$plane" ] && { echo "  [idem] $cidr VLAN already in space '$plane'"; continue; }
    vlid="$(sub_vlan_bycidr "$cidr")"; spid="$(space_id_byname "$plane")"
    fabname="$(sub_fabric_bycidr "$cidr")"
    # Same DRY-run caveat as the MOVE above: on a from-scratch build the subnet
    # (and therefore its fabric/VLAN) may not exist yet.
    say "vlan update (fabric '${fabname:-<fabric-for-$plane>}' vlan ${vlid:-<vlan-for-$plane>}) space=$plane   # was '${cur:-none}'"
    if [ "$COMMIT" -eq 1 ]; then
      [ -n "$vlid" ] && [ -n "$spid" ] || refuse "cannot bind $cidr: vlan='$vlid' space='$spid'"
      fabid="$(fab_id_byname "$fabname")"
      vidnum="$(m vlans read "$fabid" 2>/dev/null | python3 -c '
import json,sys
t=sys.argv[1]
for v in json.load(sys.stdin):
    if str(v["id"])==t: print(v["vid"]); break
' "$vlid")"
      [ -n "$vidnum" ] || refuse "could not resolve the 802.1Q vid for vlan row $vlid"
      m vlan update "$fabid" "$vidnum" space="$plane" >/dev/null 2>&1 || refuse "could not bind $cidr to space $plane"
      reload
      [ "$(vlan_space_of_sub "$cidr")" = "$plane" ] || refuse "bound $cidr to $plane but it did not read back"
    fi
  done

  # 5. the site placement tag
  tags="$(m tags read 2>/dev/null | python3 -c 'import json,sys; print(" ".join(t["name"] for t in json.load(sys.stdin)))' 2>/dev/null)"
  case " $tags " in
    *" openstack-$SITE "*) echo "  [idem] tag 'openstack-$SITE' present" ;;
    *) say "tags create name=openstack-$SITE"
       [ "$COMMIT" -eq 1 ] && { m tags create name="openstack-$SITE" comment="VR1 $SITE placement tag (bundle constraint; dc-node-v6-carve site membership)" >/dev/null 2>&1 || refuse "could not create tag openstack-$SITE"
         t2="$(m tags read 2>/dev/null | python3 -c 'import json,sys; print(" ".join(t["name"] for t in json.load(sys.stdin)))' 2>/dev/null)"
         case " $t2 " in *" openstack-$SITE "*) ;; *) refuse "created tag openstack-$SITE but it did not read back" ;; esac; } ;;
  esac

  echo
  if [ "$COMMIT" -eq 1 ]; then
    echo "-- verifying --"; do_check; return $?
  fi
  echo "DRY RUN complete (no --commit). Nothing was written."
  echo "NOTE: metal-admin's dynamic range / dns_servers / DHCP are NOT set here --"
  echo "      they are the DHCP handover step and are deliberately separate."
  return 0
}

case "$ACTION" in
  check) do_check; exit $? ;;
  apply) do_apply; exit $? ;;
esac