Newer
Older
openstack-caracal-dc-dc / tests / site-headend-install / run-tests.sh
#!/usr/bin/env bash
# tests/site-headend-install/run-tests.sh
#
# Harness for scripts/site-headend-install.sh. Exercises arg parsing, the SAFETY GUARDS,
# and --dry-run only. It NEVER installs a snap, never touches MAAS/LXD, and never mutates
# the machine (same posture as tests/prereqs and the opnsense-* harnesses).
#
# The guards are the point of this harness. The script's whole reason to exist is that
# four specific traps cost a live session on 2026-07-13; a regression that quietly drops
# one of them would not show up in any deploy until something broke in production.
# ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
S="$(cd "$HERE/../../scripts" && pwd)/site-headend-install.sh"
pass=0; fail=0
ok()  { pass=$((pass+1)); }
bad() { fail=$((fail+1)); echo "  FAIL: $1"; }
t() { local d="$1" exp="$2"; shift 2; "$@" >/dev/null 2>&1; local rc=$?; if [ "$rc" = "$exp" ]; then ok; else bad "$d (rc=$rc want=$exp)"; fi; }

# 1. arg contract
t "--help -> 0"                 0 bash "$S" --help
t "unknown arg -> 2"            2 bash "$S" --bogus
t "missing --compose-cidr -> 2" 2 bash "$S" --dry-run
t "non-/24 cidr -> 2"           2 bash "$S" --dry-run --compose-cidr 10.10.1.0/16
t "garbage cidr -> 2"           2 bash "$S" --dry-run --compose-cidr not-a-cidr
t "valid --dry-run -> 0"        0 bash "$S" --dry-run --compose-cidr 10.10.1.0/24

# 2. --compose-cidr must have NO default. A default here is how you end up serving MAAS
#    DHCP on the site LAN, where the edge's Kea is authoritative (trap 4).
out="$(bash "$S" --dry-run 2>&1)"
printf '%s' "$out" | grep -q -- "--compose-cidr is REQUIRED" && ok || bad "missing --compose-cidr must fail LOUD and say why"

# 3. --dry-run must MUTATE NOTHING: no snap install, no lxc, no maas invoked.
out="$(bash "$S" --dry-run --compose-cidr 10.10.1.0/24 2>&1)"
printf '%s' "$out" | grep -q '\[dry-run\]' && ok || bad "--dry-run printed no [dry-run] plan"
if snap list lxd >/dev/null 2>&1 || snap list maas >/dev/null 2>&1; then
  bad "--dry-run appears to have INSTALLED a snap on the test host"
else ok; fi

# 4. the derived addresses must track --compose-cidr, not be hardcoded.
out="$(bash "$S" --dry-run --compose-cidr 10.77.5.0/24 2>&1)"
printf '%s' "$out" | grep -q "10.77.5.1/24"   && ok || bad "bridge IP not derived from --compose-cidr"
printf '%s' "$out" | grep -q "10.77.5.100"    && ok || bad "range start not derived from --compose-cidr"
printf '%s' "$out" | grep -q "10.77.5.200"    && ok || bad "range end not derived from --compose-cidr"

# 5. THE FOUR TRAPS must remain encoded in the script. Each of these cost a live session;
#    if someone "simplifies" one away, this harness goes red rather than the cloud.
grep -q 'raw.dnsmasq' "$S"                    && ok || bad "trap 1 LOST: raw.dnsmasq=port=0 is what stops dnsmasq binding :53 against MAAS's bind9"
grep -q 'ipv4.dhcp=false' "$S"                && ok || bad "trap 1 LOST: lxdbr0 must have LXD's own DHCP off"

# Fold backslash-continuations first, and strip comments + echo'd strings, so we test what
# the script EXECUTES rather than what it says. (Both of these bit this harness on its
# first run: a multi-line `lxc network create` reads as missing its redirect, and an echo
# that WARNS about `lxd init --auto` reads as calling it.)
CODE="$(sed -e 's/#.*$//' "$S" | sed -e ':a' -e '/\\$/{N; s/\\\n//; ta}' | grep -vE '^[[:space:]]*echo ')"

printf '%s' "$CODE" | grep -qE '\blxd init --auto\b' \
  && bad "trap 1 LOST: 'lxd init --auto' is EXECUTED -- it FAILS when MAAS's bind9 holds :53" || ok

n="$(printf '%s' "$CODE" | grep -cE '\blxc\b.*</dev/null')"
m="$(printf '%s' "$CODE" | grep -cE '\blxc\b')"
[ "$m" -ge 1 ] && [ "$n" = "$m" ] && ok || bad "trap 2 LOST: every lxc call must redirect </dev/null ($n of $m do) -- lxc reads stdin and EATS the piped script"
grep -q '5.21/stable' "$S"                    && ok || bad "trap 3 LOST: LXD must be pinned to the 5.21 LTS track (MAAS breaks on LXD >= 6.7)"
grep -q 'is ON for a subnet that is NOT the compose network' "$S" \
                                              && ok || bad "trap 4 LOST: the guard asserting MAAS DHCP is on NO other subnet is gone"
# trap 5: MAAS discovers the compose subnet from lxdbr0 with NO gateway. A deployed machine then
# gets a netplan with no default route -- it LOOKS healthy (DNS resolves; bind9 is link-local) but
# has no egress, and the first apt install on it hangs. Cost a redeploy on 2026-07-13.
grep -q 'subnet update .*gateway_ip' "$S"     && ok || bad "trap 5 LOST: gateway_ip is never set on the compose subnet -- deployed machines will have NO default route"

# 6. RACK ROLE (D-123: ONE region on Office1 + a rack controller per DC). --role rack enrolls
#    to the EXISTING region and must SKIP every region+rack concern (DB/LXD/compose). These
#    tests are the point of the rack mode: a regression that quietly ran a region step, or that
#    leaked the enrollment secret, would not surface until a live DC build.
RURL="http://10.10.0.20:5240/MAAS"
t "bad --role -> 2"                          2 bash "$S" --role bogus --dry-run
# rack REQUIRES --region-url + --enroll-secret-file (missing -> 2), mirroring --compose-cidr.
t "rack missing --region-url -> 2"           2 bash "$S" --role rack --dry-run --enroll-secret-file /nonexistent
t "rack missing --enroll-secret-file -> 2"   2 bash "$S" --role rack --dry-run --region-url "$RURL"
t "rack bad --region-url -> 2"               2 bash "$S" --role rack --dry-run --region-url ftp://x --enroll-secret-file /nonexistent
# --compose-cidr is NOT required in rack mode (LXD-compose is a region concern).
SECFILE="$(mktemp)"; SENTINEL="RACK_ENROLL_SENTINEL_9x7"; printf '%s\n' "$SENTINEL" > "$SECFILE"
trap 'rm -f "$SECFILE"' EXIT
t "rack --dry-run (no --compose-cidr) -> 0"  0 bash "$S" --role rack --dry-run --region-url "$RURL" --enroll-secret-file "$SECFILE"

rackout="$(bash "$S" --role rack --dry-run --region-url "$RURL" --enroll-secret-file "$SECFILE" 2>&1)"
printf '%s' "$rackout" | grep -q 'maas init rack' && ok || bad "rack --dry-run must print the 'maas init rack' step"
# The enrollment secret must NEVER appear in output (read from the file at run time, never echoed).
printf '%s' "$rackout" | grep -q "$SENTINEL" && bad "rack --dry-run LEAKED the enrollment secret into output" || ok
# rack mode must NOT run any region+rack / LXD / compose step.
for m in 'lxdbr0' 'region+rack' 'compose' 'vm-host'; do
  printf '%s' "$rackout" | grep -qi "$m" && bad "rack --dry-run printed a region+rack-only step: '$m'" || ok
done

# 7. BACKWARD COMPAT: with no --role, the default is region+rack and --compose-cidr is still
#    REQUIRED (a rack-mode regression must not relax the legacy contract).
t "default role still needs --compose-cidr -> 2" 2 bash "$S" --dry-run
defout="$(bash "$S" --dry-run --compose-cidr 10.10.1.0/24 2>&1)"
printf '%s' "$defout" | grep -q 'region+rack' && ok || bad "default (no --role) must still run the region+rack flow"

# 8. D-123 MODEL B node-host (--host-nodes): a DC rack that is ALSO the nested libvirt host for
#    the inner OpenTofu root. Must attach to rack mode only, set up nested KVM + inner pool, and
#    NOT leak the secret or regress the plain-rack contract.
t "--host-nodes without --role rack -> 2" 2 bash "$S" --host-nodes --dry-run
t "rack + --host-nodes --dry-run -> 0"    0 bash "$S" --role rack --host-nodes --dry-run --region-url "$RURL" --enroll-secret-file "$SECFILE"
nhout="$(bash "$S" --role rack --host-nodes --dry-run --region-url "$RURL" --enroll-secret-file "$SECFILE" 2>&1)"
# the node-host setup steps must appear (incl. the SEC-010 DC-LOCAL forward-drop, R3-F02/Phase D)
for step in 'node-host' 'nested=1' 'qemu-kvm' 'inner pool' 'AppArmor' 'libvirt group' 'SEC-010' 'FORWARD-drop'; do
  printf '%s' "$nhout" | grep -qi "$step" && ok || bad "node-host --dry-run missing step: '$step'"
done
# SEC-010 forward-drop must key on the transit interface, overridable via --transit-if
tiout="$(bash "$S" --role rack --host-nodes --transit-if wan0 --dry-run --region-url "$RURL" --enroll-secret-file "$SECFILE" 2>&1)"
printf '%s' "$tiout" | grep -q "wan0" && ok || bad "--transit-if override not reflected in the SEC-010 forward-drop"
# SEC-010 --check must not fail-open: it must verify the keyed transit interface ACTUALLY EXISTS,
# because nftables oifname/iifname with an absent name loads clean but matches NOTHING (advisor catch)
grep -q 'ip link show "$TRANSIT_IF"' "$S" && ok || bad "SEC-010 --check must verify the transit interface exists (nftables fail-open otherwise)"
# D-125 bridge-in (OBS-3): node-host must VERIFY the WAN bridge + uplink so the inner OPNsense WAN
# has real egress. The dry-run must surface the D-125 WAN-bridge section.
printf '%s' "$nhout" | grep -qi 'WAN bridge' && ok || bad "node-host --dry-run missing the D-125 WAN-bridge verify section"
# the uplink guard is the SAME fail-open class as the transit check: it must confirm the port is
# ENSLAVED to the bridge (not merely that a name exists)
grep -q 'master $WAN_BRIDGE' "$S" && ok || bad "D-125 WAN check must verify the uplink is enslaved to the bridge (fail-open otherwise)"
# --uplink-if must be overridable and reflected in the output
upout="$(bash "$S" --role rack --host-nodes --uplink-if wanup --dry-run --region-url "$RURL" --enroll-secret-file "$SECFILE" 2>&1)"
printf '%s' "$upout" | grep -q 'wanup' && ok || bad "--uplink-if override not reflected in the D-125 WAN-bridge check"
# br_netfilter CONSTRAINT: SEC-010's FORWARD-drop must stay interface-scoped; the script must warn
# against globalizing it (else bridged WAN frames die silently under br_netfilter)
grep -qiE 'never global|do NOT globalize|br_netfilter' "$S" && ok || bad "D-125 br_netfilter constraint (keep SEC-010 scoped, never global) must be recorded in the script"
# the stale "ip_forward forced ON by the inner NAT" rationale must be gone (bridge-in has no inner NAT)
grep -q 'forced ON by the inner' "$S" && bad "stale pre-D-125 rationale (ip_forward forced by inner NAT) still present -- bridge-in removed the inner NAT" || ok
# it must still enroll the rack AND must not leak the secret
printf '%s' "$nhout" | grep -q 'maas init rack' && ok || bad "node-host mode dropped the rack enrollment"
printf '%s' "$nhout" | grep -q "$SENTINEL" && bad "node-host --dry-run LEAKED the enrollment secret" || ok
# --dry-run must MUTATE NOTHING (the real "-> install ..." line only prints in a live run)
printf '%s' "$nhout" | grep -qE '^\s+->\s+install qemu' && bad "node-host --dry-run appears to have INSTALLED packages" || ok
# plain rack (no --host-nodes) must NOT print node-host steps (backward compat)
printf '%s' "$rackout" | grep -qi 'node-host' && bad "plain rack regressed: printed node-host steps without --host-nodes" || ok
# opnsense base is staged by opnsense-prep-image.sh, not fetched inline by this script
printf '%s' "$nhout" | grep -q 'opnsense-prep-image.sh' && ok || bad "node-host must delegate the opnsense base to opnsense-prep-image.sh"

echo
total=$((pass+fail))
if [ "$fail" -eq 0 ]; then echo "site-headend-install: $pass/$total PASS"; exit 0; fi
echo "site-headend-install: $fail/$total FAIL"; exit 1