#!/usr/bin/env bash
# scripts/site-headend-install.sh [--check|--dry-run] [--role <region+rack|rack>]
#     region+rack (default): --compose-cidr <CIDR> [--maas-url-ip <IP>]
#     rack:                   --region-url <http://REGION-IP:5240/MAAS> --enroll-secret-file <path>
#
# Installs a VR1 SITE HEADEND on the host it runs on. Two roles (D-123 rules ONE MAAS
# region on Office1 + a rack controller PER DC):
#
#   --role region+rack (DEFAULT -- unchanged legacy behavior): MAAS (region+rack) + LXD, with
#     that LXD registered back into MAAS as an LXD VM host so MAAS can COMPOSE the site's
#     non-stack service machines into it. This is D-114, VR0's proven `lxd` + `tailscale`
#     pattern applied per site. This is the Office1 (voffice1) build.
#
#   --role rack: a DC rack controller (vvr1-dc0) that ENROLLS to Office1's EXISTING region
#     (D-123). A rack runs NO PostgreSQL, does its own `maas init rack`, and composes
#     NOTHING -- the DC's OpenStack nodes are OpenTofu-created and DISCOVERED via the vcloud
#     virsh vm-host registered to the REGION (not composed here). It therefore SKIPS all of
#     the region DB / createadmin / LXD / vm-host / compose-DHCP work below. Enabling
#     DHCP-on-metal-admin is a REGION-side op (Office1's MAAS, naming this rack as
#     primary_rack) done as a RUNBOOK step -- NOT by this script (see rack note below).
#
# RUN IT ON THE SITE HOST, as root:
#   region+rack (voffice1):
#     ssh <site-host> 'sudo bash -s' -- --compose-cidr 10.10.1.0/24 < scripts/site-headend-install.sh
#   rack (vvr1-dc0): the region operator reads the enrollment secret from the REGION host's
#     /var/snap/maas/common/maas/secret and drops it into a 0600 file on this host first:
#     ssh <dc-rack> 'sudo bash -s' -- --role rack \
#         --region-url http://10.10.0.20:5240/MAAS --enroll-secret-file /root/region-enroll.secret \
#         < scripts/site-headend-install.sh
#
#   --check     report what is installed/configured; mutate nothing; exit 0 ok / 1 incomplete
#   --dry-run   print what it would do; mutate nothing
#   (no flag)   install + configure (idempotent; safe to re-run)
#
# EXIT: 0 ok | 1 check-failed | 2 bad args (incl. bad/missing --role args) | 3 unsupported OS
#     | 4 install failed (incl. rack: enrollment-secret file absent/empty at mutate time). ASCII + LF.
#
# =====================================================================================
# THE FOUR TRAPS THIS SCRIPT EXISTS TO ENCODE (all hit for real on 2026-07-13; do not
# "simplify" any of them away):
#
# 1. `lxd init --auto` FAILS OUTRIGHT on a host that also runs MAAS.
#    It creates lxdbr0 with LXD's own DHCP+DNS ON, which starts a dnsmasq; dnsmasq binds
#    the WILDCARD 0.0.0.0:53, and MAAS's bind9 (`named`) already holds :53. Verbatim:
#      Failed starting network: The DNS and DHCP service exited prematurely: exit status 2
#      ("dnsmasq: failed to create listening socket for 10.10.1.1: Address already in use")
#    Note the message names the BRIDGE address, which sends you hunting the wrong thing --
#    the real conflict is the wildcard bind. So: NEVER `lxd init --auto` here. Create the
#    bridge explicitly with DHCP/DNS off AND `raw.dnsmasq=port=0`, which stops dnsmasq
#    binding :53 at all. (`ipv4.dhcp=false` + `dns.mode=none` alone are NOT enough -- LXD
#    still spawns dnsmasq for a managed bridge, and it still tries to bind :53.)
#
# 2. `lxc` READS ITS CONFIG FROM STDIN. If you pipe this script to `bash -s` over ssh, the
#    FIRST `lxc` call with no stdin redirect EATS THE REST OF THE SCRIPT as YAML and dies
#    with `yaml: unmarshal errors`, silently truncating everything after it. Every `lxc`
#    call below therefore ends in `</dev/null`. Same class as the OPNsense tcsh trap.
#
# 3. LXD SNAP TRACK CHANGES ARE ONE-WAY -- you cannot downgrade a track. LXD MUST be
#    installed DIRECTLY onto 5.21/stable and never allowed to land on `latest` first,
#    because MAAS 3.6/3.7 is INCOMPATIBLE with LXD >= 6.7 (LXD 6.7 consolidated API
#    endpoints; MAAS's pinned pylxd 2.3.5 cannot speak to them; fixed in pylxd >= 2.3.9,
#    not yet in a MAAS release). Canonical's guidance: LXD <= 6.6, or the 5.21 LTS track.
#    https://discourse.maas.io/t/maas-incompatibility-with-lxd-6-7/15749
#    Bonus: `core.trust_password` still EXISTS in 5.21 and is GONE in 6.x -- the pin also
#    preserves the scriptable registration path.
#
# 4. MAAS DHCP IS ENABLED ON THE COMPOSE NETWORK ONLY -- NEVER on the site LAN. The site
#    LAN already has an authoritative DHCP server (Kea, on the OPNsense edge). Two DHCP
#    servers on one L2 is an intermittent failure that is genuinely unpleasant to
#    diagnose. --compose-cidr is REQUIRED and has no default precisely so this can never
#    be picked by accident, and the script REFUSES to touch DHCP on any other subnet.
#
# TRAPS 1-3 (LXD/dnsmasq) DO NOT APPLY IN --role rack: a rack installs no LXD, creates no
# bridge, and runs no dnsmasq, so the :53/:67 conflict and the 5.21-track pin are moot.
# Trap 4's SPIRIT is kept absolutely: rack mode configures NO DHCP at all (metal-admin DHCP
# is a REGION-side op, not this script's), so it cannot fight the edge's Kea by construction.
# =====================================================================================
set -uo pipefail

CHECK=0; DRYRUN=0; COMPOSE_CIDR=""; MAAS_IP=""
ROLE="region+rack"; REGION_URL=""; ENROLL_SECRET_FILE=""
# D-123 MODEL B: a DC rack (vvr1-dc0) is ALSO the nested libvirt/KVM host for the INNER
# OpenTofu root that builds the node fleet (opentofu/vr1-dc0-substrate/). --host-nodes adds
# that setup to the rack install (make it a libvirt host + nested KVM + inner pool + base
# image) so the qemu+ssh inner provider (R-5) has something to connect to.
HOST_NODES=0; HOST_USER="jessea123"; INNER_POOL_PATH="/var/lib/libvirt/vr1-dc0-inner"; OPNSENSE_BASE=""
# SEC-010: the transit-leg interface name on vvr1-dc0 (the outer NIC; netplan set-name in the
# outer cloud-init is "mgmt"). The DC-LOCAL FORWARD-drop is keyed on it.
TRANSIT_IF="mgmt"
while [ $# -gt 0 ]; do
  case "$1" in
    --check)              CHECK=1 ;;
    --dry-run)            DRYRUN=1 ;;
    --role)               shift; ROLE="${1:-}" ;;
    --compose-cidr)       shift; COMPOSE_CIDR="${1:-}" ;;
    --maas-url-ip)        shift; MAAS_IP="${1:-}" ;;
    --region-url)         shift; REGION_URL="${1:-}" ;;
    --enroll-secret-file) shift; ENROLL_SECRET_FILE="${1:-}" ;;
    --host-nodes)         HOST_NODES=1 ;;
    --host-user)          shift; HOST_USER="${1:-}" ;;
    --inner-pool-path)    shift; INNER_POOL_PATH="${1:-}" ;;
    --opnsense-base)      shift; OPNSENSE_BASE="${1:-}" ;;
    --transit-if)         shift; TRANSIT_IF="${1:-}" ;;
    -h|--help)
      echo "usage: site-headend-install.sh [--check|--dry-run] [--role <region+rack|rack>]"
      echo "  region+rack (default): --compose-cidr <CIDR> [--maas-url-ip <IP>]"
      echo "  rack:                  --region-url <http://REGION-IP:5240/MAAS> --enroll-secret-file <path>"
      echo "  rack + Model B node host (vvr1-dc0): add --host-nodes [--host-user U]"
      echo "                          [--inner-pool-path P] [--opnsense-base QCOW2]"
      exit 0 ;;
    *) echo "FAIL: unknown arg '$1'" >&2; exit 2 ;;
  esac
  shift
done
# --host-nodes only makes sense on a DC rack (the Model B node host); reject on region+rack.
[ "$HOST_NODES" = "1" ] && [ "$ROLE" != "rack" ] && { echo "FAIL: --host-nodes requires --role rack (it is the D-123 Model B DC node host)" >&2; exit 2; }
[ -n "$OPNSENSE_BASE" ] || OPNSENSE_BASE="$INNER_POOL_PATH/opnsense-26.1-nano.qcow2"

case "$ROLE" in
  region+rack|rack) ;;
  *) echo "FAIL: --role must be 'region+rack' (default) or 'rack' (got '$ROLE')" >&2; exit 2 ;;
esac

# Per-role REQUIRED args are validated UNCONDITIONALLY here (mirrors the legacy --compose-cidr
# check, which already gated --check too), so a bad invocation is rejected before any mode runs.
if [ "$ROLE" = "rack" ]; then
  # rack enrolls to Office1's EXISTING region -- no compose network (LXD-compose is a region
  # concern); it needs the region URL + the region's enrollment secret (read from a file, never
  # echoed -- same discipline as $SECRETS below; the region operator obtains it from the region
  # host's /var/snap/maas/common/maas/secret).
  [ -n "$REGION_URL" ] || { echo "FAIL: --role rack REQUIRES --region-url <http://REGION-IP:5240/MAAS>" >&2; exit 2; }
  echo "$REGION_URL" | grep -qiE '^https?://.+/MAAS/?$' || { echo "FAIL: --region-url must look like http://REGION-IP:5240/MAAS (got '$REGION_URL')" >&2; exit 2; }
  [ -n "$ENROLL_SECRET_FILE" ] || { echo "FAIL: --role rack REQUIRES --enroll-secret-file <path> (the region's enrollment secret; NEVER passed on the command line)" >&2; exit 2; }
  # --compose-cidr is NOT used in rack mode; ignore it if present.
else
  [ -n "$COMPOSE_CIDR" ] || { echo "FAIL: --compose-cidr is REQUIRED (e.g. 10.10.1.0/24). It has no default on purpose: MAAS must serve DHCP on the compose network ONLY, never on the site LAN, which the edge's Kea already owns." >&2; exit 2; }
  echo "$COMPOSE_CIDR" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.0/24$' || { echo "FAIL: --compose-cidr must be a /24 like 10.10.1.0/24 (got '$COMPOSE_CIDR')" >&2; exit 2; }

  BASE="${COMPOSE_CIDR%.0/24}"           # 10.10.1
  BRIDGE_IP="${BASE}.1"
  RANGE_LO="${BASE}.100"
  RANGE_HI="${BASE}.200"
fi
SECRETS=/root/maas-secrets
MAAS_CHANNEL="3.7/stable"
PG_CHANNEL="16/stable"
LXD_CHANNEL="5.21/stable"             # TRAP 3 -- do not change without reading it

run() { # run <desc> -- <cmd...>
  local d="$1"; shift; [ "${1:-}" = "--" ] && shift
  if [ "$DRYRUN" = "1" ]; then printf '  [dry-run] %s\n            $ %s\n' "$d" "$*"; return 0; fi
  echo "  -> $d"; "$@"
}
have() { command -v "$1" >/dev/null 2>&1; }

# ---------------- check mode (read-only) ----------------
report() {
  local bad=0
  snap list maas       >/dev/null 2>&1 && echo "OK: maas snap $(snap list maas 2>/dev/null | awk 'NR>1{print $2" ("$4")"}')"     || { echo "ABSENT: maas snap"; bad=1; }
  snap list postgresql >/dev/null 2>&1 && echo "OK: postgresql $(snap list postgresql 2>/dev/null | awk 'NR>1{print $2}')"       || { echo "ABSENT: postgresql snap"; bad=1; }
  if snap list lxd >/dev/null 2>&1; then
    local tr; tr="$(snap list lxd 2>/dev/null | awk 'NR>1{print $4}')"
    if [ "$tr" = "$LXD_CHANNEL" ]; then echo "OK: lxd on $tr"
    else echo "WRONG TRACK: lxd is on '$tr', must be '$LXD_CHANNEL' (MAAS breaks on LXD >= 6.7; track changes are ONE-WAY -- a reinstall is needed)"; bad=1; fi
  else echo "ABSENT: lxd snap"; bad=1; fi

  if have lxc && lxc network show lxdbr0 </dev/null >/dev/null 2>&1; then
    echo "OK: lxdbr0 exists (ipv4.dhcp=$(lxc network get lxdbr0 ipv4.dhcp </dev/null 2>/dev/null), dns.mode=$(lxc network get lxdbr0 dns.mode </dev/null 2>/dev/null))"
    local n53 n67
    n53=$(ss -lunp 2>/dev/null | grep dnsmasq | grep -c ':53 ')
    n67=$(ss -lunp 2>/dev/null | grep dnsmasq | grep -c ':67 ')
    if [ "${n53:-0}" = "0" ] && [ "${n67:-0}" = "0" ]; then
      echo "OK: dnsmasq binds NO :53 and NO :67 -- MAAS and the edge's Kea remain sole DHCP/DNS authorities"
    else
      echo "FAIL: dnsmasq holds :53=$n53 :67=$n67 -- it is FIGHTING MAAS/Kea (see trap 1)"; bad=1
    fi
  else echo "ABSENT: lxdbr0"; bad=1; fi

  if have maas && maas status 2>/dev/null | grep -q regiond; then echo "OK: MAAS initialised"; else echo "ABSENT: MAAS not initialised"; bad=1; fi
  return "$bad"
}

# ---------------- check mode, RACK role (read-only) ----------------
# A rack runs NO PostgreSQL and NO LXD, so we assert only: the maas snap is present and
# `maas status` shows `rackd` running. If the rack has been through `maas init rack`, the
# region URL it enrolled to is recorded (offline-readable) in the rackd config -- report it
# as the connected-to-region signal, best-effort (never fatal; the file is root-only).
report_rack() {
  local bad=0
  snap list maas >/dev/null 2>&1 && echo "OK: maas snap $(snap list maas 2>/dev/null | awk 'NR>1{print $2" ("$4")"}')" || { echo "ABSENT: maas snap"; bad=1; }
  if have maas && maas status 2>/dev/null | grep -q rackd; then echo "OK: MAAS rackd running"; else echo "ABSENT: MAAS rackd not running (not enrolled to a region)"; bad=1; fi
  local ru; ru="$(grep -sh -m1 -E '^maas_url:' /var/snap/maas/current/rackd.conf 2>/dev/null | awk '{print $2}')"
  [ -n "$ru" ] && echo "OK: enrolled to region $ru"
  return "$bad"
}

# Dispatch check/precheck to the role-appropriate reporter.
# D-123 Model B node-host: report readiness of vvr1-dc0 as a nested libvirt host (--check).
node_host_check() {
  local ok=0
  echo "== node-host (Model B) readiness =="
  have virsh && echo "  [ok] libvirt/qemu installed" || { echo "  [--] libvirt/qemu NOT installed"; ok=1; }
  local kmod=""; [ -d /sys/module/kvm_amd ] && kmod=kvm_amd; [ -d /sys/module/kvm_intel ] && kmod=kvm_intel
  if [ -n "$kmod" ] && grep -qiE '^(Y|1)$' "/sys/module/$kmod/parameters/nested" 2>/dev/null; then
    echo "  [ok] nested KVM ON ($kmod)"
  else echo "  [--] nested KVM OFF (kmod=${kmod:-none}) -- inner node VMs get no /dev/kvm"; ok=1; fi
  id -nG "$HOST_USER" 2>/dev/null | grep -qw libvirt && echo "  [ok] $HOST_USER in libvirt group" || { echo "  [--] $HOST_USER NOT in libvirt group"; ok=1; }
  [ -d "$INNER_POOL_PATH" ] && echo "  [ok] inner pool dir $INNER_POOL_PATH" || { echo "  [--] inner pool dir $INNER_POOL_PATH missing"; ok=1; }
  [ -s "$OPNSENSE_BASE" ] && echo "  [ok] opnsense base $OPNSENSE_BASE" || { echo "  [--] opnsense base missing -- run: opnsense-prep-image.sh 26.1 $OPNSENSE_BASE"; ok=1; }
  # SEC-010 mechanical pre-apply gate: the transit FORWARD-drop must be present (DC-LOCAL, D-052/D-100)
  # AND must key on an interface that ACTUALLY EXISTS -- nftables oifname/iifname with an absent name
  # loads clean but matches NOTHING (fail-open), which would silently leave DC-LOCAL unenforced.
  if ! nft list table inet sec010 >/dev/null 2>&1; then
    echo "  [--] SEC-010 FORWARD-drop MISSING -- DC-LOCAL NOT enforced (metal-admin could route across the fiber)"; ok=1
  elif ! ip link show "$TRANSIT_IF" >/dev/null 2>&1; then
    echo "  [--] SEC-010 rule keys on '$TRANSIT_IF' but NO such interface -- rule matches NOTHING (FAIL-OPEN); confirm the transit interface name (--transit-if)"; ok=1
  else
    echo "  [ok] SEC-010 transit FORWARD-drop present + transit interface '$TRANSIT_IF' exists"
  fi
  return $ok
}

# D-123 Model B node-host: make this rack (vvr1-dc0) a nested libvirt/KVM host so the inner
# OpenTofu root (opentofu/vr1-dc0-substrate/, qemu+ssh from Office1, R-5) can build the fleet.
node_host_setup() {
  echo "== node-host (D-123 Model B): make this rack a nested libvirt/KVM host for the inner tofu =="
  run "install qemu-kvm + libvirt + qemu-utils" -- apt-get install -y qemu-kvm libvirt-daemon-system qemu-utils || exit 4
  run "enable libvirtd" -- systemctl enable --now libvirtd
  # nested KVM (persist + verify): inner guests need /dev/kvm, which needs the host module's
  # nested=1 AND the outer tofu's expose_nested_virt=true on vvr1-dc0.
  local kmod=""; [ -d /sys/module/kvm_amd ] && kmod=kvm_amd; [ -d /sys/module/kvm_intel ] && kmod=kvm_intel
  [ -n "$kmod" ] || { echo "FAIL: no kvm_amd/kvm_intel -- is expose_nested_virt=true on vvr1-dc0? (outer tofu)" >&2; exit 4; }
  run "persist nested=1 for $kmod" -- bash -c "echo 'options $kmod nested=1' > /etc/modprobe.d/kvm-nested.conf"
  if ! grep -qiE '^(Y|1)$' "/sys/module/$kmod/parameters/nested" 2>/dev/null; then
    run "reload $kmod for nested=1" -- bash -c "modprobe -r $kmod 2>/dev/null; modprobe $kmod" \
      || echo "  WARN: could not reload $kmod (VMs running?) -- REBOOT vvr1-dc0 to apply nested=1"
  fi
  grep -qiE '^(Y|1)$' "/sys/module/$kmod/parameters/nested" 2>/dev/null \
    && echo "  -> nested KVM ON ($kmod)" || echo "  WARN: nested KVM still OFF -- reboot vvr1-dc0 (inner nova will not boot until ON)"
  run "add $HOST_USER to libvirt group" -- usermod -aG libvirt "$HOST_USER"
  # inner pool dir + AppArmor grant (platform-traps: a non-default libvirt pool path otherwise
  # yields qemu 'Permission denied' define-then-fail on the inner domains -- a silent dead guest).
  run "create inner pool dir $INNER_POOL_PATH" -- mkdir -p "$INNER_POOL_PATH"
  run "AppArmor: allow inner pool path" -- bash -c "install -d /etc/apparmor.d/local/abstractions 2>/dev/null; grep -qF '$INNER_POOL_PATH' /etc/apparmor.d/local/abstractions/libvirt-qemu 2>/dev/null || echo '  \"$INNER_POOL_PATH/**\" rwk,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu; systemctl reload apparmor 2>/dev/null || true"
  # SEC-010 (D-052/D-100 DC-LOCAL under Model B): ip_forward is forced ON by the inner vr1-dc0-wan
  # NAT, so a global ip_forward=0 is unusable. Enforce DC-local with a FORWARD-drop across the TRANSIT
  # leg -- nothing routes ACROSS the fiber THROUGH vvr1-dc0 (the rack proxies MAAS at the app layer =
  # originated/terminated, NOT forwarded). vvr1-dc0's own region traffic (OUTPUT/INPUT) is unaffected.
  echo "== SEC-010 (DC-LOCAL): FORWARD-drop across the transit leg '$TRANSIT_IF' =="
  run "install nftables" -- apt-get install -y nftables
  if [ "$DRYRUN" = "1" ]; then
    echo "  [dry-run] write /etc/nftables-sec010.nft (FORWARD-drop in+out '$TRANSIT_IF') + enable sec010-fw.service (boot-persistent)"
  else
    cat > /etc/nftables-sec010.nft <<NFT
#!/usr/sbin/nft -f
table inet sec010 {
  chain forward {
    type filter hook forward priority -10; policy accept;
    oifname "$TRANSIT_IF" drop
    iifname "$TRANSIT_IF" drop
  }
}
NFT
    nft -f /etc/nftables-sec010.nft && echo "  -> SEC-010 FORWARD-drop loaded (transit leg '$TRANSIT_IF')"
    cat > /etc/systemd/system/sec010-fw.service <<UNIT
[Unit]
Description=SEC-010 DC-LOCAL forward-drop across the transit leg (D-052/D-100)
After=nftables.service network-pre.target
[Service]
Type=oneshot
ExecStart=/usr/sbin/nft -f /etc/nftables-sec010.nft
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
UNIT
    systemctl daemon-reload; systemctl enable sec010-fw.service >/dev/null 2>&1 && echo "  -> sec010-fw.service enabled (boot-persistent)"
  fi

  # OPNsense base for the inner edge (opnsense-prep-image.sh needs network; run on this host).
  # Verify, do NOT fetch inline (the prep script is not guaranteed present here).
  if [ -s "$OPNSENSE_BASE" ]; then echo "  -> opnsense base present: $OPNSENSE_BASE"
  else echo "  NOTE: opnsense base MISSING at $OPNSENSE_BASE -- run on this host BEFORE the inner apply:"; echo "        bash scripts/opnsense-prep-image.sh 26.1 $OPNSENSE_BASE"; fi
  echo "  -> node-host ready. Inner apply next (from Office1): cd opentofu/vr1-dc0-substrate && tofu apply."
}

do_report() {
  if [ "$ROLE" = "rack" ]; then
    report_rack; local r=$?
    if [ "$HOST_NODES" = "1" ]; then node_host_check || r=1; fi
    return $r
  else report; fi
}

report_st=0
if [ "$CHECK" = "1" ]; then do_report; exit $?; fi
do_report >/dev/null 2>&1; report_st=$?
[ "$report_st" = "0" ] && [ "$DRYRUN" = "0" ] && { echo "-- site headend already installed and configured --"; exit 0; }

have apt-get || { echo "FAIL: Debian/Ubuntu (apt) only." >&2; exit 3; }
# root is required to MUTATE, not to plan: --dry-run stays runnable by anyone (and by the harness).
[ "$DRYRUN" = "1" ] || [ "$(id -u)" = "0" ] || { echo "FAIL: must run as root (use: ssh <host> 'sudo bash -s' -- ... < $0)" >&2; exit 4; }

# =====================================================================================
# RACK ROLE (D-123): enroll a DC rack controller to Office1's EXISTING region. ONLY these
# steps run -- and NONE of the region+rack work below (no PostgreSQL, no `maas init
# region+rack`, no createadmin/apikey/login, no LXD, no LXD vm-host registration, no
# compose-network DHCP). Rationale: a rack runs no DB, composes nothing (the DC's OpenStack
# nodes are OpenTofu-created + discovered via the vcloud virsh vm-host registered to the
# REGION, not composed here), and DHCP-on-metal-admin is a REGION-side op (enabled against
# Office1's MAAS naming this rack as primary_rack -- a runbook step, NOT this script). So
# traps 1-3 (LXD/dnsmasq) do not apply, and trap 4's spirit holds: rack mode touches NO DHCP.
# =====================================================================================
if [ "$ROLE" = "rack" ]; then
  echo "== 1. time: MAAS manages time via chrony; systemd-timesyncd conflicts =="
  run "disable systemd-timesyncd" -- systemctl disable --now systemd-timesyncd >/dev/null 2>&1

  echo "== 2. MAAS $MAAS_CHANNEL (rack: NO PostgreSQL -- the rack runs no DB) =="
  snap list maas >/dev/null 2>&1 || run "install maas" -- snap install maas --channel="$MAAS_CHANNEL" || exit 4

  echo "== 3. maas init rack -- enroll to the region at $REGION_URL =="
  # `maas init rack --maas-url <URL> --secret <SECRET>` is the MAAS snap rack-registration
  # invocation -- CONFIRMED, not invented: this form is documented across the MAAS snap docs and
  # discourse and is stable across 3.x (so it holds for our 3.7/stable pin), and the region's
  # shared secret lives at /var/snap/maas/common/maas/secret on the region host (matches the
  # runbook). The secret is READ from the file at
  # execution time and NEVER echoed -- so this step is hand-rolled rather than routed through
  # run() (which would print the expanded command, leaking the secret in --dry-run).
  if [ "$DRYRUN" = "0" ]; then
    [ -s "$ENROLL_SECRET_FILE" ] || { echo "FAIL: --enroll-secret-file '$ENROLL_SECRET_FILE' is absent or empty (need the region's enrollment secret)" >&2; exit 4; }
    if maas status 2>/dev/null | grep -q rackd; then
      echo "  -> already enrolled (rackd running)"
    else
      maas init rack --maas-url "$REGION_URL" --secret "$(cat "$ENROLL_SECRET_FILE")" >/dev/null 2>&1 \
        || { echo "FAIL: maas init rack" >&2; exit 4; }
      echo "  -> rack registered to region $REGION_URL"
    fi
  else
    echo "  [dry-run] maas init rack --maas-url $REGION_URL --secret <read from $ENROLL_SECRET_FILE at run time; never printed>"
  fi

  # SKIPPED IN RACK MODE (region concerns): PostgreSQL, maas init region+rack,
  # createadmin/apikey/login, LXD (5.21 pin), LXD vm-host registration, compose-network DHCP.
  # metal-admin DHCP (this rack as primary_rack) is enabled REGION-side as a runbook step.
  # D-123 MODEL B: if this DC rack is ALSO the node host, set up nested libvirt (respects --dry-run).
  [ "$HOST_NODES" = "1" ] && node_host_setup
  if [ "$DRYRUN" = "1" ]; then echo "  (dry-run: no changes made)"; exit 0; fi
  echo
  echo "RACK ENROLLED to $REGION_URL. Next (REGION-side runbook, NOT this script): enable"
  echo "DHCP on metal-admin against Office1's MAAS, naming this host as primary_rack."
  exit 0
fi

# ---- region+rack role (default; unchanged legacy behavior) ----
[ -n "$MAAS_IP" ] || MAAS_IP="$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}')"
[ -n "$MAAS_IP" ] || { echo "FAIL: could not measure this host's outbound IP; pass --maas-url-ip" >&2; exit 4; }

install -d -m 0700 "$SECRETS"
newpass() { # newpass <file> -- generate once, 0600, NEVER echoed
  [ -s "$1" ] && return 0
  openssl rand -base64 30 | tr -dc 'A-Za-z0-9' | head -c 28 > "$1"; chmod 0600 "$1"
}

echo "== 1. time: MAAS manages time via chrony; systemd-timesyncd conflicts =="
run "disable systemd-timesyncd" -- systemctl disable --now systemd-timesyncd >/dev/null 2>&1

echo "== 2. MAAS $MAAS_CHANNEL + PostgreSQL $PG_CHANNEL =="
snap list maas       >/dev/null 2>&1 || run "install maas"       -- snap install maas --channel="$MAAS_CHANNEL"       || exit 4
snap list postgresql >/dev/null 2>&1 || run "install postgresql" -- snap install postgresql --channel="$PG_CHANNEL"   || exit 4

echo "== 3. MAAS database (password generated here, 0600, never printed) =="
if [ "$DRYRUN" = "0" ]; then
  newpass "$SECRETS/db.pass"; DBPASS="$(cat "$SECRETS/db.pass")"
  postgresql.psql -U postgres -h /tmp -tAc "SELECT 1 FROM pg_roles WHERE rolname='maas'" 2>/dev/null | grep -q 1 \
    || postgresql.psql -U postgres -h /tmp -c "CREATE ROLE maas LOGIN PASSWORD '$DBPASS'" >/dev/null 2>&1
  postgresql.psql -U postgres -h /tmp -tAc "SELECT 1 FROM pg_database WHERE datname='maas'" 2>/dev/null | grep -q 1 \
    || postgresql.createdb -U postgres -h /tmp -O maas maas >/dev/null 2>&1
  echo "  -> role + database 'maas' present"
else
  echo "  [dry-run] create role+db 'maas' with a generated password"
fi

echo "== 4. maas init region+rack =="
if [ "$DRYRUN" = "0" ]; then
  if maas status 2>/dev/null | grep -q regiond; then
    echo "  -> already initialised"
  else
    maas init region+rack --maas-url "http://${MAAS_IP}:5240/MAAS" \
      --database-uri "postgres://maas:$(cat "$SECRETS/db.pass")@localhost/maas" --force >/dev/null 2>&1 \
      || { echo "FAIL: maas init" >&2; exit 4; }
    echo "  -> initialised"
  fi
  newpass "$SECRETS/admin.pass"
  maas apikey --username admin >/dev/null 2>&1 || \
    maas createadmin --username admin --password "$(cat "$SECRETS/admin.pass")" --email "admin@$(hostname)" >/dev/null 2>&1
  maas apikey --username admin > "$SECRETS/admin.apikey" 2>/dev/null; chmod 0600 "$SECRETS/admin.apikey"
  for i in $(seq 1 60); do curl -sf -o /dev/null --max-time 3 "http://${MAAS_IP}:5240/MAAS/" && break; sleep 5; done
  maas login admin "http://${MAAS_IP}:5240/MAAS" "$(cat "$SECRETS/admin.apikey")" >/dev/null 2>&1
  echo "  -> admin ready, CLI logged in"
else
  echo "  [dry-run] maas init region+rack + createadmin + login"
fi

echo "== 5. LXD, DIRECTLY onto $LXD_CHANNEL (trap 3: track changes are ONE-WAY) =="
if snap list lxd >/dev/null 2>&1; then
  tr="$(snap list lxd 2>/dev/null | awk 'NR>1{print $4}')"
  [ "$tr" = "$LXD_CHANNEL" ] || { echo "FAIL: lxd already on track '$tr'; you CANNOT downgrade a snap track. Remove it (snap remove lxd) and re-run." >&2; exit 4; }
else
  run "install lxd" -- snap install lxd --channel="$LXD_CHANNEL" || exit 4
  run "hold lxd auto-refresh" -- snap refresh --hold lxd >/dev/null 2>&1
fi
[ "$DRYRUN" = "0" ] && lxd waitready --timeout=90

echo "== 6. LXD storage + compose bridge (built explicitly; see trap 1 -- the auto-init path FAILS here) =="
if [ "$DRYRUN" = "0" ]; then
  # dir driver: conservative inside an already-nested guest (no zfs loop file).
  lxc storage show default </dev/null >/dev/null 2>&1 || lxc storage create default dir </dev/null >/dev/null 2>&1
  if ! lxc network show lxdbr0 </dev/null >/dev/null 2>&1; then
    lxc network create lxdbr0 \
      ipv4.address="${BRIDGE_IP}/24" ipv4.nat=true ipv4.dhcp=false \
      ipv6.address=none dns.mode=none raw.dnsmasq="port=0" </dev/null >/dev/null 2>&1 \
      || { echo "FAIL: could not create lxdbr0 (see trap 1)" >&2; exit 4; }
  fi
  lxc profile device remove default eth0 </dev/null >/dev/null 2>&1
  lxc profile device add default root disk path=/ pool=default </dev/null >/dev/null 2>&1
  lxc profile device add default eth0 nic network=lxdbr0 name=eth0 </dev/null >/dev/null 2>&1
  lxc config set core.https_address '[::]:8443' </dev/null
  newpass "$SECRETS/lxd-trust.pass"
  lxc config set core.trust_password "$(cat "$SECRETS/lxd-trust.pass")" </dev/null
  echo "  -> lxdbr0 ${BRIDGE_IP}/24, NAT on, LXD DHCP/DNS OFF; API on :8443"

  n53=$(ss -lunp 2>/dev/null | grep dnsmasq | grep -c ':53 ')
  n67=$(ss -lunp 2>/dev/null | grep dnsmasq | grep -c ':67 ')
  [ "${n53:-0}" = "0" ] && [ "${n67:-0}" = "0" ] \
    || { echo "FAIL: dnsmasq is holding :53=$n53 :67=$n67 -- it will fight MAAS/Kea (trap 1)" >&2; exit 4; }
  echo "  -> VERIFIED: dnsmasq binds no :53 and no :67"
else
  echo "  [dry-run] create dir pool + lxdbr0 ${BRIDGE_IP}/24 (dhcp off, dns off, raw.dnsmasq=port=0), expose API"
fi

echo "== 7. register LXD into MAAS as a VM host =="
if [ "$DRYRUN" = "0" ]; then
  if maas admin vm-hosts read 2>/dev/null | grep -q '"type": "lxd"'; then
    echo "  -> LXD vm-host already registered"
  else
    maas admin vm-hosts create type=lxd name="$(hostname)-lxd" \
      power_address="https://${MAAS_IP}:8443" project=default \
      password="$(cat "$SECRETS/lxd-trust.pass")" >/dev/null 2>&1 \
      && echo "  -> registered as $(hostname)-lxd" || { echo "FAIL: vm-hosts create" >&2; exit 4; }
  fi
else
  echo "  [dry-run] maas admin vm-hosts create type=lxd power_address=https://${MAAS_IP}:8443"
fi

echo "== 8. MAAS DHCP on the COMPOSE network ONLY (trap 4) =="
if [ "$DRYRUN" = "0" ]; then
  SUB_JSON="$(maas admin subnets read 2>/dev/null)"
  SUB_ID="$(printf '%s' "$SUB_JSON" | python3 -c "
import sys,json
t=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get('cidr')==t: print(s['id']); break
" "$COMPOSE_CIDR")"
  [ -n "$SUB_ID" ] || { echo "FAIL: MAAS has not discovered $COMPOSE_CIDR (is lxdbr0 up?)" >&2; exit 4; }
  FAB_ID="$(printf '%s' "$SUB_JSON" | python3 -c "
import sys,json
t=sys.argv[1]
for s in json.load(sys.stdin):
    if s.get('cidr')==t: print(s['vlan']['fabric_id']); break
" "$COMPOSE_CIDR")"
  RACK="$(maas admin rack-controllers read 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)[0]['system_id'])")"

  # GATEWAY: MAAS auto-discovers the compose subnet from lxdbr0's interface, and that discovery
  # carries NO gateway. Without this, a DEPLOYED machine gets a netplan with NO DEFAULT ROUTE --
  # it comes up looking fine (DNS even resolves, because MAAS's bind9 is link-local) but has NO
  # EGRESS, so the first `apt install` on it hangs. lxdbr0 IS the gateway (it does the NAT).
  # Measured 2026-07-13: office1-netbox deployed with gateway_ip=None and had to be redeployed.
  maas admin subnet update "$SUB_ID" gateway_ip="$BRIDGE_IP" >/dev/null 2>&1 \
    && echo "  -> gateway_ip=$BRIDGE_IP set on $COMPOSE_CIDR"

  maas admin ipranges create type=dynamic subnet="$SUB_ID" start_ip="$RANGE_LO" end_ip="$RANGE_HI" \
    comment="MAAS DHCP/PXE for LXD-composed VMs (D-114)" >/dev/null 2>&1
  maas admin vlan update "$FAB_ID" 0 dhcp_on=True primary_rack="$RACK" >/dev/null 2>&1 \
    && echo "  -> DHCP ON for $COMPOSE_CIDR ($RANGE_LO-$RANGE_HI), rack=$RACK"

  # HARD GUARD: assert MAAS did not turn DHCP on for any OTHER subnet.
  BADSUB="$(maas admin subnets read 2>/dev/null | python3 -c "
import sys,json
t=sys.argv[1]
bad=[s['cidr'] for s in json.load(sys.stdin) if s['vlan'].get('dhcp_on') and s.get('cidr')!=t]
print(','.join(bad))
" "$COMPOSE_CIDR")"
  if [ -n "$BADSUB" ]; then
    echo "FAIL: MAAS DHCP is ON for a subnet that is NOT the compose network: $BADSUB" >&2
    echo "      The site LAN's DHCP belongs to the edge's Kea. Turn this OFF before proceeding." >&2
    exit 4
  fi
  echo "  -> VERIFIED: MAAS DHCP is on for $COMPOSE_CIDR and NO other subnet"
else
  echo "  [dry-run] iprange $RANGE_LO-$RANGE_HI + dhcp_on for $COMPOSE_CIDR only"
fi

[ "$DRYRUN" = "1" ] && { echo "  (dry-run: no changes made)"; exit 0; }
echo
echo "INSTALLED. MAAS UI: http://${MAAS_IP}:5240/MAAS  (admin password: $SECRETS/admin.pass, 0600)"
echo "Compose a service VM with:"
echo "  maas admin vm-host compose <id> cores=2 memory=4096 storage=root:40 hostname=<name>"
exit 0