#!/usr/bin/env bash
# scripts/site-tailscale.sh <prep|install|check> <site> -- per-DC Tailscale subnet router.
#
# The STANDING per-DC operator-access path (D-129(iii) + its 2026-08-07 amendment, rulings
# a-d): a DEDICATED subnet-router VM in the utility band at .7 advertises THIS DC's metal-admin
# /22 to the tailnet, so operators reach the DC's metal-admin plane -- SSH consoles AND the
# routed dashboards (Horizon on the metal-admin VIP) -- over Tailscale. Star topology
# (operator->DC only; no DC-to-DC), single router (HA pinned), SNAT ON (default). Control
# plane is self-hosted Headscale at tailscale.baldurkeep.com.
#
# RUNS ON THE .7 SUBNET-ROUTER VM (10.12.8.7 dc0 / 10.12.68.7 dc1), not on vcloud/voffice1.
# Invoke from a repo host over ssh, piping the script (no repo clone on the .7 VM needed):
#   prep:     ssh -J voffice1,<rack> <user>@10.12.8.7 'sudo TS_DEB=/tmp/tailscale.deb bash -s' -- prep vr1-dc0 < scripts/site-tailscale.sh
#   install:  ssh ... 'sudo TS_AUTHKEY_FILE=/tmp/ts.key bash -s' -- install vr1-dc0 < scripts/site-tailscale.sh
#   check:    ssh ... 'sudo bash -s' -- check vr1-dc0 < scripts/site-tailscale.sh
# The pre-auth key is passed via $TS_AUTHKEY_FILE (a path on the .7 VM) or $TS_AUTHKEY --
# NEVER on the command line, never printed.
#
# THREE VERBS, mapping onto the two-phase blocker:
#   prep    -- NODE-SIDE, runs any time: installs the tailscale .deb (from $TS_DEB, since the .7
#              has no external egress -- stage pool/tailscale_<ver>_amd64.deb from the rack, which
#              DOES have egress) and enables IP forwarding (the load-bearing subnet-router
#              property: 'tailscale up --advertise-routes' WARNS-and-succeeds with forwarding off,
#              so nothing reaches Horizon while every other check reads green). Idempotent.
#   install -- NODE-SIDE join: 'tailscale up' advertising this DC's metal-admin /22 + --accept-routes.
#   check   -- NODE-SIDE verify: Running + route approved + forwarding + (optional) tag + .7 leg.
#
# TAG / ACL POSTURE -- VR1 mirrors the office1 router, which is UNTAGGED (MEASURED 2026-08-07:
# office1-tailscale Tags=None, AdvertiseRoutes=[10.10.0.0/22], RouteAll=True). D-129(iii)'s TAGGED
# identity + autoApprovers + star ACL are DEFERRED to the bare-metal/Roosevelt install (operator
# 2026-08-07: "We can pin those for the bare metal install ... just follow the steps we used to
# bring office1 online" -- D-129(iii) amendment). So TS_TAG defaults EMPTY (untagged) and route
# approval is MANUAL on Headscale. Set TS_TAG=tag:subnet-router to restore the tagged design (and
# then the Headscale autoApprovers/star ACL become prerequisites again).
#
# EXIT: 0 ok | 1 check failed | 2 bad args/unknown site | 4 prep/install failed.
# Harness: tests/site-tailscale/run-tests.sh (offline; fakes tailscale/ip/sysctl/dpkg). ASCII + LF.
set -uo pipefail

MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in prep|install|check) ;; *)
  echo "usage: site-tailscale.sh <prep|install|check> <site>" >&2; exit 2 ;; esac

# ---------------------------------------------------------------------------
# Constants (measured / ruled, one place).
# ---------------------------------------------------------------------------
# Control plane, MEASURED 2026-08-07 from office1-tailscale's ControlURL.
LOGIN_SERVER="${TS_LOGIN_SERVER:-https://tailscale.baldurkeep.com:443}"
# Tag identity: DEFAULT EMPTY = untagged (VR1 office1-mirrored per the operator 2026-08-07
# deferral above). ${TS_TAG-} not :- so an explicit TS_TAG= is honoured as untagged. Set
# TS_TAG=tag:subnet-router to opt into the D-129(iii) tagged design (Roosevelt/bare-metal).
TAG="${TS_TAG-}"
# SNAT: ruling (d) = ON = tailscale's default, so NO --snat-subnet-routes flag is passed.

# ---------------------------------------------------------------------------
# Site table -- the metal-admin /22 THIS DC advertises. MEASURED (hard rule 2); every row
# cites its source. Non-overlapping per DC permanently (D-129(iii) amdt note 3 -- Headscale
# has no 4via6). The harness rejects a row without a MEASURED tag.
# ---------------------------------------------------------------------------
case "$SITE" in
  vr1-dc0) METAL_ADMIN_CIDR="10.12.8.0/22"  ;;  # MEASURED: lib-net.sh vr1-dc0 metal-admin; dashboard VIP 10.12.8.58
  vr1-dc1) METAL_ADMIN_CIDR="10.12.68.0/22" ;;  # MEASURED: lib-net.sh vr1-dc1 metal-admin (D-124 dc1 addressing)
  *) echo "site-tailscale.sh: unknown site '$SITE' (expected vr1-dc0|vr1-dc1)" >&2; exit 2 ;;
esac

TS="${TAILSCALE:-tailscale}"                       # overridable so the harness injects a fake
SYSCTL="${SYSCTL:-sysctl}"                          # overridable so the harness fakes forwarding
SYSCTL_D="${TS_SYSCTL_D:-/etc/sysctl.d}"           # overridable so the harness writes to a tmp dir
CURL="${CURL:-curl}"                                # overridable so the harness fakes control reach
say(){ printf '%s\n' "$*"; }
# jget <key> [<key>...] -- navigate nested keys of `tailscale status --json`; no eval (a
# bracket-key path in an eval string collides its own quotes). Prints Python repr (lists incl.).
jget(){ "$TS" status --json 2>/dev/null | "${PY:-python3}" -c '
import json,sys
try: d=json.load(sys.stdin)
except Exception: sys.exit(0)
for k in sys.argv[1:]:
    d = d.get(k) if isinstance(d,dict) else None
    if d is None: break
print("" if d is None else d)' "$@" 2>/dev/null; }

# fwd_ok -- both v4 and v6 forwarding on. Prints "v4=<> v6=<>" to stdout, returns 0/1.
fwd_ok(){
  local v4 v6
  v4="$("$SYSCTL" -n net.ipv4.ip_forward 2>/dev/null)"
  v6="$("$SYSCTL" -n net.ipv6.conf.all.forwarding 2>/dev/null)"
  printf 'v4=%s v6=%s' "$v4" "$v6"
  [ "$v4" = 1 ] && [ "$v6" = 1 ]
}

# derive the router's OWN metal-admin leg CIDR at runtime and cross-check the table (hard rule
# 3: prefer a runtime identity over a bare literal; a mismatch means wrong site or wrong VM).
own_leg_matches(){
  local net="${METAL_ADMIN_CIDR%/*}" pfx="${METAL_ADMIN_CIDR#*/}"
  # match on the /24 the .7 host sits in (10.12.8. or 10.12.68.), leg address ends in .7
  local base="${net%.*}"                            # 10.12.8  /  10.12.68  (net is x.y.z.0)
  ip -o -4 addr show 2>/dev/null | grep -qE "inet ${base%.*}\.[0-9]+\.7/|inet ${base}\.7/"
}

prep(){
  # 1. package present -- the .7 has NO external egress, so we do NOT apt from here. If tailscale
  #    is absent, install from a STAGED .deb ($TS_DEB, pulled on a host that has egress -- the
  #    rack). This mirrors office1's package (deb from pkgs.tailscale.com), not a snap.
  if ! command -v "$TS" >/dev/null 2>&1; then
    if [ -n "${TS_DEB:-}" ] && [ -s "${TS_DEB:-}" ]; then
      "${DPKG:-dpkg}" -i "$TS_DEB" >/dev/null 2>&1 \
        || "${APTGET:-apt-get}" -y -f install >/dev/null 2>&1 \
        || { say "prep: install of $TS_DEB failed (deps? -- needs iptables, iproute2)"; return 4; }
    else
      say "prep: tailscale absent and no \$TS_DEB staged -- stage pool/tailscale_<ver>_amd64.deb from the rack (it has egress) and pass TS_DEB=<path>"; return 4
    fi
  fi
  command -v "$TS" >/dev/null 2>&1 || { say "prep: tailscale still absent after install attempt"; return 4; }
  # 2. IP forwarding -- mirror office1 (/etc/sysctl.d/99-tailscale.conf, v4+v6). Idempotent write.
  local f="$SYSCTL_D/99-tailscale.conf"
  printf '%s\n' '# Managed by site-tailscale.sh -- subnet-router forwarding (mirrors office1).' \
                'net.ipv4.ip_forward = 1' 'net.ipv6.conf.all.forwarding = 1' > "$f" 2>/dev/null \
    || { say "prep: cannot write $f (need root)"; return 4; }
  "$SYSCTL" --system >/dev/null 2>&1
  # 3. assert forwarding actually took (a written file that never applied is the silent trap).
  local fw; fw="$(fwd_ok)" && say "prep: tailscale present ($("$TS" version 2>/dev/null | head -1)); IP forwarding $fw" \
    || { say "prep: IP forwarding not enabled ($fw) -- subnet router would not forward to Horizon"; return 4; }
}

install(){
  command -v "$TS" >/dev/null 2>&1 || { say "install: tailscale not present -- run 'prep $SITE' first"; return 4; }
  # Resolve the key to a FILE and hand it to tailscale via the file: scheme so the secret never
  # lands on argv / in ps (2026-08-07: an --authkey=<value> form leaked the key into ps AND a
  # session transcript). If only $TS_AUTHKEY is given, stage it to a 0600 temp file, never argv.
  local keyfile="${TS_AUTHKEY_FILE:-}" tmpkey=""
  if [ -z "$keyfile" ] && [ -n "${TS_AUTHKEY:-}" ]; then
    tmpkey="$(mktemp)"; chmod 600 "$tmpkey"; printf '%s' "$TS_AUTHKEY" > "$tmpkey"; keyfile="$tmpkey"
  fi
  { [ -n "$keyfile" ] && [ -s "$keyfile" ]; } || { say "install: no pre-auth key (\$TS_AUTHKEY_FILE / \$TS_AUTHKEY)"; return 4; }
  local tagflag=()
  [ -n "$TAG" ] && tagflag=(--advertise-tags="$TAG")
  # Subnet router = ADVERTISE ONLY. NO --accept-routes: on a DC .7 whose SSH path lives INSIDE the
  # advertised /22, consuming routes blackholed the node's own L3 and locked us out (2026-08-07
  # incident). A subnet router advertises; it does not need to consume routes. SNAT default on
  # (ruling d, no flag). --advertise-tags only when TS_TAG set (untagged VR1 by default).
  local rc=0
  "$TS" up --login-server="$LOGIN_SERVER" --authkey="file:$keyfile" \
        --advertise-routes="$METAL_ADMIN_CIDR" "${tagflag[@]}" \
        --hostname="${SITE}-tailscale" >/dev/null 2>&1 || rc=$?
  [ -n "$tmpkey" ] && rm -f "$tmpkey"
  [ "$rc" -eq 0 ] || { say "install: 'tailscale up' failed (key rejected, tag not authorised, or control-plane unreachable)"; return 4; }
  say "install: tailscale up issued for $SITE advertising $METAL_ADMIN_CIDR${TAG:+ as $TAG} (advertise-only)"
  say "NOTE: Headscale must APPROVE the route (manual, or autoApprovers if tagged) -- run 'check' to confirm."
}

check(){
  local rc=0 st routes tags fw
  st="$(jget BackendState)"
  [ "$st" = "Running" ] || { say "FAIL: tailscale BackendState='$st' (not Running)"; rc=1; }
  routes="$(jget Self PrimaryRoutes | tr -d "[]' " )"
  case ",$routes," in *,"$METAL_ADMIN_CIDR",*) say "OK: advertising+approved $METAL_ADMIN_CIDR";;
    *) say "FAIL: $METAL_ADMIN_CIDR not in approved PrimaryRoutes ('$routes') -- route unadvertised or not approved by Headscale"; rc=1;; esac
  # IP forwarding -- the load-bearing property. A subnet router with forwarding off joins,
  # advertises and gets approved while forwarding NOTHING (advisor 2026-08-07). Assert it here.
  fw="$(fwd_ok)" && say "OK: IP forwarding enabled ($fw)" \
    || { say "FAIL: IP forwarding off ($fw) -- subnet router will not forward to Horizon"; rc=1; }
  if [ -n "$TAG" ]; then
    tags="$(jget Self Tags | tr -d "[]' ")"
    case ",$tags," in *,"$TAG",*) say "OK: tagged identity $TAG";;
      *) say "FAIL: node not tagged $TAG (tags='$tags') -- untagged carries a key-expiry clock (note 1)"; rc=1;; esac
  else
    say "OK: untagged join (VR1 office1-mirrored; TS_TAG empty -- tagged design deferred to bare-metal, D-129(iii) amdt)"
  fi
  if own_leg_matches; then say "OK: router holds a .7 metal-admin leg in $METAL_ADMIN_CIDR"
    else say "FAIL: no .7 metal-admin leg in $METAL_ADMIN_CIDR on this host -- wrong VM or unconfigured"; rc=1; fi
  # Own-subnet guard: the node must NOT route its OWN advertised /22 via tailscale0 -- doing so
  # blackholes its metal-admin path and locks the operator out (2026-08-07 incident, the reason
  # --accept-routes is gone). This is a green-except-route failure the other checks cannot see.
  if ip route show "$METAL_ADMIN_CIDR" 2>/dev/null | grep -q tailscale0; then
    say "FAIL: $METAL_ADMIN_CIDR routed via tailscale0 -- node is blackholing its own subnet (2026-08-07)"; rc=1
  else say "OK: own subnet $METAL_ADMIN_CIDR not routed via tailscale0"; fi
  # Control-plane reachability: a subnet router that cannot reach Headscale registers/stays down
  # while every OTHER check but the route can read green (advisor 2026-08-07). Assert a live HTTP
  # response from the control URL, not just DNS.
  local code; code="$("$CURL" -s -o /dev/null -m 8 -w '%{http_code}' "$LOGIN_SERVER" 2>/dev/null)"
  if [ -n "$code" ] && [ "$code" != 000 ]; then say "OK: control plane reachable ($LOGIN_SERVER -> HTTP $code)"
    else say "FAIL: control plane $LOGIN_SERVER unreachable (http='$code') -- node cannot register/stay up"; rc=1; fi
  return $rc
}

case "$MODE" in
  prep)    prep    || exit 4 ;;
  install) install || exit 4 ; check || { say "install ran but check not yet green (route approval pending on Headscale)"; exit 1; } ;;
  check)   check   || exit 1 ;;
esac