#!/usr/bin/env bash
# scripts/dc-egress-check.sh <check> <site>
#
# Site-keyed DC EGRESS gate. RUNS ON THE RACK HOST, like dc-mirror.sh /
# dc-cache-proxy.sh / dc-rack-net.sh / dc-snap-proxy.sh. Read-only: it probes,
# it never configures.
#
# WHY THIS EXISTS -- a measured 19-hour outage that no gate could see.
# On 2026-08-01 05:48 the dc0 containment VM rebooted and the dc0 OPNsense edge
# came up broken (a partially-applied update; missing libcrypto.so.17 /
# libpython3.13.so.1.0, dropping to a single-user prompt). The DC therefore had
# NO path off-site. Nothing detected it for 19 hours:
#   * `dc-rack-net.sh check dc0` PASSED -- it asserts bridge legs and unit
#     states, which were all fine. A leg is not a path.
#   * `preflight.sh` had no egress gate at all.
#   * the mirror answered HTTP 200 from its own nginx -- a 200 from a LOCAL
#     service says nothing about upstream.
#   * the session that verified the reboot cited five true facts (rack units,
#     region API 200, mirror 200, snap proxy LISTENING, juju controller
#     connected) and none of them tested egress.
# It surfaced only when a nightly debmirror timer failed, and it surfaced as
# "mirror sync FAILED" -- three layers away from the cause. This gate exists so
# the cause is named at the layer it happens.
#
# LAYERING IS THE POINT. Each assertion is a strictly deeper dependency of the
# next, and the FIRST failure is reported as the cause rather than the last:
#   A1 default route  -> exists and points at THIS site's ruled edge LAN address
#   A2 edge answers   -> the gateway itself replies (this is what broke on 08-01)
#   A3 traffic leaves -> an off-site anchor answers, probed BOTH by ICMP and by
#                        TCP, so an ICMP-filtering path cannot fake a failure
#   A4 upstreams      -> the three artifact sources the deploy actually needs
# A4 does NOT duplicate dc-mirror.sh / dc-cache-proxy.sh / dc-snap-proxy.sh:
# those assert the LOCAL service serves content; this asserts the UPSTREAM those
# services sync FROM is reachable. Both matter and they fail independently -- on
# 08-01 the mirror served fine and its upstream was gone.
#
# ASSERT ON CONTENT, NEVER ON EXISTENCE, and REFUSE rather than default to a
# pass: an unrecognised state, a missing tool, or the wrong host all exit 2.
# "Could not look" is never "nothing there".
#
# Exit: 0 all pass | 1 an assertion FAILED | 2 REFUSE (could not evaluate).
# ASCII + LF.
set -uo pipefail

ACTION="${1:-}"; SITE="${2:-}"
case "$ACTION" in
  check) ;;
  *) echo "usage: dc-egress-check.sh check <site>" >&2; exit 2 ;;
esac

PASS=0; FAILN=0
ok()     { PASS=$((PASS+1));   printf '  OK     %s\n' "$1"; }
bad()    { FAILN=$((FAILN+1)); printf '  FAIL   %s\n' "$1"; }
refuse() { printf 'REFUSE: %s\n' "$1" >&2
           printf '  (could not evaluate -- this is NOT a pass)\n' >&2; exit 2; }

# ---------------------------------------------------------------------------
# Site table. ADD A SITE ONLY WITH MEASURED VALUES (hard rule 2). Every row
# cites where its value was measured; an uncited row is not admissible.
# ---------------------------------------------------------------------------
case "$SITE" in
  dc0|vr1-dc0)
    # MEASURED 2026-08-02 on vvr1-dc0: `ip route` -> "default via 10.12.4.1 dev
    # virbr5"; `ip -4 -o addr show virbr5` -> 10.12.4.2/22. Matches D-134's
    # provider-public band and dc-rack-net.sh's dc0 rows.
    EDGE_LAN="10.12.4.1"; RACK_LEG="10.12.4.2"
    # MEASURED 2026-08-01 (dc0 snap proxy built + proven, squid on the D-134
    # utility .4): LISTEN 10.12.8.4 port 3129.
    SNAP_PROXY="http://10.12.8.4:3129"
    ;;
  dc1|vr1-dc1)
    # MEASURED 2026-08-02 on vvr1-dc1: `ip route` -> "default via 10.12.64.1 dev
    # virbr4". Rack provider-public leg 10.12.64.2/22 recorded at the 2026-07-23
    # dc1 edge addressing (changelog-20260723-g12-dc1-edge.md).
    EDGE_LAN="10.12.64.1"; RACK_LEG="10.12.64.2"
    # dc1 has NO snap proxy ruled (D-135 gives it the apt CACHING PROXY at
    # 10.12.68.4:3142; the snap path is the D-135 items 2-3 gap). Probe direct.
    SNAP_PROXY=""
    ;;
  *) refuse "unknown site '$SITE' -- add a MEASURED row block first" ;;
esac

command -v ip   >/dev/null 2>&1 || refuse "no 'ip' on this host -- cannot evaluate egress"
command -v curl >/dev/null 2>&1 || refuse "no 'curl' on this host -- cannot evaluate egress"

# WRONG-HOST GUARD. Every value above describes the RACK. Run anywhere else and
# the probes measure a different machine's path -- the exact class recorded for
# dc-mirror.sh, which reports every item MISS when run from voffice1.
ip -4 -o addr show 2>/dev/null | grep -qw "${RACK_LEG}/22" \
  || refuse "this host does not carry ${RACK_LEG}/22, so it is not the ${SITE} rack -- \
run it there (ssh <rack> 'bash -s -- check ${SITE}' < scripts/dc-egress-check.sh)"

echo "== dc-egress-check $SITE =="
echo "   edge LAN (gateway): $EDGE_LAN     rack leg: $RACK_LEG"
[ -n "$SNAP_PROXY" ] && echo "   snap proxy: $SNAP_PROXY" || echo "   snap proxy: none ruled for this site"
echo

# ---- A1 default route ------------------------------------------------------
DEF="$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')"
if [ -z "$DEF" ]; then
  bad "A1 no default route on this rack -- nothing can leave the DC"
elif [ "$DEF" != "$EDGE_LAN" ]; then
  bad "A1 default route points at '$DEF', not this site's edge '$EDGE_LAN'"
else
  ok "A1 default route via $EDGE_LAN (this site's edge)"
fi

# ---- A2 the edge itself answers --------------------------------------------
# THE 2026-08-01 FAILURE. The edge VM was RUNNING with both NICs attached and
# did not answer at L2: `ip neigh` read "10.12.4.1 FAILED" while two neighbours
# on the same segment answered. Asserting the gateway REPLIES -- not that a
# route to it exists, not that the VM is running -- is what makes this catch it.
if ping -c2 -W3 "$EDGE_LAN" >/dev/null 2>&1; then
  ok "A2 edge $EDGE_LAN ANSWERS"
  EDGE_UP=1
else
  NEIGH="$(ip neigh show "$EDGE_LAN" 2>/dev/null | head -1)"
  bad "A2 edge $EDGE_LAN does NOT answer -- neigh: '${NEIGH:-none}'. The DC has no \
path off-site; everything below fails as a CONSEQUENCE, not as separate faults"
  EDGE_UP=0
fi

# ---- A3 traffic actually leaves --------------------------------------------
# Probed TWO ways on purpose: an ICMP-only probe would report a false outage on
# a path that filters ICMP, and a TCP-only probe would miss a DNS-layer break.
if [ "$EDGE_UP" -eq 1 ]; then
  # DIAGNOSIS, added 2026-08-02. If A2 passed and A3 fails, the edge is UP and reachable
  # but nothing comes back -- which is the signature of the edge FORWARDING WITHOUT
  # TRANSLATING. Measured on vr1-dc1 that day: simultaneous tcpdump on both edge taps
  # showed the same packet leaving the WAN leg with its LAN source address intact, so it
  # was never masqueraded and no reply could return. A2 cannot see this (the edge does
  # answer), and reporting it as a bare "anchor unreachable" sends the reader upstream
  # chasing an outage that is not there.
  NAT_HINT="  -- A2 PASSED, so the edge is up and reachable. An edge that answers but from \
which nothing returns is the FORWARDS-WITHOUT-TRANSLATING signature (no outbound NAT); \
confirm with tcpdump on both edge taps before looking upstream"
  ping -c2 -W3 1.1.1.1 >/dev/null 2>&1 \
    && ok "A3 off-site ICMP anchor 1.1.1.1 answers" \
    || bad "A3 off-site ICMP anchor 1.1.1.1 does NOT answer${NAT_HINT}"
  C="$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 --noproxy '*' \
       http://archive.ubuntu.com/ubuntu/dists/jammy/Release 2>/dev/null)"
  case "$C" in
    200) ok  "A3 off-site TCP/HTTP anchor answers 200 (archive.ubuntu.com)" ;;
    000|"") bad "A3 off-site TCP/HTTP anchor unreachable (curl code '${C:-none}')" ;;
    *)   bad "A3 off-site TCP/HTTP anchor answered '$C', not 200 -- unrecognised, \
not assumed healthy" ;;
  esac
else
  bad "A3 SKIPPED -- the edge does not answer, so an off-site probe would only \
restate A2. Fix the edge first"
fi

# ---- A4 the upstreams the deploy needs -------------------------------------
# These are the sources the LOCAL artifact services sync FROM. dc-mirror.sh /
# dc-cache-proxy.sh / dc-snap-proxy.sh assert the local service SERVES; this
# asserts its upstream is REACHABLE. On 2026-08-01 the mirror served 200 all day
# with its upstream gone, which is why both halves are needed.
probe() {  # $1 label, $2 url, $3 optional proxy
  # `Snap-Device-Series: 16` is REQUIRED by the snap store API and is not optional
  # politeness: measured 2026-08-02, the same URL returns 400 WITHOUT it both through the
  # proxy AND direct, and 200 with it. This gate reported a healthy proxy as broken for
  # exactly that reason -- a false FAIL in its own first live run, while the repo's
  # dc-snap-proxy.sh (which sends it) returned PASS with a real payload. Harmless on the
  # other probes, so it is sent unconditionally rather than special-cased.
  local lbl="$1" url="$2" px="${3:-}" code
  if [ -n "$px" ]; then
    code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 -H 'Snap-Device-Series: 16' -x "$px" "$url" 2>/dev/null)"
  else
    code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 -H 'Snap-Device-Series: 16' --noproxy '*' "$url" 2>/dev/null)"
  fi
  case "$code" in
    200|30[0-9]) ok  "A4 $lbl reachable (HTTP $code)" ;;
    000|"")      bad "A4 $lbl UNREACHABLE (curl code '${code:-none}')" ;;
    *)           bad "A4 $lbl answered '$code' -- unrecognised, not assumed healthy" ;;
  esac
}
if [ "$EDGE_UP" -eq 1 ]; then
  # apt upstream -- what debmirror pulls. THIS is what failed on 2026-08-02:
  # "500 Can't connect to ubuntu-cloud.archive.canonical.com".
  probe "apt upstream (archive.ubuntu.com)" \
        "http://archive.ubuntu.com/ubuntu/dists/jammy/Release"
  probe "UCA upstream (ubuntu-cloud.archive.canonical.com)" \
        "http://ubuntu-cloud.archive.canonical.com/ubuntu/dists/jammy-updates/caracal/Release"
  # juju agent binaries -- the bootstrap/deploy dependency with NO local copy.
  probe "juju agent stream (streams.canonical.com)" \
        "https://streams.canonical.com/juju/tools/"
  # snaps -- through this site's ruled proxy where one exists, else direct.
  if [ -n "$SNAP_PROXY" ]; then
    probe "snap store VIA this site's proxy" \
          "https://api.snapcraft.io/v2/snaps/info/core22" "$SNAP_PROXY"
  else
    probe "snap store (direct -- no proxy ruled for this site)" \
          "https://api.snapcraft.io/v2/snaps/info/core22"
  fi
else
  bad "A4 SKIPPED -- the edge does not answer. Reporting four more failures here \
would restate one cause as five faults"
fi

echo
echo "RESULT: pass=$PASS fail=$FAILN"
if [ "$FAILN" -eq 0 ]; then
  echo "PASS: dc-egress-check $SITE"
  exit 0
fi
echo "FAIL: dc-egress-check $SITE -- $FAILN assertion(s) failed"
exit 1
