#!/usr/bin/env bash # scripts/dc-egress-check.sh # # Site-keyed DC EGRESS gate. RUNS ON THE RACK HOST, like dc-mirror.sh / # dc-cache-proxy.sh / dc-rack-net.sh / dc-snap-proxy.sh. Read-only: it probes, # it never configures. # # WHY THIS EXISTS -- a measured 19-hour outage that no gate could see. # On 2026-08-01 05:48 the dc0 containment VM rebooted and the dc0 OPNsense edge # came up broken (a partially-applied update; missing libcrypto.so.17 / # libpython3.13.so.1.0, dropping to a single-user prompt). The DC therefore had # NO path off-site. Nothing detected it for 19 hours: # * `dc-rack-net.sh check dc0` PASSED -- it asserts bridge legs and unit # states, which were all fine. A leg is not a path. # * `preflight.sh` had no egress gate at all. # * the mirror answered HTTP 200 from its own nginx -- a 200 from a LOCAL # service says nothing about upstream. # * the session that verified the reboot cited five true facts (rack units, # region API 200, mirror 200, snap proxy LISTENING, juju controller # connected) and none of them tested egress. # It surfaced only when a nightly debmirror timer failed, and it surfaced as # "mirror sync FAILED" -- three layers away from the cause. This gate exists so # the cause is named at the layer it happens. # # LAYERING IS THE POINT. Each assertion is a strictly deeper dependency of the # next, and the FIRST failure is reported as the cause rather than the last: # A1 default route -> exists and points at THIS site's ruled edge LAN address # A2 edge answers -> the gateway itself replies (this is what broke on 08-01) # A3 traffic leaves -> an off-site anchor answers, probed BOTH by ICMP and by # TCP, so an ICMP-filtering path cannot fake a failure # A4 upstreams -> the three artifact sources the deploy actually needs # A4 does NOT duplicate dc-mirror.sh / dc-cache-proxy.sh / dc-snap-proxy.sh: # those assert the LOCAL service serves content; this asserts the UPSTREAM those # services sync FROM is reachable. Both matter and they fail independently -- on # 08-01 the mirror served fine and its upstream was gone. # # ASSERT ON CONTENT, NEVER ON EXISTENCE, and REFUSE rather than default to a # pass: an unrecognised state, a missing tool, or the wrong host all exit 2. # "Could not look" is never "nothing there". # # Exit: 0 all pass | 1 an assertion FAILED | 2 REFUSE (could not evaluate). # ASCII + LF. set -uo pipefail ACTION="${1:-}"; SITE="${2:-}" case "$ACTION" in check) ;; *) echo "usage: dc-egress-check.sh check " >&2; exit 2 ;; esac PASS=0; FAILN=0 ok() { PASS=$((PASS+1)); printf ' OK %s\n' "$1"; } bad() { FAILN=$((FAILN+1)); printf ' FAIL %s\n' "$1"; } refuse() { printf 'REFUSE: %s\n' "$1" >&2 printf ' (could not evaluate -- this is NOT a pass)\n' >&2; exit 2; } # --------------------------------------------------------------------------- # Site table. ADD A SITE ONLY WITH MEASURED VALUES (hard rule 2). Every row # cites where its value was measured; an uncited row is not admissible. # --------------------------------------------------------------------------- case "$SITE" in dc0|vr1-dc0) # MEASURED 2026-08-02 on vvr1-dc0: `ip route` -> "default via 10.12.4.1 dev # virbr5"; `ip -4 -o addr show virbr5` -> 10.12.4.2/22. Matches D-134's # provider-public band and dc-rack-net.sh's dc0 rows. EDGE_LAN="10.12.4.1"; RACK_LEG="10.12.4.2" # MEASURED 2026-08-01 (dc0 snap proxy built + proven, squid on the D-134 # utility .4): LISTEN 10.12.8.4 port 3129. SNAP_PROXY="http://10.12.8.4:3129" ;; dc1|vr1-dc1) # MEASURED 2026-08-02 on vvr1-dc1: `ip route` -> "default via 10.12.64.1 dev # virbr4". Rack provider-public leg 10.12.64.2/22 recorded at the 2026-07-23 # dc1 edge addressing (changelog-20260723-g12-dc1-edge.md). EDGE_LAN="10.12.64.1"; RACK_LEG="10.12.64.2" # dc1 has NO snap proxy ruled (D-135 gives it the apt CACHING PROXY at # 10.12.68.4:3142; the snap path is the D-135 items 2-3 gap). Probe direct. SNAP_PROXY="" ;; *) refuse "unknown site '$SITE' -- add a MEASURED row block first" ;; esac command -v ip >/dev/null 2>&1 || refuse "no 'ip' on this host -- cannot evaluate egress" command -v curl >/dev/null 2>&1 || refuse "no 'curl' on this host -- cannot evaluate egress" # WRONG-HOST GUARD. Every value above describes the RACK. Run anywhere else and # the probes measure a different machine's path -- the exact class recorded for # dc-mirror.sh, which reports every item MISS when run from voffice1. ip -4 -o addr show 2>/dev/null | grep -qw "${RACK_LEG}/22" \ || refuse "this host does not carry ${RACK_LEG}/22, so it is not the ${SITE} rack -- \ run it there (ssh 'bash -s -- check ${SITE}' < scripts/dc-egress-check.sh)" echo "== dc-egress-check $SITE ==" echo " edge LAN (gateway): $EDGE_LAN rack leg: $RACK_LEG" [ -n "$SNAP_PROXY" ] && echo " snap proxy: $SNAP_PROXY" || echo " snap proxy: none ruled for this site" echo # ---- A1 default route ------------------------------------------------------ DEF="$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')" if [ -z "$DEF" ]; then bad "A1 no default route on this rack -- nothing can leave the DC" elif [ "$DEF" != "$EDGE_LAN" ]; then bad "A1 default route points at '$DEF', not this site's edge '$EDGE_LAN'" else ok "A1 default route via $EDGE_LAN (this site's edge)" fi # ---- A2 the edge itself answers -------------------------------------------- # THE 2026-08-01 FAILURE. The edge VM was RUNNING with both NICs attached and # did not answer at L2: `ip neigh` read "10.12.4.1 FAILED" while two neighbours # on the same segment answered. Asserting the gateway REPLIES -- not that a # route to it exists, not that the VM is running -- is what makes this catch it. if ping -c2 -W3 "$EDGE_LAN" >/dev/null 2>&1; then ok "A2 edge $EDGE_LAN ANSWERS" EDGE_UP=1 else NEIGH="$(ip neigh show "$EDGE_LAN" 2>/dev/null | head -1)" bad "A2 edge $EDGE_LAN does NOT answer -- neigh: '${NEIGH:-none}'. The DC has no \ path off-site; everything below fails as a CONSEQUENCE, not as separate faults" EDGE_UP=0 fi # ---- A3 traffic actually leaves -------------------------------------------- # Probed TWO ways on purpose: an ICMP-only probe would report a false outage on # a path that filters ICMP, and a TCP-only probe would miss a DNS-layer break. if [ "$EDGE_UP" -eq 1 ]; then # DIAGNOSIS, added 2026-08-02. If A2 passed and A3 fails, the edge is UP and reachable # but nothing comes back -- which is the signature of the edge FORWARDING WITHOUT # TRANSLATING. Measured on vr1-dc1 that day: simultaneous tcpdump on both edge taps # showed the same packet leaving the WAN leg with its LAN source address intact, so it # was never masqueraded and no reply could return. A2 cannot see this (the edge does # answer), and reporting it as a bare "anchor unreachable" sends the reader upstream # chasing an outage that is not there. NAT_HINT=" -- A2 PASSED, so the edge is up and reachable. An edge that answers but from \ which nothing returns is the FORWARDS-WITHOUT-TRANSLATING signature (no outbound NAT); \ confirm with tcpdump on both edge taps before looking upstream" ping -c2 -W3 1.1.1.1 >/dev/null 2>&1 \ && ok "A3 off-site ICMP anchor 1.1.1.1 answers" \ || bad "A3 off-site ICMP anchor 1.1.1.1 does NOT answer${NAT_HINT}" C="$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 --noproxy '*' \ http://archive.ubuntu.com/ubuntu/dists/jammy/Release 2>/dev/null)" case "$C" in 200) ok "A3 off-site TCP/HTTP anchor answers 200 (archive.ubuntu.com)" ;; 000|"") bad "A3 off-site TCP/HTTP anchor unreachable (curl code '${C:-none}')" ;; *) bad "A3 off-site TCP/HTTP anchor answered '$C', not 200 -- unrecognised, \ not assumed healthy" ;; esac else bad "A3 SKIPPED -- the edge does not answer, so an off-site probe would only \ restate A2. Fix the edge first" fi # ---- A4 the upstreams the deploy needs ------------------------------------- # These are the sources the LOCAL artifact services sync FROM. dc-mirror.sh / # dc-cache-proxy.sh / dc-snap-proxy.sh assert the local service SERVES; this # asserts its upstream is REACHABLE. On 2026-08-01 the mirror served 200 all day # with its upstream gone, which is why both halves are needed. probe() { # $1 label, $2 url, $3 optional proxy # `Snap-Device-Series: 16` is REQUIRED by the snap store API and is not optional # politeness: measured 2026-08-02, the same URL returns 400 WITHOUT it both through the # proxy AND direct, and 200 with it. This gate reported a healthy proxy as broken for # exactly that reason -- a false FAIL in its own first live run, while the repo's # dc-snap-proxy.sh (which sends it) returned PASS with a real payload. Harmless on the # other probes, so it is sent unconditionally rather than special-cased. local lbl="$1" url="$2" px="${3:-}" code if [ -n "$px" ]; then code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 -H 'Snap-Device-Series: 16' -x "$px" "$url" 2>/dev/null)" else code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 -H 'Snap-Device-Series: 16' --noproxy '*' "$url" 2>/dev/null)" fi case "$code" in 200|30[0-9]) ok "A4 $lbl reachable (HTTP $code)" ;; 000|"") bad "A4 $lbl UNREACHABLE (curl code '${code:-none}')" ;; *) bad "A4 $lbl answered '$code' -- unrecognised, not assumed healthy" ;; esac } if [ "$EDGE_UP" -eq 1 ]; then # apt upstream -- what debmirror pulls. THIS is what failed on 2026-08-02: # "500 Can't connect to ubuntu-cloud.archive.canonical.com". probe "apt upstream (archive.ubuntu.com)" \ "http://archive.ubuntu.com/ubuntu/dists/jammy/Release" probe "UCA upstream (ubuntu-cloud.archive.canonical.com)" \ "http://ubuntu-cloud.archive.canonical.com/ubuntu/dists/jammy-updates/caracal/Release" # juju agent binaries -- the bootstrap/deploy dependency with NO local copy. probe "juju agent stream (streams.canonical.com)" \ "https://streams.canonical.com/juju/tools/" # snaps -- through this site's ruled proxy where one exists, else direct. if [ -n "$SNAP_PROXY" ]; then probe "snap store VIA this site's proxy" \ "https://api.snapcraft.io/v2/snaps/info/core22" "$SNAP_PROXY" else probe "snap store (direct -- no proxy ruled for this site)" \ "https://api.snapcraft.io/v2/snaps/info/core22" fi else bad "A4 SKIPPED -- the edge does not answer. Reporting four more failures here \ would restate one cause as five faults" fi echo echo "RESULT: pass=$PASS fail=$FAILN" if [ "$FAILN" -eq 0 ]; then echo "PASS: dc-egress-check $SITE" exit 0 fi echo "FAIL: dc-egress-check $SITE -- $FAILN assertion(s) failed" exit 1