#!/usr/bin/env bash
# tests/dc-egress-check/run-tests.sh -- offline harness for scripts/dc-egress-check.sh.
# Stubs `ip`, `ping` and `curl` in a fakebin so NOTHING touches a live rack or the
# internet. Exit: 0 all pass | 1 any case failed.  ASCII + LF.
#
# WHAT THIS HARNESS IS FOR. The gate exists because a 19-hour dc0 egress outage went
# undetected: dc-rack-net.sh check PASSED throughout (it asserts legs and units, not a
# path) and the mirror answered 200 from its own nginx the whole time. So the cases below
# are not decoration -- each one is a shape that ACTUALLY OCCURRED or that would have made
# the outage invisible again.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
S="$(cd "$HERE/../../scripts" && pwd)/dc-egress-check.sh"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0
ok()  { PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
bad() { FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; [ $# -gt 1 ] && printf '        %s\n' "$2"; }

# mkfix <dir> <onrack:yes|no> <gw:edge|other|none> <edge:up|down> <anchor:up|down> <up:ok|aptdown|jujudown|snapdown|weird>
mkfix() {
  local d="$1" onrack="$2" gw="$3" edge="$4" anchor="$5" up="$6"
  mkdir -p "$d/fakebin"
  # ip: addr show decides the wrong-host guard; route decides A1
  { echo '#!/usr/bin/env bash'
    echo "ONRACK=$onrack; GW=$gw"
    echo 'if [ "$1" = "-4" ]; then [ "$ONRACK" = yes ] && echo "2: virbr5    inet 10.12.4.2/22 brd 10.12.7.255 scope global virbr5"; exit 0; fi'
    echo 'if [ "$1" = "route" ]; then'
    echo '  case "$GW" in edge) echo "default via 10.12.4.1 dev virbr5" ;;'
    echo '                other) echo "default via 10.10.0.1 dev enp1s0" ;;'
    echo '                none) : ;; esac; exit 0; fi'
    echo 'if [ "$1" = "neigh" ]; then echo "10.12.4.1 FAILED"; exit 0; fi'
    echo 'exit 0'
  } > "$d/fakebin/ip"; chmod +x "$d/fakebin/ip"
  # ping: the edge and the off-site anchor answer independently
  { echo '#!/usr/bin/env bash'
    echo "EDGE=$edge; ANCHOR=$anchor"
    echo 'for a; do case "$a" in'
    echo '  10.12.4.1) [ "$EDGE" = up ] && exit 0 || exit 1 ;;'
    echo '  1.1.1.1)   [ "$ANCHOR" = up ] && exit 0 || exit 1 ;;'
    echo 'esac; done; exit 1'
  } > "$d/fakebin/ping"; chmod +x "$d/fakebin/ping"
  # curl: one code per upstream, so each A4 probe can fail alone
  { echo '#!/usr/bin/env bash'
    echo "UP=$up; ANCHOR=$anchor"
    echo 'URL=""; for a; do case "$a" in http*) URL="$a" ;; esac; done'
    echo 'case "$URL" in'
    echo '  *archive.ubuntu.com*)  if [ "$ANCHOR" = down ]; then echo -n 000; elif [ "$UP" = aptdown ]; then echo -n 000; elif [ "$UP" = weird ]; then echo -n 403; else echo -n 200; fi ;;'
    echo '  *ubuntu-cloud*)        if [ "$UP" = aptdown ]; then echo -n 000; elif [ "$UP" = weirdup ]; then echo -n 403; else echo -n 200; fi ;;'
    echo '  *streams.canonical*)   [ "$UP" = jujudown ] && echo -n 000 || echo -n 200 ;;'
    echo '  *api.snapcraft.io*)    [ "$UP" = snapdown ] && echo -n 000 || echo -n 200 ;;'
    echo '  *) echo -n 200 ;; esac; exit 0'
  } > "$d/fakebin/curl"; chmod +x "$d/fakebin/curl"
}
run() { local d="$1"; shift
  OUT="$(PATH="$d/fakebin:$PATH" bash "$S" "$@" 2>&1)"; RC=$?; }

# T01 the healthy path.
mkfix "$TMP/t1" yes edge up up ok; run "$TMP/t1" check dc0
[ "$RC" -eq 0 ] && ok "T01 fully healthy egress PASSES (rc=0)" || bad "T01 healthy run failed (rc=$RC)" "$OUT"

# T02 THE 2026-08-01 OUTAGE, exactly: the edge does not answer. This is the case the
# whole gate exists for -- dc-rack-net.sh check passed in this state.
mkfix "$TMP/t2" yes edge down down ok; run "$TMP/t2" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "A2 edge 10.12.4.1 does NOT answer"; } \
  && ok "T02 edge-not-answering FAILS and names the edge (the 08-01 outage)" \
  || bad "T02 did not fail/name the edge (rc=$RC)" "$OUT"

# T03 ...and it reports ONE cause, not five. A gate that restates a single fault as a
# cascade trains people to skim it.
printf '%s' "$OUT" | grep -q "A3 SKIPPED" && printf '%s' "$OUT" | grep -q "A4 SKIPPED" \
  && ok "T03 downstream assertions SKIP rather than restate one cause as five faults" \
  || bad "T03 downstream assertions did not skip" "$OUT"

# T04 wrong host REFUSES (2). Every value in the site table describes the RACK; run
# elsewhere and the probes measure a different machine -- the recorded dc-mirror.sh trap.
mkfix "$TMP/t4" no edge up up ok; run "$TMP/t4" check dc0
[ "$RC" -eq 2 ] && ok "T04 wrong host REFUSES (rc=2), never reports a clean tree" \
  || bad "T04 wrong host did not refuse (rc=$RC)" "$OUT"

# T05 unknown site REFUSES rather than guessing a gateway.
run "$TMP/t1" check dc9
[ "$RC" -eq 2 ] && ok "T05 unknown site REFUSES (rc=2)" || bad "T05 unknown site not refused (rc=$RC)"

# T06 a default route pointing somewhere OTHER than this site's edge is a FAIL, not a
# pass -- routing out through the wrong path is how a DC silently loses its isolation.
mkfix "$TMP/t6" yes other up up ok; run "$TMP/t6" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "not this site's edge"; } \
  && ok "T06 default route via the WRONG gateway FAILS" || bad "T06 wrong gateway not caught" "$OUT"

# T07 no default route at all is a FAIL.
mkfix "$TMP/t7" yes none up up ok; run "$TMP/t7" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "no default route"; } \
  && ok "T07 absent default route FAILS" || bad "T07 absent default route not caught" "$OUT"

# T08 THE 2026-08-02 SYMPTOM: edge and anchor fine, apt upstream gone. The mirror served
# 200 from its own nginx throughout, so only an UPSTREAM probe sees this.
mkfix "$TMP/t8" yes edge up up aptdown; run "$TMP/t8" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "UCA upstream"; } \
  && ok "T08 apt/UCA upstream down FAILS even with a healthy edge" \
  || bad "T08 apt upstream failure not caught" "$OUT"

# T09 juju agent stream down FAILS on its own -- the deploy dependency with no local copy.
mkfix "$TMP/t9" yes edge up up jujudown; run "$TMP/t9" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "juju agent stream"; } \
  && ok "T09 juju agent stream down FAILS alone" || bad "T09 agent stream failure not caught" "$OUT"

# T10 snap store down FAILS on its own.
mkfix "$TMP/t10" yes edge up up snapdown; run "$TMP/t10" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "snap store"; } \
  && ok "T10 snap store down FAILS alone" || bad "T10 snap failure not caught" "$OUT"

# T11 an UNRECOGNISED http code is a FAIL, never "not 000 so probably fine". This is the
# standing rule that an unrecognised state refuses rather than defaulting to success.
mkfix "$TMP/t11" yes edge up up weird; run "$TMP/t11" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "unrecognised, not assumed healthy"; } \
  && ok "T11 an unrecognised HTTP code FAILS rather than passing" || bad "T11 weird code passed" "$OUT"

# T12 dc1 is a DIFFERENT site with its own measured gateway, and it has NO ruled snap
# proxy -- checking it with dc0's values would be the cross-DC error this repo has hit.
mkfix "$TMP/t12" yes edge up up ok; run "$TMP/t12" check dc1
{ [ "$RC" -eq 2 ] && printf '%s' "$OUT" | grep -q "not the dc1 rack"; } \
  && ok "T12 dc0 host checked as dc1 REFUSES (cross-DC guard)" || bad "T12 cross-DC not guarded (rc=$RC)" "$OUT"

# T14 A4's OWN unrecognised-code branch. ADDED after a mutation pass proved T11 was
# DECORATION: T11's fixture makes archive.ubuntu.com odd, which A3 catches first, so
# A4's `*)` branch was never exercised and flipping it to ok() left the suite fully
# green. This fixture keeps the A3 anchor healthy (200) and makes only the UCA
# upstream odd, so the failure can only come from A4.
mkfix "$TMP/t14" yes edge up up weirdup; run "$TMP/t14" check dc0
{ [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "A4 UCA upstream.*answered '403'"; } \
  && ok "T14 A4 unrecognised HTTP code FAILS (A3 healthy, so only A4 can fail it)" \
  || bad "T14 A4 unrecognised code not caught" "$OUT"

# T15 the snap probe must send `Snap-Device-Series: 16`. ADDED 2026-08-02 after the gate's
# FIRST LIVE RUN produced a false FAIL: the store API returns 400 without that header, both
# through the proxy and direct, and 200 with it -- while the repo's dc-snap-proxy.sh (which
# sends it) returned PASS with a real payload. The gate reported a healthy path as broken.
# Asserted on the SCRIPT here because the fixture curl cannot observe headers; the live
# behaviour is what the 2026-08-02 capture records.
# ASSERT ON THE CODE, not on any occurrence: a first cut grepped the whole file and passed
# on the COMMENT that explains the header, so deleting it from both curl invocations left
# the suite green. Count only curl lines that carry it -- there are two (proxied, direct).
t15n=$(grep -c "curl .*-H 'Snap-Device-Series: 16'" "$S")
[ "$t15n" -eq 2 ] \
  && ok "T15 both curl probes send the header the store API requires" \
  || bad "T15 expected 2 curl probes carrying Snap-Device-Series, found $t15n"

# T16 when A2 PASSES and A3 fails, the output must name the forwards-without-translating
# signature. Measured on vr1-dc1 2026-08-02: the edge answered, packets left the WAN leg
# with their LAN source intact, and nothing returned. Without this, the reader is sent
# upstream chasing an outage that is not there.
mkfix "$TMP/t16" yes edge up down ok; run "$TMP/t16" check dc0
{ printf '%s' "$OUT" | grep -q "A2 edge 10.12.4.1 ANSWERS"; } \
  && { printf '%s' "$OUT" | grep -qi "FORWARDS-WITHOUT-TRANSLATING\|no outbound NAT"; } \
  && ok "T16 edge up + anchor dead names the no-NAT signature, not a bare 'unreachable'" \
  || bad "T16 no-NAT diagnosis missing"

# T13 usage without an action REFUSES.
OUT="$(bash "$S" 2>&1)"; RC=$?
[ "$RC" -eq 2 ] && ok "T13 no action REFUSES (rc=2)" || bad "T13 no action not refused (rc=$RC)"

echo
echo "RESULT: PASS=$PASS FAIL=$FAIL"
[ "$FAIL" -eq 0 ] && { echo "ALL PASS"; exit 0; } || exit 1