#!/usr/bin/env bash
# scripts/dc-cache-proxy.sh <check|install|node> <site> -- per-DC apt CACHING PROXY
# (apt-cacher-ng). D-135 amendment 2026-07-24 (operator-ruled): the INTERIM DC1
# apt path so the DC1 build starts WITHOUT waiting on the full ~950 GiB
# debmirror. DC0 stays the full-mirror path (D-135 item 1); DC1's proxy is
# the DC1 STRATEGY, not an interim step (D-135 AMENDMENT 2026-07-27: "DC0 is the
# test of a full mirror. DC1 is the test of the mirror proxy."). The earlier
# "DC1's proxy is REPLACED by a full mirror afterward" plan is SUPERSEDED -- dc1's
# partial mirror and its whole apparatus were REMOVED 2026-07-27. Roosevelt:
# cache-proxy vs full-mirror is a per-DC utility-node artifact-delivery choice
# (D-132/D-133), and each DC proves one of them.
#
# RUNS ON THE DC RACK HOST, invoked over ssh like dc-mirror.sh (no clone there):
#   check:   ssh -i <dc-key> -J voffice1 <user>@<rack-transit-ip> \
#               'sudo bash -s' -- check dc1 < scripts/dc-cache-proxy.sh
#   install: same with 'install dc1' (idempotent; safe to re-run).
#
# ...EXCEPT `node`, WHICH RUNS ON A DC NODE (gate G17). Added 2026-07-29.
#   G17's dc1 half named `dc-cache-proxy.sh:210-217` as its node-side check, but that
#   code had NO subcommand entry point and lives inside `check`, whose other assertions
#   are RACK-side facts a node cannot satisfy (the acng drop-in, the systemd net unit,
#   the .4 alias on the libvirt bridge, the rack default route). Pointing a node at
#   `check` therefore fails for reasons that say nothing about node reachability.
#   `node` asserts ONLY what a node can, and touches ONLY curl + chronyc + the network
#   -- no root, no systemctl, no virsh, no generated-file diffs:
#     node:    bash -s -- node dc1 < scripts/dc-cache-proxy.sh     (ON the node)
#   G17's three assertions, in order:
#     (1) ARTIFACT REACHABILITY ON CONTENT, with an exit-code predicate -- a proxied
#         fetch of archive AND UCA `Release` through <LISTEN>:3142, asserting curl
#         exit 0 AND HTTP 200 AND that the BODY is that dist's Release file. A 200 is
#         not evidence: apt-cacher-ng serves its own HTML report pages with one.
#     (2) NODE TIME SOURCE -- `chronyc sources` names the MAAS-served rack and NOT the
#         DC edge (D-129(iv): the edge has NO NTP role; the MAAS hierarchy is the time
#         authority).
#     (3) An UNRECOGNISED or UNREACHABLE result REFUSES (exit 3) rather than defaulting
#         to success -- "could not look" is never "nothing there".
#
# PREREQS on the rack:
#   - apt-cacher-ng must be apt-installed first (fail-loud with the exact line).
#
# THIS SCRIPT NOW OWNS ITS OWN UTILITY NET LAYER (changed 2026-07-27).
#   It previously REUSED dc-mirror.sh's <site>-mirror-net.service for the .4
#   alias + the edge default route, and refused to install without it. That
#   coupling was a measured trap: on dc1 -- whose RULED artifact path is this
#   proxy, NOT a mirror -- three files named `dc1-mirror-*` (the net unit, its
#   apply helper, and the resolved drop-in) were load-bearing for the PROXY. So
#   the recorded mirror-removal procedure (07-23 changelog: "stop/disable
#   <site>-mirror-sync.timer + -net.service, rm the generated files") would have
#   silently killed the proxy, and anyone tidying up "leftover mirror stuff" on a
#   proxy-only DC would have done the same. Worse, the proxy could not be REBUILT
#   after such a teardown: nothing but `dc-mirror.sh install` created the net
#   layer, so rebuilding a proxy meant reinstalling the whole mirror apparatus
#   (sync unit, enabled daily timer, nginx vhost, debmirror package).
#
#   Now: <site>-cache-proxy-net.service + <site>-cache-proxy-net-apply +
#   /etc/systemd/resolved.conf.d/<site>-cache-proxy.conf. Neutrally named, owned
#   here, so a proxy-only DC has NO mirror-named dependency and is independently
#   rebuildable. The two net units are SAFE TO COEXIST on a site running both
#   (`ip addr replace` / `ip route replace` are idempotent and both set the same
#   MEASURED values), so this should be additive for dc0, which keeps its mirror.
#   COEXISTENCE IS REASONED, NOT MEASURED (hard rule 2): the proxy has only ever been
#   installed on dc1, where the mirror net unit was then removed. Nobody has run both
#   units on one host. The reasoning is that `ip addr replace` / `ip route replace` are
#   idempotent and both units write the SAME measured values, so the second is a no-op --
#   but if dc0 is ever given the proxy, VERIFY that rather than trusting this sentence.
#   Roosevelt: a DC's artifact-delivery strategy is a per-DC choice that must be
#   changeable without a rebuild of the other strategy's apparatus.
#
# CONSUMPTION (Stage 5, PROXY mode -- all http apt cached, incl. UCA, no remap):
#   juju model-config apt-http-proxy=http://<LISTEN>:3142
#   (or per-node /etc/apt/apt.conf.d: Acquire::http::Proxy "http://<LISTEN>:3142";)
#
# EXIT: 0 ok | 1 check failed | 2 bad args/unknown site | 3 node check REFUSED
#       (could not evaluate -- never a pass) | 4 install failed.
# ASCII + LF only.
set -uo pipefail

MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in check|install|node) ;; *)
  echo "usage: dc-cache-proxy.sh <check|install|node> <site>" >&2; exit 2 ;; esac

# ---------------------------------------------------------------------------
# Site table. Same MEASURED/RULED utility .4 + edge gateway per DC as
# dc-mirror.sh (the proxy listens at the SAME utility address the deploy
# already expects). ADD A SITE ONLY WITH MEASURED/RULED VALUES (hard rule 2);
# the harness rejects rows without a MEASURED tag.
# ---------------------------------------------------------------------------
case "$SITE" in
  dc0)
    # MEASURED/RULED 2026-07-23 (D-134 amendment + D-135). NOTE: dc0's RULED
    # path is the FULL MIRROR (D-135 item 1); this row exists for tooling
    # parity only -- dc0 is not deployed via the proxy.
    # CONSEQUENCE, so a failing run is not mistaken for a regression (the same
    # warning dc-mirror.sh's dc1 row now carries): `node dc0` will REFUSE on the
    # artifact half BY DESIGN -- there is no proxy listening at 10.12.8.4:3142 on
    # dc0. dc0's G17 artifact assertion is a package-path fetch from the MIRROR,
    # which is dc-mirror.sh's territory, not this script's. Do NOT "fix" it by
    # installing the proxy on dc0: that is a deliberate strategy change (D-135
    # AMENDMENT 2026-07-27), never a repair.
    UTIL_NET="vr1-dc0-metal-admin"
    LISTEN_CIDR="10.12.8.4/22"
    EDGE_GW="10.12.4.1"
    # RACK_IP -- the rack's metal-admin leg, i.e. the MAAS-served time source a node
    # must sync from under D-129(iv). MEASURED: scripts/dc-rack-net.sh:62 LEGS,
    # docs/audit/dc-rack-net-install-20260721.txt (the metal-admin leg row), and
    # the rendered dhcpd option itself -- "option ntp-servers 10.12.8.2" in
    # docs/audit/d131-sub3-dhcpd-option6-20260721.txt.
    RACK_IP="10.12.8.2"
    ;;
  dc1)
    # MEASURED/RULED 2026-07-24 (D-135 amendment -- interim DC1 apt path):
    #   LISTEN 10.12.68.4 -- the D-134 utility .4, reused from the mirror-net
    #     layer; EDGE_GW 10.12.64.1 -- dc1 edge LAN gw (upstream egress).
    UTIL_NET="vr1-dc1-metal-admin"
    LISTEN_CIDR="10.12.68.4/22"
    EDGE_GW="10.12.64.1"
    # RACK_IP -- MEASURED as an ADDRESS (scripts/dc-rack-net.sh:78 LEGS;
    # docs/audit/dc1-rack-net-install-20260723.txt, the metal-admin leg row;
    # D-124 amendment / docs/design-decisions.md:4587,4597). HONEST LIMIT (hard rule 2):
    # unlike dc0, dc1's RENDERED dhcpd `option ntp-servers` has NOT been captured. The
    # expectation here is the D-129(iv) INVARIANT (MAAS hierarchy is the time authority),
    # not a measurement of dc1's live DHCP render -- so if dc1 serves something else at
    # first boot this check goes red, which is the intended behaviour, not a false red.
    RACK_IP="10.12.68.2"
    ;;
  *) echo "FAIL: unknown site '$SITE' -- add a MEASURED row block first" >&2; exit 2 ;;
esac
LISTEN="${LISTEN_CIDR%%/*}"
PROXY_PORT="3142"
# The two artifact paths the deploy actually needs (Ubuntu archive + UCA), shared by
# the rack-side smoke test and the node-side G17 check so the two cannot drift apart.
# An ARRAY, never a space-joined string -- word-splitting a URL list is a measured
# strict-bash trap in this repo.
ARTIFACT_URLS=(
  "http://archive.ubuntu.com/ubuntu/dists/jammy/Release"
  "http://ubuntu-cloud.archive.canonical.com/ubuntu/dists/jammy-updates/caracal/Release"
)
ACNG_DROPIN="/etc/apt-cacher-ng/acng.conf.d/${SITE}-cache-proxy.conf"
# Net layer owned HERE (see the header note). Named for the PROXY, never "mirror".
NET_HELPER="/usr/local/sbin/${SITE}-cache-proxy-net-apply"
NET_UNIT="/etc/systemd/system/${SITE}-cache-proxy-net.service"
RESOLVED_DROPIN="/etc/systemd/resolved.conf.d/${SITE}-cache-proxy.conf"

# ---------------------------------------------------------------------------
# Generated config (single source of truth for check AND install). Minimal:
# apt-cacher-ng in proxy mode caches all http apt (archive/security/UCA) with
# no per-repo remap; we only pin the port so a package-default change cannot
# move it. Default CacheDir (/var/cache/apt-cacher-ng) is kept (owner-correct).
# ---------------------------------------------------------------------------
gen_acng_dropin() {
  cat <<EOF
# ${SITE} apt caching proxy -- generated by scripts/dc-cache-proxy.sh
# (D-135 amendment 2026-07-24); do not hand-edit. apt-cacher-ng proxy mode:
# caches all http apt requested by the DC nodes (archive/security/UCA), no
# per-repo remap. Consume via: juju model-config apt-http-proxy=http://${LISTEN}:${PROXY_PORT}
Port: ${PROXY_PORT}
EOF
}

# --- the utility net layer this script OWNS ---------------------------------
# Same MEASURED mechanism dc-mirror.sh proved on both racks 2026-07-23: resolve
# the bridge from the libvirt network NAME at runtime, then idempotently place
# the .4 alias and the edge default route. Deliberately identical values, so the
# two units coexist without fighting where a site runs both.
gen_net_helper() {
  cat <<EOF
#!/usr/bin/env bash
# ${SITE}-cache-proxy-net-apply -- generated by scripts/dc-cache-proxy.sh; do not hand-edit.
# Places the proxy LISTEN alias (D-134 utility .4) and the default route via the
# DC edge (the proxy's upstream egress). Bridge resolved from the libvirt network
# NAME at runtime (virbrN is auto-assigned, not stable identity -- hard rule 3).
set -euo pipefail
br_of() {
  virsh -c qemu:///system net-dumpxml "\$1" | sed -n "s/.*bridge name='\\([^']*\\)'.*/\\1/p"
}
ip addr replace ${LISTEN_CIDR} dev "\$(br_of ${UTIL_NET})"
ip route replace default via ${EDGE_GW}
EOF
}

gen_net_unit() {
  cat <<EOF
[Unit]
Description=${SITE} cache-proxy net (dc-cache-proxy.sh; D-135 amendment -- LISTEN alias + edge default route)
After=libvirtd.service ${SITE}-rack-legs.service network-online.target
Wants=network-online.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${NET_HELPER}

[Install]
WantedBy=multi-user.target
EOF
}

gen_resolved_dropin() {
  cat <<EOF
# ${SITE} cache-proxy DNS (generated by scripts/dc-cache-proxy.sh; D-135 amendment).
# System DNS = the DC edge resolver: the PROXY must resolve archive.ubuntu.com and
# the UCA host to serve a cache miss, and the rack has no other working resolver.
# The MAAS rack agent ignores resolv.conf (D-131), so node DNS is unaffected.
[Resolve]
DNS=${EDGE_GW}
EOF
}

# Bridge for the utility net, resolved from the libvirt net NAME at runtime
# (virbrN is auto-assigned, not stable identity -- hard rule 3).
util_bridge() {
  virsh -c qemu:///system net-dumpxml "$UTIL_NET" 2>/dev/null \
    | sed -n "s/.*bridge name='\([^']*\)'.*/\1/p"
}

# ---------------------------------------------------------------------------
# check -- mutate no CONFIG/service state; report and exit 0 ok / 1 incomplete.
# (The behavioral smoke test may populate the proxy cache with two Release
# files -- benign and self-managing; it is the proof the proxy actually serves.)
# ---------------------------------------------------------------------------
FAILED=0
say(){ echo "  $1"; }
miss(){ echo "  MISS $1"; FAILED=1; }

do_check() {
  command -v apt-cacher-ng >/dev/null 2>&1 || dpkg -s apt-cacher-ng >/dev/null 2>&1 \
    && say "OK   apt-cacher-ng installed" || miss "apt-cacher-ng not installed"
  if [ ! -f "$ACNG_DROPIN" ]; then miss "$ACNG_DROPIN absent"; else
    if diff -q <(gen_acng_dropin) "$ACNG_DROPIN" >/dev/null 2>&1; then
      say "OK   $ACNG_DROPIN matches"; else miss "$ACNG_DROPIN DIFFERS from generated"; fi
  fi
  systemctl is-active --quiet apt-cacher-ng \
    && say "OK   apt-cacher-ng active" || miss "apt-cacher-ng not active"
  ss -ltn 2>/dev/null | grep -q ":${PROXY_PORT} " \
    && say "OK   listening on :${PROXY_PORT}" || miss "not listening on :${PROXY_PORT}"
  # Utility net layer -- OWNED HERE since 2026-07-27, so assert the ARTIFACTS too,
  # not just the live effect. Asserting only the live address/route would pass
  # while the layer was actually provided by <site>-mirror-net.service, which is
  # exactly the hidden coupling this change removes: it would look independent and
  # break the moment the mirror was torn down.
  for pair in "$NET_HELPER:gen_net_helper" "$NET_UNIT:gen_net_unit" \
              "$RESOLVED_DROPIN:gen_resolved_dropin"; do
    f="${pair%%:*}"; g="${pair##*:}"
    if [ ! -f "$f" ]; then miss "$f absent (this script owns the net layer now)"
    elif diff -q <("$g") "$f" >/dev/null 2>&1; then say "OK   $f matches"
    else miss "$f DIFFERS from generated"; fi
  done
  systemctl is-enabled --quiet "${SITE}-cache-proxy-net.service" 2>/dev/null \
    && say "OK   ${SITE}-cache-proxy-net.service enabled" \
    || miss "${SITE}-cache-proxy-net.service not enabled (the layer would not survive a reboot)"
  br="$(util_bridge)"
  if [ -n "$br" ] && ip -4 -o addr show dev "$br" 2>/dev/null | grep -q " ${LISTEN}/"; then
    say "OK   ${LISTEN_CIDR} on $br ($UTIL_NET)"
  else miss "${LISTEN_CIDR} absent on metal-admin bridge (run: dc-cache-proxy.sh install $SITE)"; fi
  ip route show default 2>/dev/null | grep -q "via ${EDGE_GW}" \
    && say "OK   default route via edge ${EDGE_GW}" \
    || miss "default route via ${EDGE_GW} absent"
  # Behavioral smoke test: fetch through the proxy (proxy mode) -- archive AND
  # UCA, the two the deploy needs; proves cache-miss passthrough + upstream reach.
  if command -v curl >/dev/null 2>&1; then
    for u in "${ARTIFACT_URLS[@]}"; do
      HTTP="$(curl -s -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" -m 20 "$u" 2>/dev/null)"
      [ "$HTTP" = "200" ] && say "OK   proxy serves $u (200)" \
        || miss "proxy did NOT serve $u (got '$HTTP')"
    done
  fi
  [ "$FAILED" -eq 0 ] && { echo "dc-cache-proxy check ($SITE): PASS"; exit 0; }
  echo "dc-cache-proxy check ($SITE): FAIL"; exit 1
}

# ---------------------------------------------------------------------------
# install -- idempotent; requires root. Net layer is a PREREQ (owned by
# dc-mirror.sh), verified fail-loud; apt-cacher-ng package is fail-loud too.
# ---------------------------------------------------------------------------
do_install() {
  [ "$(id -u)" = "0" ] || { echo "FAIL: install requires root" >&2; exit 4; }
  umask 022
  # Prereq: the libvirt utility net must EXIST (we resolve its bridge). This is a
  # rack-standup artifact (dc-rack-net.sh), not a mirror artifact -- so a proxy-only
  # DC has no dependency on the mirror apparatus at any point.
  br="$(util_bridge)"
  if [ -z "$br" ]; then
    echo "FAIL: libvirt network '${UTIL_NET}' not found -- cannot resolve its bridge" >&2
    echo "  the rack net layer comes first: dc-rack-net.sh install $SITE" >&2
    exit 4
  fi
  # NET LAYER (owned here since 2026-07-27). Written and started BEFORE the proxy,
  # so the LISTEN address and upstream egress exist independently of any mirror.
  install -m 0755 /dev/null "$NET_HELPER" || exit 4
  gen_net_helper > "$NET_HELPER" || exit 4
  chmod 0755 "$NET_HELPER" || exit 4
  gen_net_unit > "$NET_UNIT" || exit 4
  mkdir -p "$(dirname "$RESOLVED_DROPIN")" || exit 4
  gen_resolved_dropin > "$RESOLVED_DROPIN" || exit 4
  systemctl daemon-reload || exit 4
  systemctl enable "${SITE}-cache-proxy-net.service" >/dev/null 2>&1 || exit 4
  systemctl restart "${SITE}-cache-proxy-net.service" \
    || { echo "FAIL: ${SITE}-cache-proxy-net.service did not apply" >&2; exit 4; }
  systemctl restart systemd-resolved >/dev/null 2>&1 || true
  # Prereq 2: the package (fail-loud, like dc-mirror.sh does for debmirror/nginx).
  if ! dpkg -s apt-cacher-ng >/dev/null 2>&1; then
    echo "FAIL: apt-cacher-ng not installed" >&2
    echo "  net layer is up -- run: apt-get update && apt-get install -y apt-cacher-ng" >&2
    echo "  then re-run: dc-cache-proxy.sh install $SITE" >&2
    exit 4
  fi
  mkdir -p "$(dirname "$ACNG_DROPIN")" || exit 4
  gen_acng_dropin > "$ACNG_DROPIN" || exit 4
  systemctl enable apt-cacher-ng >/dev/null 2>&1 || exit 4
  systemctl restart apt-cacher-ng || { echo "FAIL: apt-cacher-ng restart" >&2; exit 4; }
  echo "dc-cache-proxy install ($SITE): done -- running check:"
  do_check
}

# ---------------------------------------------------------------------------
# node -- GATE G17, run FROM A DC NODE at Stage-5 first boot. Read-only, no root,
# and deliberately touches NOTHING rack-side: only curl, chronyc, and the network.
# Three outcomes, never two: PASS (0), FAIL (1), REFUSE (3).
# ---------------------------------------------------------------------------
NFAILED=0; NREFUSED=0
nok(){     echo "  OK      $1"; }
nno(){     echo "  FAIL    $1"; NFAILED=1; }
nrefuse(){ echo "  REFUSE  $1"; NREFUSED=1; }

do_node() {
  echo "dc-cache-proxy node check ($SITE) -- from this node, against ${LISTEN}:${PROXY_PORT}"

  # -- G17 (1): artifact reachability asserted on CONTENT, with an exit-code predicate.
  # A bare HTTP 200 proves nothing here: apt-cacher-ng answers its own report and error
  # pages with 200, and the superseded G17 wording (`curl -sI`) exited 0 on 404/403/500.
  # So all four must hold: curl exit 0, HTTP 200, the body is not HTML, and the body is
  # THAT DIST's Release file. The Codename expectation is derived from the URL's own
  # /dists/ path rather than hardcoded -- MEASURED 2026-07-29 against both upstreams:
  # archive .../dists/jammy/Release        -> "Codename: jammy"
  # UCA     .../dists/jammy-updates/caracal/Release -> "Codename: jammy-updates/caracal"
  if ! command -v curl >/dev/null 2>&1; then
    nrefuse "curl is not present on this node -- cannot evaluate artifact reachability"
  else
    for u in "${ARTIFACT_URLS[@]}"; do
      dist="${u#*/dists/}"; dist="${dist%/Release}"
      body="$(mktemp)"; cerr="$(mktemp)"
      code="$(curl -sS -o "$body" -w '%{http_code}' \
              -x "http://${LISTEN}:${PROXY_PORT}" -m 30 "$u" 2>"$cerr")"; crc=$?
      if [ "$crc" -ne 0 ]; then
        nrefuse "could not reach $u THROUGH http://${LISTEN}:${PROXY_PORT} (curl exit $crc: $(head -1 "$cerr")) -- unreachable is not 'nothing there'"
      elif [ "$code" != "200" ]; then
        nno "proxy returned HTTP $code for $u (want 200)"
      elif grep -qi '<html' "$body"; then
        nno "proxy returned an HTML page for $u, not a Release file -- a 200 with an apt-cacher-ng report/error body"
      elif ! grep -qx "Codename: ${dist}" "$body"; then
        nno "body served for $u is not that dist's Release file (no exact 'Codename: ${dist}' line) -- content, not merely a 200"
      elif ! grep -qE '^(MD5Sum|SHA256):' "$body"; then
        nno "Release body for $u carries no checksum section -- what was served is not a package index"
      else
        nok "proxied $u: curl exit 0, HTTP 200, Codename: ${dist}, checksum section present"
      fi
      rm -f "$body" "$cerr"
    done
  fi

  # -- G17 (2): node time source is the MAAS-served rack, NOT the DC edge (D-129(iv):
  # "Keep MAAS hierarchy", no NTP role on the edge). NOTE ON METHOD: chrony is NOT
  # installed on the authoring jumphost, so `chronyc sources`' COLUMN FORMAT could not
  # be measured. This check therefore parses no columns -- it matches only on the two
  # address literals from the site table, and an output naming NEITHER is an
  # unrecognised state that REFUSES.
  if ! command -v chronyc >/dev/null 2>&1; then
    nrefuse "chronyc is not present on this node -- cannot evaluate the node time source"
  else
    CS="$(chronyc sources 2>&1)"; crc=$?
    if [ "$crc" -ne 0 ] || [ -z "$CS" ]; then
      nrefuse "chronyc sources returned nothing (exit $crc) -- the time source is UNKNOWN, not absent"
    elif grep -qw "$EDGE_GW" <<<"$CS"; then
      nno "node time source includes the DC edge ${EDGE_GW} -- D-129(iv) gives the edge NO NTP role; the MAAS hierarchy is the time authority"
    elif grep -qw "$RACK_IP" <<<"$CS"; then
      nok "node time source is the MAAS-served rack ${RACK_IP} (D-129(iv))"
    else
      nrefuse "chronyc sources names NEITHER the MAAS rack ${RACK_IP} nor the edge ${EDGE_GW} -- unrecognised time source; refusing to default to success. Output:"
      printf '%s\n' "$CS" | sed 's/^/          /'
    fi
  fi

  if [ "$NFAILED" -ne 0 ]; then
    echo "dc-cache-proxy node ($SITE): FAIL"; exit 1
  fi
  if [ "$NREFUSED" -ne 0 ]; then
    echo "dc-cache-proxy node ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
  fi
  echo "dc-cache-proxy node ($SITE): PASS"; exit 0
}

case "$MODE" in
  check)   do_check ;;
  install) do_install ;;
  node)    do_node ;;
esac
