#!/usr/bin/env bash
# scripts/dc-snap-proxy.sh <check|install|node> <site> -- per-DC snap FORWARD
# PROXY (squid, CONNECT-only). Builds D-135/D-107 RULING 2, RULED 2026-07-31
# (GA-R5), operator utterance verbatim:
#   "HTTP(S) forward proxy in the DC utility band + juju snap-https-proxy (Recommended)"
#
# WHY IT EXISTS. mysql-innodb-cluster (mysql-shell), ovn-central
# (prometheus-ovn-exporter) and vault (core) all fail their install hook because
# they cannot reach api.snapcraft.io -- the D-135 items 2-3 gap. It hits dc1
# IDENTICALLY: apt-cacher-ng proxies apt, never snaps. D-107's core statement
# stays TRUE and UNAMENDED -- nodes reach an IN-DC proxy, not the store.
#
# RUNS ON THE DC RACK HOST, invoked over ssh exactly like dc-mirror.sh /
# dc-cache-proxy.sh (no repo clone needed there):
#   check:   ssh -i <dc-key> -J voffice1 <user>@<rack-transit-ip> \
#               'sudo bash -s' -- check dc0 < scripts/dc-snap-proxy.sh
#   install: same with 'install dc0' (idempotent; safe to re-run).
#
# ...EXCEPT `node`, WHICH RUNS ON A DC NODE OR A CHARM CONTAINER. Same split
# dc-cache-proxy.sh had to make for gate G17: `check` asserts rack-side facts a
# client cannot satisfy, so pointing a client at it fails for reasons that say
# nothing about client reachability.
#   node:    bash -s -- node dc0 < scripts/dc-snap-proxy.sh    (ON the client)
# `node` touches ONLY curl, grep and /etc/environment -- no root, no systemctl,
# no virsh, no generated-file diffs.
#
# ---------------------------------------------------------------------------
# THE MEASUREMENT THE WHOLE DESIGN TURNS ON (docs/audit/stage5-snap-proxy-
# measurements-20260731.txt section 1). The three failing applications are LXD
# CONTAINERS, not nodes: `juju status` prints mysql-innodb-cluster/0 at
# 10.12.12.116, which is no D-134 node octet. Measured INSIDE the container:
#   eth0 10.12.8.122/22 (metal-admin)   eth1 10.12.12.116/22 (metal-internal)
#   ip route get 10.12.8.4  ->  dev eth0 src 10.12.8.122
#   NO default route; DIRECT api.snapcraft.io -> curl (7); via a proxy -> 200
# So the client ACL is the METAL-ADMIN /22, NOT metal-internal. Binding the ACL
# to the address `juju status` displays would have denied every client this
# proxy exists to serve.
#
# WHY A DEDICATED SQUID INSTANCE AND NOT THE PACKAGED squid.service. MEASURED on
# BOTH racks: MAAS's own squid holds *:3128 AND *:8000, WILDCARD-bound
# (`squid -N -d 5 -f /var/snap/maas/<rev>/proxy/maas-proxy.conf`). Ubuntu noble's
# squid.conf pins `http_port 3128` and /etc/default/squid is EMPTY, so there is
# no supported way to move the packaged unit off 3128 -- it would FATAL on bind.
# `include /etc/squid/conf.d/*.conf` does sit before `http_access deny all`, but
# a drop-in cannot UNSET an http_port. Hence: our own config, our own unit, our
# own pid file, and `install` DISABLES the packaged unit so it cannot crash-loop
# and poison triage. Port 3129 -- 3128/8000 are MAAS's, 3142 is dc1's
# apt-cacher-ng, 80 is dc0's mirror nginx. Bound to LISTEN specifically, never
# wildcard. Coexistence with MAAS's squid is MEASURED, not reasoned: snapd
# namespaces its shm as `snap.maas.squid-cf__*` while a host squid creates bare
# `squid-cf__*`, and MAAS's squid runs `-N` with no pid file.
#
# THIS SCRIPT OWNS ITS OWN NETWORK PREREQUISITES -- the hard-won D-135 lesson
# (dc-cache-proxy.sh, 2026-07-27). It never reuses <site>-mirror-net.service or
# <site>-cache-proxy-net.service: on a DC running only one of those, tearing that
# one down must not be able to kill this proxy, and this proxy must be rebuildable
# without reinstalling the other's apparatus. Artifacts are named for THIS
# service. The three net units set IDENTICAL MEASURED values with `ip addr
# replace` / `ip route replace`, so they are no-ops for each other.
#
# WHY SQUID (research capture section 9; "research is cheap, guesswork is
# expensive"). Every documented snap endpoint is :443
# (snapcraft.io/docs/reference/administration/network-requirements/), so a
# CONNECT-capable forward proxy is REQUIRED and there is NO caching benefit --
# squid "does not have access to those encrypted messages"
# (wiki.squid-cache.org/Features/HTTPS) and the only way to change that is
# SslBump, i.e. a deliberate MITM, which is not wanted. So the choice is on
# support and default posture: squid is in noble MAIN (6.14-0ubuntu0.24.04.4,
# security-supported) and ships `http_access deny CONNECT !SSL_ports` plus a
# terminal `deny all` by DEFAULT; tinyproxy is universe with six CVEs in "Needs
# evaluation" and documents "If no ConnectPort line is found, then all ports are
# allowed". nginx cannot do CONNECT without a third-party module.
#
# >>> BUILD-TIME CHOICE, NOT A RULING -- READ BEFORE APPLYING. <<<
# The ruling covers "forward proxy + juju snap-https-proxy" and leaves the
# service choice to build time. It does NOT rule on DESTINATION ALLOWLISTING.
# This build restricts CONNECT to the Canonical-documented snap-store and CDN
# hosts (STORE_DOMAINS below), because "allow CONNECT to any :443" would hand
# the node planes general HTTPS egress and hollow out D-107 in spirit while
# leaving its letter intact. The cost is that an UNDOCUMENTED store/CDN host
# would be denied. Widening is ONE edit: add the host to STORE_DOMAINS, or
# replace the `http_access deny !snap_store` line with nothing to allow any
# :443. If the operator prefers the open form, say so and it changes here.
#
# CONSUMPTION (Stage 5), verified against THIS controller (juju 3.6.27):
#   juju model-config -m <model> snap-http-proxy=http://<LISTEN>:3129
#   juju model-config -m <model> snap-https-proxy=http://<LISTEN>:3129
#   # LEAVE snap-store-proxy / snap-store-proxy-url / snap-store-assertions EMPTY
#   # -- those are the Canonical Enterprise Store (ex Snap Store Proxy) product:
#   # snapd rejects proxy.store without a matching signed store assertion.
# Both keys take an http:// URL (a plain-HTTP proxy speaking CONNECT), never
# https://. juju's proxyupdater shells out `snap set system proxy.http=...
# proxy.https=...` and is a WATCHER, so setting the key after machines exist is
# expected to propagate.
# UNVERIFIED, CARRIED DELIBERATELY: whether a RUNNING snapd re-reads the value
# without a restart. snapd writes /etc/environment and reads it via
# EnvironmentFile= AT SERVICE START (snapd configcore/proxy.go; LP#1737332,
# LP#1791587). So per client verify BOTH `snap get system proxy` AND a real
# egress probe; if the first passes and the second fails, `systemctl restart
# snapd` and re-probe. `node` below asserts the /etc/environment half without
# needing root.
#
# READ THIS BEFORE TRIAGING A `REFUSE`. Two of check's probes reach THIRD-PARTY
# hosts, so a Canonical-side outage, a DNS failure or a dead upstream makes a
# perfectly healthy proxy REFUSE. THAT IS BY DESIGN, not a defect and not "the
# proxy is broken": the probe cannot tell "the CDN is down" from "the proxy is
# misconfigured", and guessing either way is the failure this repo keeps meeting.
# REFUSE means EVALUATE AGAIN, WITH MORE INFORMATION -- it is never a pass.
#
# >>> ONE ASSERTION IS NOT YET CONFIRMED AGAINST A LIVE SQUID -- READ BEFORE
# CITING `check` AS A GATE. <<< The DENY probe (3) expects the literal
# 'CONNECT tunnel failed, response 403'. That shape was MEASURED from a PORT
# deny (`deny CONNECT !SSL_ports`), never from the DSTDOMAIN deny
# (`deny !snap_store`) that is the rule actually being proven. If squid answers a
# dstdomain denial differently, that branch falls to REFUSE and `check` can then
# never PASS -- and under GA-R6 a gate that cannot PASS blocks a stage close as
# hard as one that cannot fail. So the FIRST live run is the confirmation: read
# the deny probe's actual output, and if it is not this string, re-point the
# assertion to the MEASURED shape and prove the new assertion can fail. A REFUSE
# on the first run is the expected-and-handled path, not a defect.
# The CDN probe in particular is a LIVENESS-COUPLED assertion: it exists because
# a config diff proves the wildcard `.snapcraftcontent.com` is WRITTEN, and only
# a real tunnel proves squid's dot-prefix matching actually ADMITS a subdomain.
# (Measured aside worth knowing: `core`'s own download URL came back on
# api.snapcraft.io, not a CDN host -- so for THIS snap the CDN entry may be
# unused. It stays because the allowlist must cover every snap, not the one
# probed, and snapd downloads through the same proxy as the API.)
#
# EXIT: 0 ok | 1 check/node failed | 2 bad args/unknown site
#       | 3 REFUSED (could not evaluate -- never a pass) | 4 install failed.
# ASCII + LF only.
set -uo pipefail

MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in check|install|node) ;; *)
  echo "usage: dc-snap-proxy.sh <check|install|node> <site>" >&2; exit 2 ;; esac

# ---------------------------------------------------------------------------
# Site table. ADD A SITE ONLY WITH MEASURED/RULED VALUES (hard rule 2); the
# harness rejects rows without a MEASURED tag. CLIENT_CIDR is the plane the
# charm containers SOURCE FROM toward the utility .4 -- measured, never the
# address juju status happens to print.
# ---------------------------------------------------------------------------
case "$SITE" in
  dc0)
    # MEASURED/RULED 2026-07-31 (this build's capture,
    # docs/audit/stage5-snap-proxy-measurements-20260731.txt):
    #   LISTEN 10.12.8.4/22 -- the D-134 utility .4 (artifact service), ALREADY
    #     aliased on virbr2 by dc0-mirror-net.service; no new octet is needed,
    #     which is what the ruling note asked for.
    #   EDGE_GW 10.12.4.1 -- dc0 edge LAN gw; the rack's measured default route
    #     (`default via 10.12.4.1 dev virbr5`) and the proxy's upstream egress.
    #   CLIENT_CIDR 10.12.8.0/22 -- metal-admin. MEASURED from inside
    #     mysql-innodb-cluster/0: `ip route get 10.12.8.4` -> src 10.12.8.122.
    #     Cross-check: lib-net.sh vr1-dc0 PLANE_CIDRS[1] = 10.12.8.0/22.
    UTIL_NET="vr1-dc0-metal-admin"
    LISTEN_CIDR="10.12.8.4/22"
    EDGE_GW="10.12.4.1"
    CLIENT_CIDR="10.12.8.0/22"
    ;;
  dc1)
    # MEASURED/RULED 2026-07-31:
    #   LISTEN 10.12.68.4/22 -- utility .4, MEASURED already aliased on virbr6
    #     by dc1-cache-proxy-net.service (`ip -4 -o addr` on vr1-dc1-rack).
    #   EDGE_GW 10.12.64.1 -- MEASURED as the dc1 rack's default route
    #     (`default via 10.12.64.1 dev virbr4`).
    #   CLIENT_CIDR 10.12.68.0/22 -- dc1 metal-admin, from lib-net.sh's vr1-dc1
    #     arm (PLANE_CIDRS[1]), NOT mirrored from dc0 by symmetry.
    # HONEST LIMIT (hard rule 2): dc1 has NO deployed model, so the
    # container-sources-from-metal-admin finding is measured on dc0 ONLY. If dc1
    # ever carves containers differently, `node dc1` is what will say so -- and a
    # red there is the intended behaviour, not a false red.
    UTIL_NET="vr1-dc1-metal-admin"
    LISTEN_CIDR="10.12.68.4/22"
    EDGE_GW="10.12.64.1"
    CLIENT_CIDR="10.12.68.0/22"
    ;;
  *) echo "FAIL: unknown site '$SITE' -- add a MEASURED row block first" >&2; exit 2 ;;
esac
LISTEN="${LISTEN_CIDR%%/*}"

# 3129: MEASURED FREE on both racks while 3128 and 8000 are MAAS's squid, 3142 is
# dc1's apt-cacher-ng and 80 is dc0's mirror nginx.
PROXY_PORT="3129"
# TEST SEAM. EMPTY in every production invocation, so the paths below are
# byte-identical to the absolute ones. The harness points it at a mktemp tree so
# that check's five file assertions and node's /etc/environment assertion can be
# driven in BOTH directions -- an assertion nobody can turn red is decoration.
# In-repo precedent for the idiom: MAAS_SNAP_ROOT (maas-region-power-key.sh),
# CREDS_ROOT (creds-audit.sh), CLOUD_SNAPSHOT_ROOT (cloud-snapshot.sh).
SYSROOT="${DC_SNAP_PROXY_SYSROOT:-}"
SQUID_CONF="${SYSROOT}/etc/squid/${SITE}-snap-proxy.conf"
PROXY_UNIT="${SYSROOT}/etc/systemd/system/${SITE}-snap-proxy.service"
NET_HELPER="${SYSROOT}/usr/local/sbin/${SITE}-snap-proxy-net-apply"
NET_UNIT="${SYSROOT}/etc/systemd/system/${SITE}-snap-proxy-net.service"
RESOLVED_DROPIN="${SYSROOT}/etc/systemd/resolved.conf.d/${SITE}-snap-proxy.conf"
ENV_FILE="${SYSROOT}/etc/environment"
RUNTIME_DIR_NAME="${SITE}-snap-proxy"
PID_FILE="/run/${RUNTIME_DIR_NAME}/squid.pid"
LOG_DIR="/var/log/squid"

# The Canonical-documented snap endpoint set, all :443
# (snapcraft.io/docs/reference/administration/network-requirements/ and its
# source of record forum.snapcraft.io/t/network-requirements/5147, CDN list dated
# 2021-10-25). A leading dot is squid's "this domain and all subdomains", so
# `.snapcraftcontent.com` covers storage./canonical-lgw01.cdn./cloudfront.cdn.
# `.cdn.snapcraft.io` is marked DEPRECATED upstream and kept only so an older
# client is not denied. THE DOWNLOAD IS PROXIED TOO -- snapd uses ONE http client
# for the API and the CDN fetch (snapd store/store.go), so an allowlist naming
# only api.snapcraft.io would authenticate and then fail the download.
STORE_DOMAINS="api.snapcraft.io dashboard.snapcraft.io login.ubuntu.com .snapcraftcontent.com .cdn.snapcraft.io"

# Behavioral probe targets, shared by the rack-side `check` and the client-side
# `node` so the two cannot drift apart. Shapes MEASURED 2026-07-31 (capture s7).
STORE_PROBE_URL="https://api.snapcraft.io/v2/snaps/info/core"
CDN_PROBE_URL="https://canonical-lgw01.cdn.snapcraftcontent.com/"
# A destination deliberately NOT in STORE_DOMAINS. RFC 2606 / IANA reserved, so it
# can never become a real dependency. Its only job is to prove the allowlist is in
# force -- a proxy that answers this one is an OPEN proxy.
DENY_PROBE_URL="https://example.com/"

# ---------------------------------------------------------------------------
# Generated content -- the single source of truth for check AND install. Every
# generator is diffed by `check`, so a hand-edit on the rack is a FAILURE, not a
# silent drift (assert the ARTIFACT, never the intent).
# ---------------------------------------------------------------------------
gen_squid_conf() {
  cat <<EOF
# ${SITE} snap forward proxy -- generated by scripts/dc-snap-proxy.sh
# (D-135/D-107 RULING 2, 2026-07-31); do not hand-edit.
#
# DEDICATED INSTANCE. The packaged squid.service cannot serve: Ubuntu noble's
# squid.conf pins http_port 3128 and MAAS's own squid holds *:3128 wildcard-bound
# on this rack, so the packaged unit FATALs on bind. /etc/default/squid is empty,
# so there is no supported way to repoint it. install disables it.
#
# CONNECT-ONLY, DESTINATION-RESTRICTED. Every documented snap endpoint is :443,
# so this proxy never sees a plain GET from snapd. Safe_ports is 443 alone on
# purpose: this is a snap path, not a general egress.
#
# Consume via:
#   juju model-config -m <model> snap-http-proxy=http://${LISTEN}:${PROXY_PORT}
#   juju model-config -m <model> snap-https-proxy=http://${LISTEN}:${PROXY_PORT}
# Leave snap-store-proxy / -url / -assertions EMPTY (different product).
visible_hostname ${SITE}-snap-proxy
pid_filename ${PID_FILE}
http_port ${LISTEN}:${PROXY_PORT}

# Clients: the metal-admin plane. MEASURED as the source the charm CONTAINERS
# select toward this address, not the address juju status displays.
acl snap_clients src ${CLIENT_CIDR}
# The rack's own probe source, so this script's behavioral smoke test can run.
# NOTE THE MECHANISM, because an earlier version of this rule misdescribed it and
# was DEAD: squid binds ${LISTEN}:${PROXY_PORT} ONLY, so a probe can never arrive
# on 127.0.0.1 and a loopback ACL matched nothing. The rack connecting to a
# LOCAL address selects that address as its source, so ${LISTEN}/32 is the
# expected match; if it is not, the probe still falls through to snap_clients
# because ${LISTEN} lies inside ${CLIENT_CIDR}. Both paths are admitted on
# purpose: this rule exists so that NARROWING snap_clients cannot silently break
# \`check\`. It proves upstream reach, CONNECT capability and that the destination
# rule is in force; CLIENT AUTHORIZATION is what \`node\` proves, from a real client.
acl snap_probe src ${LISTEN}/32
acl SSL_ports port 443
acl Safe_ports port 443
acl CONNECT method CONNECT
acl snap_store dstdomain ${STORE_DOMAINS}

http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny !snap_store
http_access allow snap_clients
http_access allow snap_probe
http_access deny all

# No caching is POSSIBLE here (opaque CONNECT tunnels) -- stated explicitly so
# nobody later "fixes" the absence of a cache_dir by adding SslBump.
cache deny all

access_log ${LOG_DIR}/${SITE}-snap-proxy.access.log
cache_log ${LOG_DIR}/${SITE}-snap-proxy.cache.log
coredump_dir /var/spool/squid
EOF
}

gen_proxy_unit() {
  cat <<EOF
[Unit]
Description=${SITE} snap forward proxy (dc-snap-proxy.sh; D-135/D-107 ruling 2, 2026-07-31)
Documentation=man:squid(8)
After=network-online.target nss-lookup.target ${SITE}-snap-proxy-net.service
Requires=${SITE}-snap-proxy-net.service
Wants=network-online.target

[Service]
Type=notify
NotifyAccess=all
PIDFile=${PID_FILE}
Group=proxy
RuntimeDirectory=${RUNTIME_DIR_NAME}
RuntimeDirectoryMode=0775
ExecStart=/usr/sbin/squid --foreground -f ${SQUID_CONF}
ExecReload=/bin/kill -HUP \$MAINPID
KillMode=mixed
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF
}

# --- the utility net layer THIS script owns ---------------------------------
# Same MEASURED mechanism dc-mirror.sh proved on both racks 2026-07-23 and
# dc-cache-proxy.sh took ownership of 2026-07-27: resolve the bridge from the
# libvirt network NAME at runtime, then idempotently place the .4 alias and the
# edge default route. Deliberately identical values, so the units coexist.
gen_net_helper() {
  cat <<EOF
#!/usr/bin/env bash
# ${SITE}-snap-proxy-net-apply -- generated by scripts/dc-snap-proxy.sh; do not hand-edit.
# Places the proxy LISTEN alias (D-134 utility .4) and the default route via the
# DC edge (the proxy's upstream egress). Bridge resolved from the libvirt network
# NAME at runtime (virbrN is auto-assigned, not stable identity -- hard rule 3).
set -euo pipefail
br_of() {
  virsh -c qemu:///system net-dumpxml "\$1" | sed -n "s/.*bridge name='\\([^']*\\)'.*/\\1/p"
}
ip addr replace ${LISTEN_CIDR} dev "\$(br_of ${UTIL_NET})"
ip route replace default via ${EDGE_GW}
EOF
}

gen_net_unit() {
  cat <<EOF
[Unit]
Description=${SITE} snap-proxy net (dc-snap-proxy.sh; D-135/D-107 -- LISTEN alias + edge default route)
After=libvirtd.service ${SITE}-rack-legs.service network-online.target
Wants=network-online.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${NET_HELPER}

[Install]
WantedBy=multi-user.target
EOF
}

gen_resolved_dropin() {
  cat <<EOF
# ${SITE} snap-proxy DNS (generated by scripts/dc-snap-proxy.sh; D-135/D-107).
# System DNS = the DC edge resolver: the PROXY must resolve api.snapcraft.io and
# the snap CDN to open a tunnel, and the rack has no other working resolver.
# The MAAS rack agent ignores resolv.conf (D-131), so node DNS is unaffected.
[Resolve]
DNS=${EDGE_GW}
EOF
}

# Bridge for the utility net, resolved from the libvirt net NAME at runtime
# (virbrN is auto-assigned, not stable identity -- hard rule 3).
util_bridge() {
  virsh -c qemu:///system net-dumpxml "$UTIL_NET" 2>/dev/null \
    | sed -n "s/.*bridge name='\([^']*\)'.*/\1/p"
}

# ---------------------------------------------------------------------------
# check -- THREE outcomes, never two: PASS (0), FAIL (1), REFUSE (3).
# GA-R6: this is what a stage would close on, so it must be able to fail, it
# asserts CONTENT and not existence, and a state it cannot evaluate REFUSES
# rather than defaulting to success. "Could not look" is never "nothing there".
# It mutates no config or service state (the probes may add lines to squid's
# access log -- benign, and the proof it actually serves).
# ---------------------------------------------------------------------------
FAILED=0; REFUSED=0
say(){     echo "  OK      $1"; }
miss(){    echo "  MISS    $1"; FAILED=1; }
refuse(){  echo "  REFUSE  $1"; REFUSED=1; }

check_file() { # path gen_fn
  if [ ! -f "$1" ]; then miss "$1 absent"; return; fi
  if diff -q <("$2") "$1" >/dev/null 2>&1; then say "$1 matches generated"
  else miss "$1 DIFFERS from generated content"; fi
}

do_check() {
  echo "dc-snap-proxy check ($SITE) -- ${LISTEN}:${PROXY_PORT}, clients ${CLIENT_CIDR}"

  # -- the BINARY, not the package record. An absence here is an OBSERVED absence,
  # so it FAILS; it does not refuse. Fail-loud with the exact apt line, like
  # dc-mirror/dc-cache-proxy.
  # WHY NOT `dpkg -s squid`: it exits 0 for `Status: deinstall ok config-files`,
  # so a PURGED squid reports as present and the failure resurfaces later as a
  # mysterious unit that will not start. The existence-vs-content class again --
  # assert the thing that has to be there, which is the executable.
  if [ -x "${SYSROOT}/usr/sbin/squid" ]; then say "squid binary present (${SYSROOT}/usr/sbin/squid)"
  else miss "squid binary absent at ${SYSROOT}/usr/sbin/squid -- run: apt-get update && apt-get install -y squid"; fi

  # -- the five artifacts this script owns, asserted by CONTENT.
  check_file "$SQUID_CONF"       gen_squid_conf
  check_file "$PROXY_UNIT"       gen_proxy_unit
  check_file "$NET_HELPER"       gen_net_helper
  check_file "$NET_UNIT"         gen_net_unit
  check_file "$RESOLVED_DROPIN"  gen_resolved_dropin

  # -- units. ENABLED matters as much as active: without it the LISTEN alias and
  # the proxy both vanish at the next rack reboot, and this rack has rebooted.
  if ! command -v systemctl >/dev/null 2>&1; then
    refuse "systemctl is not present -- cannot evaluate unit state"
  else
    for u in "${SITE}-snap-proxy.service" "${SITE}-snap-proxy-net.service"; do
      if [ "$(systemctl is-enabled "$u" 2>/dev/null)" = "enabled" ]; then
        say "$u enabled"
      else miss "$u not enabled (it would not survive a reboot)"; fi
    done
    if systemctl is-active --quiet "${SITE}-snap-proxy.service" 2>/dev/null; then
      say "${SITE}-snap-proxy.service active"
    else miss "${SITE}-snap-proxy.service not active"; fi
    # The PACKAGED unit must be neither active nor enabled: its squid.conf pins
    # http_port 3128, which MAAS's squid holds, so it can only crash-loop -- and
    # a crash-looping squid alongside ours poisons every later triage.
    if systemctl is-active --quiet squid 2>/dev/null; then
      miss "the PACKAGED squid.service is ACTIVE -- it pins http_port 3128 (MAAS's squid holds it); run: systemctl disable --now squid"
    elif [ "$(systemctl is-enabled squid 2>/dev/null)" = "enabled" ]; then
      miss "the PACKAGED squid.service is ENABLED -- it will fight for 3128 at the next boot; run: systemctl disable --now squid"
    else say "packaged squid.service neither active nor enabled"; fi
  fi

  # -- the listener must be on the LISTEN address SPECIFICALLY, not wildcard: a
  # wildcard bind would collide with MAAS's squid and would also expose the proxy
  # on the transit and provider legs.
  if ! command -v ss >/dev/null 2>&1; then
    refuse "ss is not present -- cannot evaluate the listener"
  elif ss -ltn 2>/dev/null | grep -q "${LISTEN}:${PROXY_PORT} "; then
    say "listening on ${LISTEN}:${PROXY_PORT}"
  elif ss -ltn 2>/dev/null | grep -qE "(\*|0\.0\.0\.0):${PROXY_PORT} "; then
    miss "port ${PROXY_PORT} is bound WILDCARD, not on ${LISTEN} -- refusing to accept a broader bind than generated"
  else
    miss "nothing listening on ${LISTEN}:${PROXY_PORT}"
  fi

  # -- the net layer's live effect (the artifacts were diffed above).
  if ! command -v virsh >/dev/null 2>&1; then
    refuse "virsh is not present -- cannot resolve ${UTIL_NET}'s bridge, so the LISTEN alias is UNKNOWN"
  else
    br="$(util_bridge)"
    if [ -z "$br" ]; then
      refuse "libvirt network '${UTIL_NET}' did not resolve to a bridge -- the alias is UNKNOWN, not absent"
    elif ip -4 -o addr show dev "$br" 2>/dev/null | grep -q " ${LISTEN}/"; then
      say "${LISTEN_CIDR} on $br ($UTIL_NET)"
    else
      miss "${LISTEN_CIDR} absent on the metal-admin bridge (run: dc-snap-proxy.sh install $SITE)"
    fi
  fi
  if ip route show default 2>/dev/null | grep -q "via ${EDGE_GW}"; then
    say "default route via edge ${EDGE_GW}"
  else miss "default route via ${EDGE_GW} absent -- the proxy has no upstream egress"; fi

  # -- BEHAVIORAL. Three probes from the SAME source, so the difference between
  # them isolates exactly one rule each. A 200 alone is never evidence.
  if ! command -v curl >/dev/null 2>&1; then
    refuse "curl is not present -- cannot evaluate whether the proxy actually serves"
  else
    # (1) ALLOW direction, asserted on CONTENT. MEASURED body shape 2026-07-31:
    #     the store API answers with a "channel-map" array carrying a sha3-384
    #     download digest. No squid error page contains either.
    body="$(mktemp)"; cerr="$(mktemp)"
    code="$(curl -sS -o "$body" -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
            -m 30 -H 'Snap-Device-Series: 16' "$STORE_PROBE_URL" 2>"$cerr")"; crc=$?
    if [ "$crc" -ne 0 ]; then
      refuse "could not reach $STORE_PROBE_URL THROUGH http://${LISTEN}:${PROXY_PORT} (curl exit $crc: $(head -1 "$cerr")) -- unreachable is not 'nothing there'"
    elif [ "$code" != "200" ]; then
      miss "store API returned HTTP $code through the proxy (want 200)"
    elif ! grep -q '"channel-map"' "$body"; then
      miss "body served for the store API has no \"channel-map\" -- what came back is not the snap store's answer, whatever its status line said"
    elif ! grep -q 'sha3-384' "$body"; then
      miss "store API body carries no sha3-384 download digest -- not a usable snap info response"
    else
      say "CONNECT to the store API: curl exit 0, HTTP 200, channel-map + sha3-384 present"
    fi
    rm -f "$body" "$cerr"

    # (2) The CDN tunnel must be ALLOWED. The CDN answers 403 to a bare GET of /,
    #     so the assertion is NOT on the status code -- it is that squid did not
    #     refuse the tunnel. MEASURED distinction: a proxy denial is curl exit 56
    #     with "CONNECT tunnel failed, response 403"; a server 403 is curl exit 0.
    #     Without this the allowlist could name only api.snapcraft.io and every
    #     snap would authenticate and then fail its DOWNLOAD.
    cout="$(curl -sS -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
            -m 30 "$CDN_PROBE_URL" 2>&1)"; crc=$?
    if grep -q 'CONNECT tunnel failed' <<<"$cout"; then
      miss "the proxy REFUSED the CDN tunnel to $CDN_PROBE_URL -- the allowlist is too narrow and snap DOWNLOADS would fail after a successful API call"
    elif [ "$crc" -eq 0 ]; then
      say "CDN tunnel allowed (curl exit 0; the CDN's own status was '$cout')"
    else
      refuse "CDN probe gave curl exit $crc with '$cout' -- neither an allowed tunnel nor a proxy denial; refusing to interpret it"
    fi

    # (3) DENY direction. This is the assertion that proves the destination
    #     allowlist is actually in force rather than merely written down. A proxy
    #     that answers this one is an OPEN proxy on the node plane.
    dout="$(curl -sS -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
            -m 20 "$DENY_PROBE_URL" 2>&1)"; drc=$?
    if grep -q 'CONNECT tunnel failed, response 403' <<<"$dout"; then
      say "non-allowlisted destination is DENIED (403 at CONNECT) -- the allowlist is in force"
    elif [ "$drc" -eq 0 ]; then
      miss "the proxy SERVED a non-allowlisted destination ($DENY_PROBE_URL, result '$dout') -- this is an OPEN forward proxy on the node plane, not a snap path"
    else
      refuse "deny probe gave curl exit $drc with '$dout' -- neither a 403 denial nor a success; the allowlist state is UNKNOWN"
    fi
  fi

  if [ "$FAILED" -ne 0 ]; then echo "dc-snap-proxy check ($SITE): FAIL"; exit 1; fi
  if [ "$REFUSED" -ne 0 ]; then
    echo "dc-snap-proxy check ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
  fi
  echo "dc-snap-proxy check ($SITE): PASS"; exit 0
}

# ---------------------------------------------------------------------------
# install -- idempotent; requires root. Net layer FIRST (nothing can bind the
# LISTEN address until it exists), then fail loud on the missing package, then
# the config, then a config PARSE before anything is enabled.
# ---------------------------------------------------------------------------
do_install() {
  [ "$(id -u)" = "0" ] || { echo "FAIL: install requires root" >&2; exit 4; }
  umask 022
  # Prereq: the libvirt utility net must EXIST (we resolve its bridge). That is a
  # rack-standup artifact (dc-rack-net.sh), never a mirror or cache-proxy one, so
  # this service depends on no other artifact strategy's apparatus.
  br="$(util_bridge)"
  if [ -z "$br" ]; then
    echo "FAIL: libvirt network '${UTIL_NET}' not found -- cannot resolve its bridge" >&2
    echo "  the rack net layer comes first: dc-rack-net.sh install $SITE" >&2
    exit 4
  fi
  install -m 0755 /dev/null "$NET_HELPER" || exit 4
  gen_net_helper > "$NET_HELPER" || exit 4
  chmod 0755 "$NET_HELPER" || exit 4
  gen_net_unit > "$NET_UNIT" || exit 4
  mkdir -p "$(dirname "$RESOLVED_DROPIN")" || exit 4
  gen_resolved_dropin > "$RESOLVED_DROPIN" || exit 4
  systemctl daemon-reload || exit 4
  systemctl enable "${SITE}-snap-proxy-net.service" >/dev/null 2>&1 || exit 4
  systemctl restart "${SITE}-snap-proxy-net.service" \
    || { echo "FAIL: ${SITE}-snap-proxy-net.service did not apply" >&2; exit 4; }
  systemctl restart systemd-resolved >/dev/null 2>&1 || true

  # Prereq 2: the BINARY. NEVER apt-installed from here -- the repo pattern is
  # fail-loud with the exact line, so package installs stay operator-visible.
  # Same reason as `check`: `dpkg -s` exits 0 on a purged `deinstall ok
  # config-files` package, which would let install sail past this prereq and die
  # confusingly at `squid -k parse` or at unit start instead.
  if [ ! -x "${SYSROOT}/usr/sbin/squid" ]; then
    echo "FAIL: squid binary absent at ${SYSROOT}/usr/sbin/squid" >&2
    echo "  net layer is up -- run: apt-get update && apt-get install -y squid" >&2
    echo "  then re-run: dc-snap-proxy.sh install $SITE" >&2
    exit 4
  fi
  # The packaged unit is stood DOWN before ours goes up: its squid.conf pins
  # http_port 3128, which MAAS's squid holds wildcard-bound, so it can only fail.
  systemctl disable --now squid >/dev/null 2>&1 || true

  mkdir -p "$(dirname "$SQUID_CONF")" "$LOG_DIR" || exit 4
  gen_squid_conf > "$SQUID_CONF" || exit 4
  # PARSE BEFORE ENABLE. A config that squid rejects must fail here, loudly, and
  # not as a mysterious unit that will not start.
  if ! squid -k parse -f "$SQUID_CONF" >/dev/null 2>&1; then
    echo "FAIL: squid rejected ${SQUID_CONF}:" >&2
    squid -k parse -f "$SQUID_CONF" >&2
    exit 4
  fi
  gen_proxy_unit > "$PROXY_UNIT" || exit 4
  systemctl daemon-reload || exit 4
  systemctl enable "${SITE}-snap-proxy.service" >/dev/null 2>&1 || exit 4
  systemctl restart "${SITE}-snap-proxy.service" \
    || { echo "FAIL: ${SITE}-snap-proxy.service did not start" >&2; exit 4; }
  echo "dc-snap-proxy install ($SITE): done -- running check:"
  do_check
}

# ---------------------------------------------------------------------------
# node -- run FROM a DC node OR a charm LXD container. Read-only, no root, and
# deliberately touches NOTHING rack-side: only curl, grep and /etc/environment.
# Three outcomes, never two: PASS (0), FAIL (1), REFUSE (3).
# ---------------------------------------------------------------------------
NFAILED=0; NREFUSED=0
nok(){     echo "  OK      $1"; }
nno(){     echo "  FAIL    $1"; NFAILED=1; }
nrefuse(){ echo "  REFUSE  $1"; NREFUSED=1; }

do_node() {
  echo "dc-snap-proxy node check ($SITE) -- from this client, against ${LISTEN}:${PROXY_PORT}"

  if ! command -v curl >/dev/null 2>&1; then
    nrefuse "curl is not present on this client -- cannot evaluate proxy reachability"
  else
    # (1) The client can actually CONNECT through the proxy to the store, and what
    # comes back is the store's answer -- not a 200 from something else.
    body="$(mktemp)"; cerr="$(mktemp)"
    code="$(curl -sS -o "$body" -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
            -m 30 -H 'Snap-Device-Series: 16' "$STORE_PROBE_URL" 2>"$cerr")"; crc=$?
    if [ "$crc" -ne 0 ]; then
      if grep -q 'CONNECT tunnel failed, response 403' "$cerr"; then
        nno "the proxy DENIED this client (403 at CONNECT) -- this client's source address is outside ${CLIENT_CIDR}; measure it with 'ip route get ${LISTEN}' and fix the site row, do not widen the ACL blind"
      else
        nrefuse "could not reach $STORE_PROBE_URL THROUGH http://${LISTEN}:${PROXY_PORT} (curl exit $crc: $(head -1 "$cerr")) -- unreachable is not 'nothing there'"
      fi
    elif [ "$code" != "200" ]; then
      nno "store API returned HTTP $code through the proxy (want 200)"
    elif ! grep -q '"channel-map"' "$body"; then
      nno "body served for the store API has no \"channel-map\" -- not the snap store's answer, whatever its status line said"
    elif ! grep -q 'sha3-384' "$body"; then
      nno "store API body carries no sha3-384 download digest -- not a usable snap info response"
    else
      nok "CONNECT to the store API through the proxy: curl exit 0, HTTP 200, channel-map + sha3-384 present"
    fi
    rm -f "$body" "$cerr"

    # (2) The allowlist must be in force FROM HERE too. Asserting it only on the
    # rack would miss a client-specific allow rule, and an open proxy on the node
    # plane is a bigger problem than a broken snap install.
    dout="$(curl -sS -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
            -m 20 "$DENY_PROBE_URL" 2>&1)"; drc=$?
    if grep -q 'CONNECT tunnel failed, response 403' <<<"$dout"; then
      nok "non-allowlisted destination is DENIED from this client -- the allowlist is in force"
    elif [ "$drc" -eq 0 ]; then
      nno "the proxy SERVED a non-allowlisted destination from this client ($DENY_PROBE_URL, result '$dout') -- OPEN forward proxy on the node plane"
    else
      nrefuse "deny probe gave curl exit $drc with '$dout' -- neither a 403 denial nor a success; the allowlist state is UNKNOWN from here"
    fi
  fi

  # (3) snapd's own view of the proxy, WITHOUT root. juju's proxyupdater shells
  # out `snap set system proxy.http/https`, and snapd's configcore writes those
  # into /etc/environment (envFilePath = "/etc/environment"), which is
  # world-readable -- whereas `snap get system proxy` needs root and would break
  # this subcommand's no-root property.
  if [ ! -r "$ENV_FILE" ]; then
    nrefuse "$ENV_FILE is not readable -- cannot evaluate whether the juju snap proxy keys reached this client"
  else
    ENVLINE="$(grep -E '^[[:space:]]*https_proxy=' "$ENV_FILE" 2>/dev/null | tail -1)"
    if [ -z "$ENVLINE" ]; then
      nno "no https_proxy in $ENV_FILE -- the juju keys have not reached this client; run: juju model-config -m <model> snap-http-proxy=http://${LISTEN}:${PROXY_PORT} snap-https-proxy=http://${LISTEN}:${PROXY_PORT}"
    elif grep -q "http://${LISTEN}:${PROXY_PORT}" <<<"$ENVLINE"; then
      nok "snapd's proxy is this DC's proxy ($ENV_FILE: $ENVLINE)"
    else
      nno "https_proxy in $ENV_FILE points somewhere ELSE: $ENVLINE -- expected http://${LISTEN}:${PROXY_PORT}"
    fi
  fi

  if [ "$NFAILED" -ne 0 ]; then echo "dc-snap-proxy node ($SITE): FAIL"; exit 1; fi
  if [ "$NREFUSED" -ne 0 ]; then
    echo "dc-snap-proxy node ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
  fi
  echo "dc-snap-proxy node ($SITE): PASS"; exit 0
}

case "$MODE" in
  check)   do_check ;;
  install) do_install ;;
  node)    do_node ;;
esac