#!/usr/bin/env bash
# scripts/dc-snap-proxy.sh <check|install|node> <site> -- per-DC snap FORWARD
# PROXY (squid, CONNECT-only). Builds D-135/D-107 RULING 2, RULED 2026-07-31
# (GA-R5), operator utterance verbatim:
# "HTTP(S) forward proxy in the DC utility band + juju snap-https-proxy (Recommended)"
#
# WHY IT EXISTS. mysql-innodb-cluster (mysql-shell), ovn-central
# (prometheus-ovn-exporter) and vault (core) all fail their install hook because
# they cannot reach api.snapcraft.io -- the D-135 items 2-3 gap. It hits dc1
# IDENTICALLY: apt-cacher-ng proxies apt, never snaps. D-107's core statement
# stays TRUE and UNAMENDED -- nodes reach an IN-DC proxy, not the store.
#
# RUNS ON THE DC RACK HOST, invoked over ssh exactly like dc-mirror.sh /
# dc-cache-proxy.sh (no repo clone needed there):
# check: ssh -i <dc-key> -J voffice1 <user>@<rack-transit-ip> \
# 'sudo bash -s' -- check dc0 < scripts/dc-snap-proxy.sh
# install: same with 'install dc0' (idempotent; safe to re-run).
#
# ...EXCEPT `node`, WHICH RUNS ON A DC NODE OR A CHARM CONTAINER. Same split
# dc-cache-proxy.sh had to make for gate G17: `check` asserts rack-side facts a
# client cannot satisfy, so pointing a client at it fails for reasons that say
# nothing about client reachability.
# node: bash -s -- node dc0 < scripts/dc-snap-proxy.sh (ON the client)
# `node` touches ONLY curl, grep and /etc/environment -- no root, no systemctl,
# no virsh, no generated-file diffs.
#
# ---------------------------------------------------------------------------
# THE MEASUREMENT THE WHOLE DESIGN TURNS ON (docs/audit/stage5-snap-proxy-
# measurements-20260731.txt section 1). The three failing applications are LXD
# CONTAINERS, not nodes: `juju status` prints mysql-innodb-cluster/0 at
# 10.12.12.116, which is no D-134 node octet. Measured INSIDE the container:
# eth0 10.12.8.122/22 (metal-admin) eth1 10.12.12.116/22 (metal-internal)
# ip route get 10.12.8.4 -> dev eth0 src 10.12.8.122
# NO default route; DIRECT api.snapcraft.io -> curl (7); via a proxy -> 200
# So the client ACL is the METAL-ADMIN /22, NOT metal-internal. Binding the ACL
# to the address `juju status` displays would have denied every client this
# proxy exists to serve.
#
# WHY A DEDICATED SQUID INSTANCE AND NOT THE PACKAGED squid.service. MEASURED on
# BOTH racks: MAAS's own squid holds *:3128 AND *:8000, WILDCARD-bound
# (`squid -N -d 5 -f /var/snap/maas/<rev>/proxy/maas-proxy.conf`). Ubuntu noble's
# squid.conf pins `http_port 3128` and /etc/default/squid is EMPTY, so there is
# no supported way to move the packaged unit off 3128 -- it would FATAL on bind.
# `include /etc/squid/conf.d/*.conf` does sit before `http_access deny all`, but
# a drop-in cannot UNSET an http_port. Hence: our own config, our own unit, our
# own pid file, and `install` DISABLES the packaged unit so it cannot crash-loop
# and poison triage. Port 3129 -- 3128/8000 are MAAS's, 3142 is dc1's
# apt-cacher-ng, 80 is dc0's mirror nginx. Bound to LISTEN specifically, never
# wildcard. Coexistence with MAAS's squid is MEASURED, not reasoned: snapd
# namespaces its shm as `snap.maas.squid-cf__*` while a host squid creates bare
# `squid-cf__*`, and MAAS's squid runs `-N` with no pid file.
#
# THIS SCRIPT OWNS ITS OWN NETWORK PREREQUISITES -- the hard-won D-135 lesson
# (dc-cache-proxy.sh, 2026-07-27). It never reuses <site>-mirror-net.service or
# <site>-cache-proxy-net.service: on a DC running only one of those, tearing that
# one down must not be able to kill this proxy, and this proxy must be rebuildable
# without reinstalling the other's apparatus. Artifacts are named for THIS
# service. The three net units set IDENTICAL MEASURED values with `ip addr
# replace` / `ip route replace`, so they are no-ops for each other.
#
# WHY SQUID (research capture section 9; "research is cheap, guesswork is
# expensive"). Every documented snap endpoint is :443
# (snapcraft.io/docs/reference/administration/network-requirements/), so a
# CONNECT-capable forward proxy is REQUIRED and there is NO caching benefit --
# squid "does not have access to those encrypted messages"
# (wiki.squid-cache.org/Features/HTTPS) and the only way to change that is
# SslBump, i.e. a deliberate MITM, which is not wanted. So the choice is on
# support and default posture: squid is in noble MAIN (6.14-0ubuntu0.24.04.4,
# security-supported) and ships `http_access deny CONNECT !SSL_ports` plus a
# terminal `deny all` by DEFAULT; tinyproxy is universe with six CVEs in "Needs
# evaluation" and documents "If no ConnectPort line is found, then all ports are
# allowed". nginx cannot do CONNECT without a third-party module.
#
# >>> BUILD-TIME CHOICE, NOT A RULING -- READ BEFORE APPLYING. <<<
# The ruling covers "forward proxy + juju snap-https-proxy" and leaves the
# service choice to build time. It does NOT rule on DESTINATION ALLOWLISTING.
# This build restricts CONNECT to the Canonical-documented snap-store and CDN
# hosts (STORE_DOMAINS below), because "allow CONNECT to any :443" would hand
# the node planes general HTTPS egress and hollow out D-107 in spirit while
# leaving its letter intact. The cost is that an UNDOCUMENTED store/CDN host
# would be denied. Widening is ONE edit: add the host to STORE_DOMAINS, or
# replace the `http_access deny !snap_store` line with nothing to allow any
# :443. If the operator prefers the open form, say so and it changes here.
#
# CONSUMPTION (Stage 5), verified against THIS controller (juju 3.6.27):
# juju model-config -m <model> snap-http-proxy=http://<LISTEN>:3129
# juju model-config -m <model> snap-https-proxy=http://<LISTEN>:3129
# # LEAVE snap-store-proxy / snap-store-proxy-url / snap-store-assertions EMPTY
# # -- those are the Canonical Enterprise Store (ex Snap Store Proxy) product:
# # snapd rejects proxy.store without a matching signed store assertion.
# Both keys take an http:// URL (a plain-HTTP proxy speaking CONNECT), never
# https://. juju's proxyupdater shells out `snap set system proxy.http=...
# proxy.https=...` and is a WATCHER, so setting the key after machines exist is
# expected to propagate.
# UNVERIFIED, CARRIED DELIBERATELY: whether a RUNNING snapd re-reads the value
# without a restart. snapd writes /etc/environment and reads it via
# EnvironmentFile= AT SERVICE START (snapd configcore/proxy.go; LP#1737332,
# LP#1791587). So per client verify BOTH `snap get system proxy` AND a real
# egress probe; if the first passes and the second fails, `systemctl restart
# snapd` and re-probe. `node` below asserts the /etc/environment half without
# needing root.
#
# READ THIS BEFORE TRIAGING A `REFUSE`. Two of check's probes reach THIRD-PARTY
# hosts, so a Canonical-side outage, a DNS failure or a dead upstream makes a
# perfectly healthy proxy REFUSE. THAT IS BY DESIGN, not a defect and not "the
# proxy is broken": the probe cannot tell "the CDN is down" from "the proxy is
# misconfigured", and guessing either way is the failure this repo keeps meeting.
# REFUSE means EVALUATE AGAIN, WITH MORE INFORMATION -- it is never a pass.
# The CDN probe in particular is a LIVENESS-COUPLED assertion: it exists because
# a config diff proves the wildcard `.snapcraftcontent.com` is WRITTEN, and only
# a real tunnel proves squid's dot-prefix matching actually ADMITS a subdomain.
# (Measured aside worth knowing: `core`'s own download URL came back on
# api.snapcraft.io, not a CDN host -- so for THIS snap the CDN entry may be
# unused. It stays because the allowlist must cover every snap, not the one
# probed, and snapd downloads through the same proxy as the API.)
#
# EXIT: 0 ok | 1 check/node failed | 2 bad args/unknown site
# | 3 REFUSED (could not evaluate -- never a pass) | 4 install failed.
# ASCII + LF only.
set -uo pipefail
MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in check|install|node) ;; *)
echo "usage: dc-snap-proxy.sh <check|install|node> <site>" >&2; exit 2 ;; esac
# ---------------------------------------------------------------------------
# Site table. ADD A SITE ONLY WITH MEASURED/RULED VALUES (hard rule 2); the
# harness rejects rows without a MEASURED tag. CLIENT_CIDR is the plane the
# charm containers SOURCE FROM toward the utility .4 -- measured, never the
# address juju status happens to print.
# ---------------------------------------------------------------------------
case "$SITE" in
dc0)
# MEASURED/RULED 2026-07-31 (this build's capture,
# docs/audit/stage5-snap-proxy-measurements-20260731.txt):
# LISTEN 10.12.8.4/22 -- the D-134 utility .4 (artifact service), ALREADY
# aliased on virbr2 by dc0-mirror-net.service; no new octet is needed,
# which is what the ruling note asked for.
# EDGE_GW 10.12.4.1 -- dc0 edge LAN gw; the rack's measured default route
# (`default via 10.12.4.1 dev virbr5`) and the proxy's upstream egress.
# CLIENT_CIDR 10.12.8.0/22 -- metal-admin. MEASURED from inside
# mysql-innodb-cluster/0: `ip route get 10.12.8.4` -> src 10.12.8.122.
# Cross-check: lib-net.sh vr1-dc0 PLANE_CIDRS[1] = 10.12.8.0/22.
UTIL_NET="vr1-dc0-metal-admin"
LISTEN_CIDR="10.12.8.4/22"
EDGE_GW="10.12.4.1"
CLIENT_CIDR="10.12.8.0/22"
;;
dc1)
# MEASURED/RULED 2026-07-31:
# LISTEN 10.12.68.4/22 -- utility .4, MEASURED already aliased on virbr6
# by dc1-cache-proxy-net.service (`ip -4 -o addr` on vr1-dc1-rack).
# EDGE_GW 10.12.64.1 -- MEASURED as the dc1 rack's default route
# (`default via 10.12.64.1 dev virbr4`).
# CLIENT_CIDR 10.12.68.0/22 -- dc1 metal-admin, from lib-net.sh's vr1-dc1
# arm (PLANE_CIDRS[1]), NOT mirrored from dc0 by symmetry.
# HONEST LIMIT (hard rule 2): dc1 has NO deployed model, so the
# container-sources-from-metal-admin finding is measured on dc0 ONLY. If dc1
# ever carves containers differently, `node dc1` is what will say so -- and a
# red there is the intended behaviour, not a false red.
UTIL_NET="vr1-dc1-metal-admin"
LISTEN_CIDR="10.12.68.4/22"
EDGE_GW="10.12.64.1"
CLIENT_CIDR="10.12.68.0/22"
;;
*) echo "FAIL: unknown site '$SITE' -- add a MEASURED row block first" >&2; exit 2 ;;
esac
LISTEN="${LISTEN_CIDR%%/*}"
# 3129: MEASURED FREE on both racks while 3128 and 8000 are MAAS's squid, 3142 is
# dc1's apt-cacher-ng and 80 is dc0's mirror nginx.
PROXY_PORT="3129"
# TEST SEAM. EMPTY in every production invocation, so the paths below are
# byte-identical to the absolute ones. The harness points it at a mktemp tree so
# that check's five file assertions and node's /etc/environment assertion can be
# driven in BOTH directions -- an assertion nobody can turn red is decoration.
# In-repo precedent for the idiom: MAAS_SNAP_ROOT (maas-region-power-key.sh),
# CREDS_ROOT (creds-audit.sh), CLOUD_SNAPSHOT_ROOT (cloud-snapshot.sh).
SYSROOT="${DC_SNAP_PROXY_SYSROOT:-}"
SQUID_CONF="${SYSROOT}/etc/squid/${SITE}-snap-proxy.conf"
PROXY_UNIT="${SYSROOT}/etc/systemd/system/${SITE}-snap-proxy.service"
NET_HELPER="${SYSROOT}/usr/local/sbin/${SITE}-snap-proxy-net-apply"
NET_UNIT="${SYSROOT}/etc/systemd/system/${SITE}-snap-proxy-net.service"
RESOLVED_DROPIN="${SYSROOT}/etc/systemd/resolved.conf.d/${SITE}-snap-proxy.conf"
ENV_FILE="${SYSROOT}/etc/environment"
RUNTIME_DIR_NAME="${SITE}-snap-proxy"
PID_FILE="/run/${RUNTIME_DIR_NAME}/squid.pid"
LOG_DIR="/var/log/squid"
# The Canonical-documented snap endpoint set, all :443
# (snapcraft.io/docs/reference/administration/network-requirements/ and its
# source of record forum.snapcraft.io/t/network-requirements/5147, CDN list dated
# 2021-10-25). A leading dot is squid's "this domain and all subdomains", so
# `.snapcraftcontent.com` covers storage./canonical-lgw01.cdn./cloudfront.cdn.
# `.cdn.snapcraft.io` is marked DEPRECATED upstream and kept only so an older
# client is not denied. THE DOWNLOAD IS PROXIED TOO -- snapd uses ONE http client
# for the API and the CDN fetch (snapd store/store.go), so an allowlist naming
# only api.snapcraft.io would authenticate and then fail the download.
STORE_DOMAINS="api.snapcraft.io dashboard.snapcraft.io login.ubuntu.com .snapcraftcontent.com .cdn.snapcraft.io"
# Behavioral probe targets, shared by the rack-side `check` and the client-side
# `node` so the two cannot drift apart. Shapes MEASURED 2026-07-31 (capture s7).
STORE_PROBE_URL="https://api.snapcraft.io/v2/snaps/info/core"
CDN_PROBE_URL="https://canonical-lgw01.cdn.snapcraftcontent.com/"
# A destination deliberately NOT in STORE_DOMAINS. RFC 2606 / IANA reserved, so it
# can never become a real dependency. Its only job is to prove the allowlist is in
# force -- a proxy that answers this one is an OPEN proxy.
DENY_PROBE_URL="https://example.com/"
# ---------------------------------------------------------------------------
# Generated content -- the single source of truth for check AND install. Every
# generator is diffed by `check`, so a hand-edit on the rack is a FAILURE, not a
# silent drift (assert the ARTIFACT, never the intent).
# ---------------------------------------------------------------------------
gen_squid_conf() {
cat <<EOF
# ${SITE} snap forward proxy -- generated by scripts/dc-snap-proxy.sh
# (D-135/D-107 RULING 2, 2026-07-31); do not hand-edit.
#
# DEDICATED INSTANCE. The packaged squid.service cannot serve: Ubuntu noble's
# squid.conf pins http_port 3128 and MAAS's own squid holds *:3128 wildcard-bound
# on this rack, so the packaged unit FATALs on bind. /etc/default/squid is empty,
# so there is no supported way to repoint it. install disables it.
#
# CONNECT-ONLY, DESTINATION-RESTRICTED. Every documented snap endpoint is :443,
# so this proxy never sees a plain GET from snapd. Safe_ports is 443 alone on
# purpose: this is a snap path, not a general egress.
#
# Consume via:
# juju model-config -m <model> snap-http-proxy=http://${LISTEN}:${PROXY_PORT}
# juju model-config -m <model> snap-https-proxy=http://${LISTEN}:${PROXY_PORT}
# Leave snap-store-proxy / -url / -assertions EMPTY (different product).
visible_hostname ${SITE}-snap-proxy
pid_filename ${PID_FILE}
http_port ${LISTEN}:${PROXY_PORT}
# Clients: the metal-admin plane. MEASURED as the source the charm CONTAINERS
# select toward this address, not the address juju status displays.
acl snap_clients src ${CLIENT_CIDR}
# The rack's own loopback, so this script's behavioral smoke test can run. It
# proves upstream reach, CONNECT capability and that the destination rule is in
# force; CLIENT AUTHORIZATION is what \`node\` proves, from a real client.
acl snap_probe src 127.0.0.1/32
acl SSL_ports port 443
acl Safe_ports port 443
acl CONNECT method CONNECT
acl snap_store dstdomain ${STORE_DOMAINS}
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny !snap_store
http_access allow snap_clients
http_access allow snap_probe
http_access deny all
# No caching is POSSIBLE here (opaque CONNECT tunnels) -- stated explicitly so
# nobody later "fixes" the absence of a cache_dir by adding SslBump.
cache deny all
access_log ${LOG_DIR}/${SITE}-snap-proxy.access.log
cache_log ${LOG_DIR}/${SITE}-snap-proxy.cache.log
coredump_dir /var/spool/squid
EOF
}
gen_proxy_unit() {
cat <<EOF
[Unit]
Description=${SITE} snap forward proxy (dc-snap-proxy.sh; D-135/D-107 ruling 2, 2026-07-31)
Documentation=man:squid(8)
After=network-online.target nss-lookup.target ${SITE}-snap-proxy-net.service
Requires=${SITE}-snap-proxy-net.service
Wants=network-online.target
[Service]
Type=notify
NotifyAccess=all
PIDFile=${PID_FILE}
Group=proxy
RuntimeDirectory=${RUNTIME_DIR_NAME}
RuntimeDirectoryMode=0775
ExecStart=/usr/sbin/squid --foreground -f ${SQUID_CONF}
ExecReload=/bin/kill -HUP \$MAINPID
KillMode=mixed
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
}
# --- the utility net layer THIS script owns ---------------------------------
# Same MEASURED mechanism dc-mirror.sh proved on both racks 2026-07-23 and
# dc-cache-proxy.sh took ownership of 2026-07-27: resolve the bridge from the
# libvirt network NAME at runtime, then idempotently place the .4 alias and the
# edge default route. Deliberately identical values, so the units coexist.
gen_net_helper() {
cat <<EOF
#!/usr/bin/env bash
# ${SITE}-snap-proxy-net-apply -- generated by scripts/dc-snap-proxy.sh; do not hand-edit.
# Places the proxy LISTEN alias (D-134 utility .4) and the default route via the
# DC edge (the proxy's upstream egress). Bridge resolved from the libvirt network
# NAME at runtime (virbrN is auto-assigned, not stable identity -- hard rule 3).
set -euo pipefail
br_of() {
virsh -c qemu:///system net-dumpxml "\$1" | sed -n "s/.*bridge name='\\([^']*\\)'.*/\\1/p"
}
ip addr replace ${LISTEN_CIDR} dev "\$(br_of ${UTIL_NET})"
ip route replace default via ${EDGE_GW}
EOF
}
gen_net_unit() {
cat <<EOF
[Unit]
Description=${SITE} snap-proxy net (dc-snap-proxy.sh; D-135/D-107 -- LISTEN alias + edge default route)
After=libvirtd.service ${SITE}-rack-legs.service network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${NET_HELPER}
[Install]
WantedBy=multi-user.target
EOF
}
gen_resolved_dropin() {
cat <<EOF
# ${SITE} snap-proxy DNS (generated by scripts/dc-snap-proxy.sh; D-135/D-107).
# System DNS = the DC edge resolver: the PROXY must resolve api.snapcraft.io and
# the snap CDN to open a tunnel, and the rack has no other working resolver.
# The MAAS rack agent ignores resolv.conf (D-131), so node DNS is unaffected.
[Resolve]
DNS=${EDGE_GW}
EOF
}
# Bridge for the utility net, resolved from the libvirt net NAME at runtime
# (virbrN is auto-assigned, not stable identity -- hard rule 3).
util_bridge() {
virsh -c qemu:///system net-dumpxml "$UTIL_NET" 2>/dev/null \
| sed -n "s/.*bridge name='\([^']*\)'.*/\1/p"
}
# ---------------------------------------------------------------------------
# check -- THREE outcomes, never two: PASS (0), FAIL (1), REFUSE (3).
# GA-R6: this is what a stage would close on, so it must be able to fail, it
# asserts CONTENT and not existence, and a state it cannot evaluate REFUSES
# rather than defaulting to success. "Could not look" is never "nothing there".
# It mutates no config or service state (the probes may add lines to squid's
# access log -- benign, and the proof it actually serves).
# ---------------------------------------------------------------------------
FAILED=0; REFUSED=0
say(){ echo " OK $1"; }
miss(){ echo " MISS $1"; FAILED=1; }
refuse(){ echo " REFUSE $1"; REFUSED=1; }
check_file() { # path gen_fn
if [ ! -f "$1" ]; then miss "$1 absent"; return; fi
if diff -q <("$2") "$1" >/dev/null 2>&1; then say "$1 matches generated"
else miss "$1 DIFFERS from generated content"; fi
}
do_check() {
echo "dc-snap-proxy check ($SITE) -- ${LISTEN}:${PROXY_PORT}, clients ${CLIENT_CIDR}"
# -- package. An absence here is an OBSERVED absence, so it FAILS; it does not
# refuse. Fail-loud with the exact apt line, like dc-mirror/dc-cache-proxy.
if dpkg -s squid >/dev/null 2>&1; then say "squid installed"
else miss "squid not installed -- run: apt-get update && apt-get install -y squid"; fi
# -- the five artifacts this script owns, asserted by CONTENT.
check_file "$SQUID_CONF" gen_squid_conf
check_file "$PROXY_UNIT" gen_proxy_unit
check_file "$NET_HELPER" gen_net_helper
check_file "$NET_UNIT" gen_net_unit
check_file "$RESOLVED_DROPIN" gen_resolved_dropin
# -- units. ENABLED matters as much as active: without it the LISTEN alias and
# the proxy both vanish at the next rack reboot, and this rack has rebooted.
if ! command -v systemctl >/dev/null 2>&1; then
refuse "systemctl is not present -- cannot evaluate unit state"
else
for u in "${SITE}-snap-proxy.service" "${SITE}-snap-proxy-net.service"; do
if [ "$(systemctl is-enabled "$u" 2>/dev/null)" = "enabled" ]; then
say "$u enabled"
else miss "$u not enabled (it would not survive a reboot)"; fi
done
if systemctl is-active --quiet "${SITE}-snap-proxy.service" 2>/dev/null; then
say "${SITE}-snap-proxy.service active"
else miss "${SITE}-snap-proxy.service not active"; fi
# The PACKAGED unit must be neither active nor enabled: its squid.conf pins
# http_port 3128, which MAAS's squid holds, so it can only crash-loop -- and
# a crash-looping squid alongside ours poisons every later triage.
if systemctl is-active --quiet squid 2>/dev/null; then
miss "the PACKAGED squid.service is ACTIVE -- it pins http_port 3128 (MAAS's squid holds it); run: systemctl disable --now squid"
elif [ "$(systemctl is-enabled squid 2>/dev/null)" = "enabled" ]; then
miss "the PACKAGED squid.service is ENABLED -- it will fight for 3128 at the next boot; run: systemctl disable --now squid"
else say "packaged squid.service neither active nor enabled"; fi
fi
# -- the listener must be on the LISTEN address SPECIFICALLY, not wildcard: a
# wildcard bind would collide with MAAS's squid and would also expose the proxy
# on the transit and provider legs.
if ! command -v ss >/dev/null 2>&1; then
refuse "ss is not present -- cannot evaluate the listener"
elif ss -ltn 2>/dev/null | grep -q "${LISTEN}:${PROXY_PORT} "; then
say "listening on ${LISTEN}:${PROXY_PORT}"
elif ss -ltn 2>/dev/null | grep -qE "(\*|0\.0\.0\.0):${PROXY_PORT} "; then
miss "port ${PROXY_PORT} is bound WILDCARD, not on ${LISTEN} -- refusing to accept a broader bind than generated"
else
miss "nothing listening on ${LISTEN}:${PROXY_PORT}"
fi
# -- the net layer's live effect (the artifacts were diffed above).
if ! command -v virsh >/dev/null 2>&1; then
refuse "virsh is not present -- cannot resolve ${UTIL_NET}'s bridge, so the LISTEN alias is UNKNOWN"
else
br="$(util_bridge)"
if [ -z "$br" ]; then
refuse "libvirt network '${UTIL_NET}' did not resolve to a bridge -- the alias is UNKNOWN, not absent"
elif ip -4 -o addr show dev "$br" 2>/dev/null | grep -q " ${LISTEN}/"; then
say "${LISTEN_CIDR} on $br ($UTIL_NET)"
else
miss "${LISTEN_CIDR} absent on the metal-admin bridge (run: dc-snap-proxy.sh install $SITE)"
fi
fi
if ip route show default 2>/dev/null | grep -q "via ${EDGE_GW}"; then
say "default route via edge ${EDGE_GW}"
else miss "default route via ${EDGE_GW} absent -- the proxy has no upstream egress"; fi
# -- BEHAVIORAL. Three probes from the SAME source, so the difference between
# them isolates exactly one rule each. A 200 alone is never evidence.
if ! command -v curl >/dev/null 2>&1; then
refuse "curl is not present -- cannot evaluate whether the proxy actually serves"
else
# (1) ALLOW direction, asserted on CONTENT. MEASURED body shape 2026-07-31:
# the store API answers with a "channel-map" array carrying a sha3-384
# download digest. No squid error page contains either.
body="$(mktemp)"; cerr="$(mktemp)"
code="$(curl -sS -o "$body" -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
-m 30 -H 'Snap-Device-Series: 16' "$STORE_PROBE_URL" 2>"$cerr")"; crc=$?
if [ "$crc" -ne 0 ]; then
refuse "could not reach $STORE_PROBE_URL THROUGH http://${LISTEN}:${PROXY_PORT} (curl exit $crc: $(head -1 "$cerr")) -- unreachable is not 'nothing there'"
elif [ "$code" != "200" ]; then
miss "store API returned HTTP $code through the proxy (want 200)"
elif ! grep -q '"channel-map"' "$body"; then
miss "body served for the store API has no \"channel-map\" -- what came back is not the snap store's answer, whatever its status line said"
elif ! grep -q 'sha3-384' "$body"; then
miss "store API body carries no sha3-384 download digest -- not a usable snap info response"
else
say "CONNECT to the store API: curl exit 0, HTTP 200, channel-map + sha3-384 present"
fi
rm -f "$body" "$cerr"
# (2) The CDN tunnel must be ALLOWED. The CDN answers 403 to a bare GET of /,
# so the assertion is NOT on the status code -- it is that squid did not
# refuse the tunnel. MEASURED distinction: a proxy denial is curl exit 56
# with "CONNECT tunnel failed, response 403"; a server 403 is curl exit 0.
# Without this the allowlist could name only api.snapcraft.io and every
# snap would authenticate and then fail its DOWNLOAD.
cout="$(curl -sS -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
-m 30 "$CDN_PROBE_URL" 2>&1)"; crc=$?
if grep -q 'CONNECT tunnel failed' <<<"$cout"; then
miss "the proxy REFUSED the CDN tunnel to $CDN_PROBE_URL -- the allowlist is too narrow and snap DOWNLOADS would fail after a successful API call"
elif [ "$crc" -eq 0 ]; then
say "CDN tunnel allowed (curl exit 0; the CDN's own status was '$cout')"
else
refuse "CDN probe gave curl exit $crc with '$cout' -- neither an allowed tunnel nor a proxy denial; refusing to interpret it"
fi
# (3) DENY direction. This is the assertion that proves the destination
# allowlist is actually in force rather than merely written down. A proxy
# that answers this one is an OPEN proxy on the node plane.
dout="$(curl -sS -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
-m 20 "$DENY_PROBE_URL" 2>&1)"; drc=$?
if grep -q 'CONNECT tunnel failed, response 403' <<<"$dout"; then
say "non-allowlisted destination is DENIED (403 at CONNECT) -- the allowlist is in force"
elif [ "$drc" -eq 0 ]; then
miss "the proxy SERVED a non-allowlisted destination ($DENY_PROBE_URL, result '$dout') -- this is an OPEN forward proxy on the node plane, not a snap path"
else
refuse "deny probe gave curl exit $drc with '$dout' -- neither a 403 denial nor a success; the allowlist state is UNKNOWN"
fi
fi
if [ "$FAILED" -ne 0 ]; then echo "dc-snap-proxy check ($SITE): FAIL"; exit 1; fi
if [ "$REFUSED" -ne 0 ]; then
echo "dc-snap-proxy check ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
fi
echo "dc-snap-proxy check ($SITE): PASS"; exit 0
}
# ---------------------------------------------------------------------------
# install -- idempotent; requires root. Net layer FIRST (nothing can bind the
# LISTEN address until it exists), then fail loud on the missing package, then
# the config, then a config PARSE before anything is enabled.
# ---------------------------------------------------------------------------
do_install() {
[ "$(id -u)" = "0" ] || { echo "FAIL: install requires root" >&2; exit 4; }
umask 022
# Prereq: the libvirt utility net must EXIST (we resolve its bridge). That is a
# rack-standup artifact (dc-rack-net.sh), never a mirror or cache-proxy one, so
# this service depends on no other artifact strategy's apparatus.
br="$(util_bridge)"
if [ -z "$br" ]; then
echo "FAIL: libvirt network '${UTIL_NET}' not found -- cannot resolve its bridge" >&2
echo " the rack net layer comes first: dc-rack-net.sh install $SITE" >&2
exit 4
fi
install -m 0755 /dev/null "$NET_HELPER" || exit 4
gen_net_helper > "$NET_HELPER" || exit 4
chmod 0755 "$NET_HELPER" || exit 4
gen_net_unit > "$NET_UNIT" || exit 4
mkdir -p "$(dirname "$RESOLVED_DROPIN")" || exit 4
gen_resolved_dropin > "$RESOLVED_DROPIN" || exit 4
systemctl daemon-reload || exit 4
systemctl enable "${SITE}-snap-proxy-net.service" >/dev/null 2>&1 || exit 4
systemctl restart "${SITE}-snap-proxy-net.service" \
|| { echo "FAIL: ${SITE}-snap-proxy-net.service did not apply" >&2; exit 4; }
systemctl restart systemd-resolved >/dev/null 2>&1 || true
# Prereq 2: the package. NEVER apt-installed from here -- the repo pattern is
# fail-loud with the exact line, so package installs stay operator-visible.
if ! dpkg -s squid >/dev/null 2>&1; then
echo "FAIL: squid not installed" >&2
echo " net layer is up -- run: apt-get update && apt-get install -y squid" >&2
echo " then re-run: dc-snap-proxy.sh install $SITE" >&2
exit 4
fi
# The packaged unit is stood DOWN before ours goes up: its squid.conf pins
# http_port 3128, which MAAS's squid holds wildcard-bound, so it can only fail.
systemctl disable --now squid >/dev/null 2>&1 || true
mkdir -p "$(dirname "$SQUID_CONF")" "$LOG_DIR" || exit 4
gen_squid_conf > "$SQUID_CONF" || exit 4
# PARSE BEFORE ENABLE. A config that squid rejects must fail here, loudly, and
# not as a mysterious unit that will not start.
if ! squid -k parse -f "$SQUID_CONF" >/dev/null 2>&1; then
echo "FAIL: squid rejected ${SQUID_CONF}:" >&2
squid -k parse -f "$SQUID_CONF" >&2
exit 4
fi
gen_proxy_unit > "$PROXY_UNIT" || exit 4
systemctl daemon-reload || exit 4
systemctl enable "${SITE}-snap-proxy.service" >/dev/null 2>&1 || exit 4
systemctl restart "${SITE}-snap-proxy.service" \
|| { echo "FAIL: ${SITE}-snap-proxy.service did not start" >&2; exit 4; }
echo "dc-snap-proxy install ($SITE): done -- running check:"
do_check
}
# ---------------------------------------------------------------------------
# node -- run FROM a DC node OR a charm LXD container. Read-only, no root, and
# deliberately touches NOTHING rack-side: only curl, grep and /etc/environment.
# Three outcomes, never two: PASS (0), FAIL (1), REFUSE (3).
# ---------------------------------------------------------------------------
NFAILED=0; NREFUSED=0
nok(){ echo " OK $1"; }
nno(){ echo " FAIL $1"; NFAILED=1; }
nrefuse(){ echo " REFUSE $1"; NREFUSED=1; }
do_node() {
echo "dc-snap-proxy node check ($SITE) -- from this client, against ${LISTEN}:${PROXY_PORT}"
if ! command -v curl >/dev/null 2>&1; then
nrefuse "curl is not present on this client -- cannot evaluate proxy reachability"
else
# (1) The client can actually CONNECT through the proxy to the store, and what
# comes back is the store's answer -- not a 200 from something else.
body="$(mktemp)"; cerr="$(mktemp)"
code="$(curl -sS -o "$body" -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
-m 30 -H 'Snap-Device-Series: 16' "$STORE_PROBE_URL" 2>"$cerr")"; crc=$?
if [ "$crc" -ne 0 ]; then
if grep -q 'CONNECT tunnel failed, response 403' "$cerr"; then
nno "the proxy DENIED this client (403 at CONNECT) -- this client's source address is outside ${CLIENT_CIDR}; measure it with 'ip route get ${LISTEN}' and fix the site row, do not widen the ACL blind"
else
nrefuse "could not reach $STORE_PROBE_URL THROUGH http://${LISTEN}:${PROXY_PORT} (curl exit $crc: $(head -1 "$cerr")) -- unreachable is not 'nothing there'"
fi
elif [ "$code" != "200" ]; then
nno "store API returned HTTP $code through the proxy (want 200)"
elif ! grep -q '"channel-map"' "$body"; then
nno "body served for the store API has no \"channel-map\" -- not the snap store's answer, whatever its status line said"
elif ! grep -q 'sha3-384' "$body"; then
nno "store API body carries no sha3-384 download digest -- not a usable snap info response"
else
nok "CONNECT to the store API through the proxy: curl exit 0, HTTP 200, channel-map + sha3-384 present"
fi
rm -f "$body" "$cerr"
# (2) The allowlist must be in force FROM HERE too. Asserting it only on the
# rack would miss a client-specific allow rule, and an open proxy on the node
# plane is a bigger problem than a broken snap install.
dout="$(curl -sS -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" \
-m 20 "$DENY_PROBE_URL" 2>&1)"; drc=$?
if grep -q 'CONNECT tunnel failed, response 403' <<<"$dout"; then
nok "non-allowlisted destination is DENIED from this client -- the allowlist is in force"
elif [ "$drc" -eq 0 ]; then
nno "the proxy SERVED a non-allowlisted destination from this client ($DENY_PROBE_URL, result '$dout') -- OPEN forward proxy on the node plane"
else
nrefuse "deny probe gave curl exit $drc with '$dout' -- neither a 403 denial nor a success; the allowlist state is UNKNOWN from here"
fi
fi
# (3) snapd's own view of the proxy, WITHOUT root. juju's proxyupdater shells
# out `snap set system proxy.http/https`, and snapd's configcore writes those
# into /etc/environment (envFilePath = "/etc/environment"), which is
# world-readable -- whereas `snap get system proxy` needs root and would break
# this subcommand's no-root property.
if [ ! -r "$ENV_FILE" ]; then
nrefuse "$ENV_FILE is not readable -- cannot evaluate whether the juju snap proxy keys reached this client"
else
ENVLINE="$(grep -E '^[[:space:]]*https_proxy=' "$ENV_FILE" 2>/dev/null | tail -1)"
if [ -z "$ENVLINE" ]; then
nno "no https_proxy in $ENV_FILE -- the juju keys have not reached this client; run: juju model-config -m <model> snap-http-proxy=http://${LISTEN}:${PROXY_PORT} snap-https-proxy=http://${LISTEN}:${PROXY_PORT}"
elif grep -q "http://${LISTEN}:${PROXY_PORT}" <<<"$ENVLINE"; then
nok "snapd's proxy is this DC's proxy ($ENV_FILE: $ENVLINE)"
else
nno "https_proxy in $ENV_FILE points somewhere ELSE: $ENVLINE -- expected http://${LISTEN}:${PROXY_PORT}"
fi
fi
if [ "$NFAILED" -ne 0 ]; then echo "dc-snap-proxy node ($SITE): FAIL"; exit 1; fi
if [ "$NREFUSED" -ne 0 ]; then
echo "dc-snap-proxy node ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
fi
echo "dc-snap-proxy node ($SITE): PASS"; exit 0
}
case "$MODE" in
check) do_check ;;
install) do_install ;;
node) do_node ;;
esac
