#!/usr/bin/env bash
# scripts/dc-cache-proxy.sh <check|install|node> <site> -- per-DC apt CACHING PROXY
# (apt-cacher-ng). D-135 amendment 2026-07-24 (operator-ruled): the INTERIM DC1
# apt path so the DC1 build starts WITHOUT waiting on the full ~950 GiB
# debmirror. DC0 stays the full-mirror path (D-135 item 1); DC1's proxy is
# the DC1 STRATEGY, not an interim step (D-135 AMENDMENT 2026-07-27: "DC0 is the
# test of a full mirror. DC1 is the test of the mirror proxy."). The earlier
# "DC1's proxy is REPLACED by a full mirror afterward" plan is SUPERSEDED -- dc1's
# partial mirror and its whole apparatus were REMOVED 2026-07-27. Roosevelt:
# cache-proxy vs full-mirror is a per-DC utility-node artifact-delivery choice
# (D-132/D-133), and each DC proves one of them.
#
# RUNS ON THE DC RACK HOST, invoked over ssh like dc-mirror.sh (no clone there):
# check: ssh -i <dc-key> -J voffice1 <user>@<rack-transit-ip> \
# 'sudo bash -s' -- check dc1 < scripts/dc-cache-proxy.sh
# install: same with 'install dc1' (idempotent; safe to re-run).
#
# ...EXCEPT `node`, WHICH RUNS ON A DC NODE (gate G17). Added 2026-07-29.
# G17's dc1 half named `dc-cache-proxy.sh:210-217` as its node-side check, but that
# code had NO subcommand entry point and lives inside `check`, whose other assertions
# are RACK-side facts a node cannot satisfy (the acng drop-in, the systemd net unit,
# the .4 alias on the libvirt bridge, the rack default route). Pointing a node at
# `check` therefore fails for reasons that say nothing about node reachability.
# `node` asserts ONLY what a node can, and touches ONLY curl + chronyc + the network
# -- no root, no systemctl, no virsh, no generated-file diffs:
# node: bash -s -- node dc1 < scripts/dc-cache-proxy.sh (ON the node)
# G17's three assertions, in order:
# (1) ARTIFACT REACHABILITY ON CONTENT, with an exit-code predicate -- a proxied
# fetch of archive AND UCA `Release` through <LISTEN>:3142, asserting curl
# exit 0 AND HTTP 200 AND that the BODY is that dist's Release file. A 200 is
# not evidence: apt-cacher-ng serves its own HTML report pages with one.
# (2) NODE TIME SOURCE -- `chronyc sources` names the MAAS-served rack and NOT the
# DC edge (D-129(iv): the edge has NO NTP role; the MAAS hierarchy is the time
# authority).
# (3) An UNRECOGNISED or UNREACHABLE result REFUSES (exit 3) rather than defaulting
# to success -- "could not look" is never "nothing there".
#
# PREREQS on the rack:
# - apt-cacher-ng must be apt-installed first (fail-loud with the exact line).
#
# THIS SCRIPT NOW OWNS ITS OWN UTILITY NET LAYER (changed 2026-07-27).
# It previously REUSED dc-mirror.sh's <site>-mirror-net.service for the .4
# alias + the edge default route, and refused to install without it. That
# coupling was a measured trap: on dc1 -- whose RULED artifact path is this
# proxy, NOT a mirror -- three files named `dc1-mirror-*` (the net unit, its
# apply helper, and the resolved drop-in) were load-bearing for the PROXY. So
# the recorded mirror-removal procedure (07-23 changelog: "stop/disable
# <site>-mirror-sync.timer + -net.service, rm the generated files") would have
# silently killed the proxy, and anyone tidying up "leftover mirror stuff" on a
# proxy-only DC would have done the same. Worse, the proxy could not be REBUILT
# after such a teardown: nothing but `dc-mirror.sh install` created the net
# layer, so rebuilding a proxy meant reinstalling the whole mirror apparatus
# (sync unit, enabled daily timer, nginx vhost, debmirror package).
#
# Now: <site>-cache-proxy-net.service + <site>-cache-proxy-net-apply +
# /etc/systemd/resolved.conf.d/<site>-cache-proxy.conf. Neutrally named, owned
# here, so a proxy-only DC has NO mirror-named dependency and is independently
# rebuildable. The two net units are SAFE TO COEXIST on a site running both
# (`ip addr replace` / `ip route replace` are idempotent and both set the same
# MEASURED values), so this should be additive for dc0, which keeps its mirror.
# COEXISTENCE IS REASONED, NOT MEASURED (hard rule 2): the proxy has only ever been
# installed on dc1, where the mirror net unit was then removed. Nobody has run both
# units on one host. The reasoning is that `ip addr replace` / `ip route replace` are
# idempotent and both units write the SAME measured values, so the second is a no-op --
# but if dc0 is ever given the proxy, VERIFY that rather than trusting this sentence.
# Roosevelt: a DC's artifact-delivery strategy is a per-DC choice that must be
# changeable without a rebuild of the other strategy's apparatus.
#
# CONSUMPTION (Stage 5, PROXY mode -- all http apt cached, incl. UCA, no remap):
# juju model-config apt-http-proxy=http://<LISTEN>:3142
# (or per-node /etc/apt/apt.conf.d: Acquire::http::Proxy "http://<LISTEN>:3142";)
#
# EXIT: 0 ok | 1 check failed | 2 bad args/unknown site | 3 node check REFUSED
# (could not evaluate -- never a pass) | 4 install failed.
# ASCII + LF only.
set -uo pipefail
MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in check|install|node) ;; *)
echo "usage: dc-cache-proxy.sh <check|install|node> <site>" >&2; exit 2 ;; esac
# ---------------------------------------------------------------------------
# Site table. Same MEASURED/RULED utility .4 + edge gateway per DC as
# dc-mirror.sh (the proxy listens at the SAME utility address the deploy
# already expects). ADD A SITE ONLY WITH MEASURED/RULED VALUES (hard rule 2);
# the harness rejects rows without a MEASURED tag.
# ---------------------------------------------------------------------------
case "$SITE" in
dc0)
# MEASURED/RULED 2026-07-23 (D-134 amendment + D-135). NOTE: dc0's RULED
# path is the FULL MIRROR (D-135 item 1); this row exists for tooling
# parity only -- dc0 is not deployed via the proxy.
# CONSEQUENCE, so a failing run is not mistaken for a regression (the same
# warning dc-mirror.sh's dc1 row now carries): `node dc0` will REFUSE on the
# artifact half BY DESIGN -- there is no proxy listening at 10.12.8.4:3142 on
# dc0. dc0's G17 artifact assertion is a package-path fetch from the MIRROR,
# which is dc-mirror.sh's territory, not this script's. Do NOT "fix" it by
# installing the proxy on dc0: that is a deliberate strategy change (D-135
# AMENDMENT 2026-07-27), never a repair.
UTIL_NET="vr1-dc0-metal-admin"
LISTEN_CIDR="10.12.8.4/22"
EDGE_GW="10.12.4.1"
# RACK_IP -- the rack's metal-admin leg, i.e. the MAAS-served time source a node
# must sync from under D-129(iv). MEASURED: scripts/dc-rack-net.sh:62 LEGS,
# docs/audit/dc-rack-net-install-20260721.txt (the metal-admin leg row), and
# the rendered dhcpd option itself -- "option ntp-servers 10.12.8.2" in
# docs/audit/d131-sub3-dhcpd-option6-20260721.txt.
RACK_IP="10.12.8.2"
;;
dc1)
# MEASURED/RULED 2026-07-24 (D-135 amendment -- interim DC1 apt path):
# LISTEN 10.12.68.4 -- the D-134 utility .4, reused from the mirror-net
# layer; EDGE_GW 10.12.64.1 -- dc1 edge LAN gw (upstream egress).
UTIL_NET="vr1-dc1-metal-admin"
LISTEN_CIDR="10.12.68.4/22"
EDGE_GW="10.12.64.1"
# RACK_IP -- MEASURED as an ADDRESS (scripts/dc-rack-net.sh:78 LEGS;
# docs/audit/dc1-rack-net-install-20260723.txt, the metal-admin leg row;
# D-124 amendment / docs/design-decisions.md:4587,4597). HONEST LIMIT (hard rule 2):
# unlike dc0, dc1's RENDERED dhcpd `option ntp-servers` has NOT been captured. The
# expectation here is the D-129(iv) INVARIANT (MAAS hierarchy is the time authority),
# not a measurement of dc1's live DHCP render -- so if dc1 serves something else at
# first boot this check goes red, which is the intended behaviour, not a false red.
RACK_IP="10.12.68.2"
;;
*) echo "FAIL: unknown site '$SITE' -- add a MEASURED row block first" >&2; exit 2 ;;
esac
LISTEN="${LISTEN_CIDR%%/*}"
PROXY_PORT="3142"
# The two artifact paths the deploy actually needs (Ubuntu archive + UCA), shared by
# the rack-side smoke test and the node-side G17 check so the two cannot drift apart.
# An ARRAY, never a space-joined string -- word-splitting a URL list is a measured
# strict-bash trap in this repo.
ARTIFACT_URLS=(
"http://archive.ubuntu.com/ubuntu/dists/jammy/Release"
"http://ubuntu-cloud.archive.canonical.com/ubuntu/dists/jammy-updates/caracal/Release"
)
ACNG_DROPIN="/etc/apt-cacher-ng/acng.conf.d/${SITE}-cache-proxy.conf"
# Net layer owned HERE (see the header note). Named for the PROXY, never "mirror".
NET_HELPER="/usr/local/sbin/${SITE}-cache-proxy-net-apply"
NET_UNIT="/etc/systemd/system/${SITE}-cache-proxy-net.service"
RESOLVED_DROPIN="/etc/systemd/resolved.conf.d/${SITE}-cache-proxy.conf"
# ---------------------------------------------------------------------------
# Generated config (single source of truth for check AND install). Minimal:
# apt-cacher-ng in proxy mode caches all http apt (archive/security/UCA) with
# no per-repo remap; we only pin the port so a package-default change cannot
# move it. Default CacheDir (/var/cache/apt-cacher-ng) is kept (owner-correct).
# ---------------------------------------------------------------------------
gen_acng_dropin() {
cat <<EOF
# ${SITE} apt caching proxy -- generated by scripts/dc-cache-proxy.sh
# (D-135 amendment 2026-07-24); do not hand-edit. apt-cacher-ng proxy mode:
# caches all http apt requested by the DC nodes (archive/security/UCA), no
# per-repo remap. Consume via: juju model-config apt-http-proxy=http://${LISTEN}:${PROXY_PORT}
Port: ${PROXY_PORT}
EOF
}
# --- the utility net layer this script OWNS ---------------------------------
# Same MEASURED mechanism dc-mirror.sh proved on both racks 2026-07-23: resolve
# the bridge from the libvirt network NAME at runtime, then idempotently place
# the .4 alias and the edge default route. Deliberately identical values, so the
# two units coexist without fighting where a site runs both.
gen_net_helper() {
cat <<EOF
#!/usr/bin/env bash
# ${SITE}-cache-proxy-net-apply -- generated by scripts/dc-cache-proxy.sh; do not hand-edit.
# Places the proxy LISTEN alias (D-134 utility .4) and the default route via the
# DC edge (the proxy's upstream egress). Bridge resolved from the libvirt network
# NAME at runtime (virbrN is auto-assigned, not stable identity -- hard rule 3).
set -euo pipefail
br_of() {
virsh -c qemu:///system net-dumpxml "\$1" | sed -n "s/.*bridge name='\\([^']*\\)'.*/\\1/p"
}
ip addr replace ${LISTEN_CIDR} dev "\$(br_of ${UTIL_NET})"
ip route replace default via ${EDGE_GW}
EOF
}
gen_net_unit() {
cat <<EOF
[Unit]
Description=${SITE} cache-proxy net (dc-cache-proxy.sh; D-135 amendment -- LISTEN alias + edge default route)
After=libvirtd.service ${SITE}-rack-legs.service network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${NET_HELPER}
[Install]
WantedBy=multi-user.target
EOF
}
gen_resolved_dropin() {
cat <<EOF
# ${SITE} cache-proxy DNS (generated by scripts/dc-cache-proxy.sh; D-135 amendment).
# System DNS = the DC edge resolver: the PROXY must resolve archive.ubuntu.com and
# the UCA host to serve a cache miss, and the rack has no other working resolver.
# The MAAS rack agent ignores resolv.conf (D-131), so node DNS is unaffected.
[Resolve]
DNS=${EDGE_GW}
EOF
}
# Bridge for the utility net, resolved from the libvirt net NAME at runtime
# (virbrN is auto-assigned, not stable identity -- hard rule 3).
util_bridge() {
virsh -c qemu:///system net-dumpxml "$UTIL_NET" 2>/dev/null \
| sed -n "s/.*bridge name='\([^']*\)'.*/\1/p"
}
# ---------------------------------------------------------------------------
# check -- mutate no CONFIG/service state; report and exit 0 ok / 1 incomplete.
# (The behavioral smoke test may populate the proxy cache with two Release
# files -- benign and self-managing; it is the proof the proxy actually serves.)
# ---------------------------------------------------------------------------
FAILED=0
say(){ echo " $1"; }
miss(){ echo " MISS $1"; FAILED=1; }
do_check() {
command -v apt-cacher-ng >/dev/null 2>&1 || dpkg -s apt-cacher-ng >/dev/null 2>&1 \
&& say "OK apt-cacher-ng installed" || miss "apt-cacher-ng not installed"
if [ ! -f "$ACNG_DROPIN" ]; then miss "$ACNG_DROPIN absent"; else
if diff -q <(gen_acng_dropin) "$ACNG_DROPIN" >/dev/null 2>&1; then
say "OK $ACNG_DROPIN matches"; else miss "$ACNG_DROPIN DIFFERS from generated"; fi
fi
systemctl is-active --quiet apt-cacher-ng \
&& say "OK apt-cacher-ng active" || miss "apt-cacher-ng not active"
ss -ltn 2>/dev/null | grep -q ":${PROXY_PORT} " \
&& say "OK listening on :${PROXY_PORT}" || miss "not listening on :${PROXY_PORT}"
# Utility net layer -- OWNED HERE since 2026-07-27, so assert the ARTIFACTS too,
# not just the live effect. Asserting only the live address/route would pass
# while the layer was actually provided by <site>-mirror-net.service, which is
# exactly the hidden coupling this change removes: it would look independent and
# break the moment the mirror was torn down.
for pair in "$NET_HELPER:gen_net_helper" "$NET_UNIT:gen_net_unit" \
"$RESOLVED_DROPIN:gen_resolved_dropin"; do
f="${pair%%:*}"; g="${pair##*:}"
if [ ! -f "$f" ]; then miss "$f absent (this script owns the net layer now)"
elif diff -q <("$g") "$f" >/dev/null 2>&1; then say "OK $f matches"
else miss "$f DIFFERS from generated"; fi
done
systemctl is-enabled --quiet "${SITE}-cache-proxy-net.service" 2>/dev/null \
&& say "OK ${SITE}-cache-proxy-net.service enabled" \
|| miss "${SITE}-cache-proxy-net.service not enabled (the layer would not survive a reboot)"
br="$(util_bridge)"
if [ -n "$br" ] && ip -4 -o addr show dev "$br" 2>/dev/null | grep -q " ${LISTEN}/"; then
say "OK ${LISTEN_CIDR} on $br ($UTIL_NET)"
else miss "${LISTEN_CIDR} absent on metal-admin bridge (run: dc-cache-proxy.sh install $SITE)"; fi
ip route show default 2>/dev/null | grep -q "via ${EDGE_GW}" \
&& say "OK default route via edge ${EDGE_GW}" \
|| miss "default route via ${EDGE_GW} absent"
# Behavioral smoke test: fetch through the proxy (proxy mode) -- archive AND
# UCA, the two the deploy needs; proves cache-miss passthrough + upstream reach.
if command -v curl >/dev/null 2>&1; then
for u in "${ARTIFACT_URLS[@]}"; do
HTTP="$(curl -s -o /dev/null -w '%{http_code}' -x "http://${LISTEN}:${PROXY_PORT}" -m 20 "$u" 2>/dev/null)"
[ "$HTTP" = "200" ] && say "OK proxy serves $u (200)" \
|| miss "proxy did NOT serve $u (got '$HTTP')"
done
fi
[ "$FAILED" -eq 0 ] && { echo "dc-cache-proxy check ($SITE): PASS"; exit 0; }
echo "dc-cache-proxy check ($SITE): FAIL"; exit 1
}
# ---------------------------------------------------------------------------
# install -- idempotent; requires root. Net layer is a PREREQ (owned by
# dc-mirror.sh), verified fail-loud; apt-cacher-ng package is fail-loud too.
# ---------------------------------------------------------------------------
do_install() {
[ "$(id -u)" = "0" ] || { echo "FAIL: install requires root" >&2; exit 4; }
umask 022
# Prereq: the libvirt utility net must EXIST (we resolve its bridge). This is a
# rack-standup artifact (dc-rack-net.sh), not a mirror artifact -- so a proxy-only
# DC has no dependency on the mirror apparatus at any point.
br="$(util_bridge)"
if [ -z "$br" ]; then
echo "FAIL: libvirt network '${UTIL_NET}' not found -- cannot resolve its bridge" >&2
echo " the rack net layer comes first: dc-rack-net.sh install $SITE" >&2
exit 4
fi
# NET LAYER (owned here since 2026-07-27). Written and started BEFORE the proxy,
# so the LISTEN address and upstream egress exist independently of any mirror.
install -m 0755 /dev/null "$NET_HELPER" || exit 4
gen_net_helper > "$NET_HELPER" || exit 4
chmod 0755 "$NET_HELPER" || exit 4
gen_net_unit > "$NET_UNIT" || exit 4
mkdir -p "$(dirname "$RESOLVED_DROPIN")" || exit 4
gen_resolved_dropin > "$RESOLVED_DROPIN" || exit 4
systemctl daemon-reload || exit 4
systemctl enable "${SITE}-cache-proxy-net.service" >/dev/null 2>&1 || exit 4
systemctl restart "${SITE}-cache-proxy-net.service" \
|| { echo "FAIL: ${SITE}-cache-proxy-net.service did not apply" >&2; exit 4; }
systemctl restart systemd-resolved >/dev/null 2>&1 || true
# Prereq 2: the package (fail-loud, like dc-mirror.sh does for debmirror/nginx).
if ! dpkg -s apt-cacher-ng >/dev/null 2>&1; then
echo "FAIL: apt-cacher-ng not installed" >&2
echo " net layer is up -- run: apt-get update && apt-get install -y apt-cacher-ng" >&2
echo " then re-run: dc-cache-proxy.sh install $SITE" >&2
exit 4
fi
mkdir -p "$(dirname "$ACNG_DROPIN")" || exit 4
gen_acng_dropin > "$ACNG_DROPIN" || exit 4
systemctl enable apt-cacher-ng >/dev/null 2>&1 || exit 4
systemctl restart apt-cacher-ng || { echo "FAIL: apt-cacher-ng restart" >&2; exit 4; }
echo "dc-cache-proxy install ($SITE): done -- running check:"
do_check
}
# ---------------------------------------------------------------------------
# node -- GATE G17, run FROM A DC NODE at Stage-5 first boot. Read-only, no root,
# and deliberately touches NOTHING rack-side: only curl, chronyc, and the network.
# Three outcomes, never two: PASS (0), FAIL (1), REFUSE (3).
# ---------------------------------------------------------------------------
NFAILED=0; NREFUSED=0
nok(){ echo " OK $1"; }
nno(){ echo " FAIL $1"; NFAILED=1; }
nrefuse(){ echo " REFUSE $1"; NREFUSED=1; }
do_node() {
echo "dc-cache-proxy node check ($SITE) -- from this node, against ${LISTEN}:${PROXY_PORT}"
# -- G17 (1): artifact reachability asserted on CONTENT, with an exit-code predicate.
# A bare HTTP 200 proves nothing here: apt-cacher-ng answers its own report and error
# pages with 200, and the superseded G17 wording (`curl -sI`) exited 0 on 404/403/500.
# So all four must hold: curl exit 0, HTTP 200, the body is not HTML, and the body is
# THAT DIST's Release file. The Codename expectation is derived from the URL's own
# /dists/ path rather than hardcoded -- MEASURED 2026-07-29 against both upstreams:
# archive .../dists/jammy/Release -> "Codename: jammy"
# UCA .../dists/jammy-updates/caracal/Release -> "Codename: jammy-updates/caracal"
if ! command -v curl >/dev/null 2>&1; then
nrefuse "curl is not present on this node -- cannot evaluate artifact reachability"
else
for u in "${ARTIFACT_URLS[@]}"; do
dist="${u#*/dists/}"; dist="${dist%/Release}"
body="$(mktemp)"; cerr="$(mktemp)"
code="$(curl -sS -o "$body" -w '%{http_code}' \
-x "http://${LISTEN}:${PROXY_PORT}" -m 30 "$u" 2>"$cerr")"; crc=$?
if [ "$crc" -ne 0 ]; then
nrefuse "could not reach $u THROUGH http://${LISTEN}:${PROXY_PORT} (curl exit $crc: $(head -1 "$cerr")) -- unreachable is not 'nothing there'"
elif [ "$code" != "200" ]; then
nno "proxy returned HTTP $code for $u (want 200)"
elif grep -qi '<html' "$body"; then
nno "proxy returned an HTML page for $u, not a Release file -- a 200 with an apt-cacher-ng report/error body"
elif ! grep -qx "Codename: ${dist}" "$body"; then
nno "body served for $u is not that dist's Release file (no exact 'Codename: ${dist}' line) -- content, not merely a 200"
elif ! grep -qE '^(MD5Sum|SHA256):' "$body"; then
nno "Release body for $u carries no checksum section -- what was served is not a package index"
else
nok "proxied $u: curl exit 0, HTTP 200, Codename: ${dist}, checksum section present"
fi
rm -f "$body" "$cerr"
done
fi
# -- G17 (2): node time source is the MAAS-served rack, NOT the DC edge (D-129(iv):
# "Keep MAAS hierarchy", no NTP role on the edge). NOTE ON METHOD: chrony is NOT
# installed on the authoring jumphost, so `chronyc sources`' COLUMN FORMAT could not
# be measured. This check therefore parses no columns -- it matches only on the two
# address literals from the site table, and an output naming NEITHER is an
# unrecognised state that REFUSES.
if ! command -v chronyc >/dev/null 2>&1; then
nrefuse "chronyc is not present on this node -- cannot evaluate the node time source"
else
CS="$(chronyc sources 2>&1)"; crc=$?
if [ "$crc" -ne 0 ] || [ -z "$CS" ]; then
nrefuse "chronyc sources returned nothing (exit $crc) -- the time source is UNKNOWN, not absent"
elif grep -qw "$EDGE_GW" <<<"$CS"; then
nno "node time source includes the DC edge ${EDGE_GW} -- D-129(iv) gives the edge NO NTP role; the MAAS hierarchy is the time authority"
elif grep -qw "$RACK_IP" <<<"$CS"; then
nok "node time source is the MAAS-served rack ${RACK_IP} (D-129(iv))"
else
nrefuse "chronyc sources names NEITHER the MAAS rack ${RACK_IP} nor the edge ${EDGE_GW} -- unrecognised time source; refusing to default to success. Output:"
printf '%s\n' "$CS" | sed 's/^/ /'
fi
fi
if [ "$NFAILED" -ne 0 ]; then
echo "dc-cache-proxy node ($SITE): FAIL"; exit 1
fi
if [ "$NREFUSED" -ne 0 ]; then
echo "dc-cache-proxy node ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
fi
echo "dc-cache-proxy node ($SITE): PASS"; exit 0
}
case "$MODE" in
check) do_check ;;
install) do_install ;;
node) do_node ;;
esac
