#!/usr/bin/env bash
# scripts/dc-mirror.sh <check|install|sync> <site> -- per-DC artifact mirror,
# D-135 item 1 (apt + Ubuntu Cloud Archive; the D-107 build-out, staged).
# Part of DC-standup definition-of-done from Stage 4 on.
#
# RUNS ON THE DC RACK HOST (e.g. vvr1-dc0), not on vcloud/voffice1. Invoke
# from a repo host over ssh, piping the script (no repo clone needed there):
#
# check: ssh -i <dc-key> -J voffice1 <user>@<rack-transit-ip> \
# 'sudo bash -s' -- check dc0 < scripts/dc-mirror.sh
# install: same with 'install dc0' (idempotent; safe to re-run).
# sync: same with 'sync dc0' -- starts the systemd sync job (async;
# the initial jammy+UCA pull is several hundred GB, hours).
#
# WHAT IT OWNS (site-keyed table below; every value MEASURED or RULED, cited):
# 1. <site>-mirror-net.service -- oneshot, After=<site>-rack-legs.service:
# (a) adds the mirror LISTEN alias (the FIRST D-134 utility-band address,
# .4 on metal-admin) and (b) sets the rack's default route via the DC
# edge's LAN gateway -- the D-107 "mirror's upstream sync" egress path,
# MEASURED WORKING 2026-07-23 (rack->edge->internet ping 0% loss, curl
# HTTP 301 -- the D-125 signature; as-executed log stage4-carve). Bridge
# devices are resolved FROM LIBVIRT NETWORK NAMES at runtime (virbrN is
# auto-assigned, a drifting ID -- hard rule 3).
# 2. /etc/systemd/resolved.conf.d/<site>-mirror.conf -- system DNS = the DC
# edge's resolver (MEASURED 2026-07-23: edge Unbound answers
# archive.ubuntu.com from the rack). Additive: the rack previously had NO
# working system DNS; the MAAS rack agent ignores resolv.conf entirely
# (measured, D-131), so this cannot regress node DNS.
# 3. /usr/local/sbin/<site>-mirror-sync -- debmirror runner: Ubuntu jammy
# amd64 BINARY mirror (main/restricted/universe/multiverse; jammy,
# jammy-updates, jammy-security; no sources) into
# /var/lib/dc-mirror/ubuntu, plus Ubuntu Cloud Archive
# jammy-updates/caracal into /var/lib/dc-mirror/cloud-archive. GPG
# verified against the ubuntu-archive / ubuntu-cloud keyrings. Status in
# /var/lib/dc-mirror/last-sync.status.
# 4. <site>-mirror-sync.service (oneshot) + <site>-mirror-sync.timer (daily,
# randomized) -- the upstream re-sync.
# 5. /etc/nginx/sites-available/<site>-mirror (+ sites-enabled symlink) --
# serves /var/lib/dc-mirror on http://<LISTEN>/ (autoindex). Stage-5
# wiring consumes this via `juju model-config apt-mirror=` per DC model.
#
# PREREQ PACKAGES (install FAILS LOUD if absent; run the printed apt line
# after the net unit is up -- the rack has no egress until it is):
# debmirror nginx ubuntu-keyring ubuntu-cloud-keyring
#
# EXIT: 0 ok | 1 check failed | 2 bad args/unknown site | 4 install failed.
# ASCII + LF only.
set -uo pipefail
MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in check|install|sync) ;; *)
echo "usage: dc-mirror.sh <check|install|sync> <site>" >&2; exit 2 ;; esac
# ---------------------------------------------------------------------------
# Site table. ADD A SITE ONLY WITH MEASURED/RULED VALUES (hard rule 2); the
# harness rejects rows without a MEASURED tag.
# ---------------------------------------------------------------------------
case "$SITE" in
dc0)
# MEASURED/RULED 2026-07-23 (Stage 4, D-134 amendment + D-135; egress
# probe in as-executed log stage4-carve):
# LISTEN 10.12.8.4/22 on vr1-dc0-metal-admin -- FIRST utility-band
# address (D-134 .4-.9), mirror service address (D-135).
# EDGE_GW 10.12.4.1 -- dc0 edge LAN gw (lib-net vr1-dc0 PLANE_GW;
# ruled D-124/D-113 addressing), reached via the rack's existing
# provider-public leg 10.12.4.2/22 (dc-rack-net.sh dc0 row).
MIRROR_NET="vr1-dc0-metal-admin"
LISTEN_CIDR="10.12.8.4/22"
EDGE_GW="10.12.4.1"
;;
dc1)
# MEASURED/RULED 2026-07-23 (same session; dc1 values from the D-124
# amendment plane map + lib-net vr1-dc1 arm):
# LISTEN 10.12.68.4/22 on vr1-dc1-metal-admin -- utility-band .4.
# EDGE_GW 10.12.64.1 -- dc1 edge LAN gw (PLANE_GW, ruled provider
# gateway), via the rack's provider-public leg 10.12.64.2/22.
MIRROR_NET="vr1-dc1-metal-admin"
LISTEN_CIDR="10.12.68.4/22"
EDGE_GW="10.12.64.1"
;;
*) echo "FAIL: unknown site '$SITE' -- add a MEASURED row block first" >&2; exit 2 ;;
esac
LISTEN="${LISTEN_CIDR%%/*}"
MIRROR_ROOT="/var/lib/dc-mirror"
HELPER="/usr/local/sbin/${SITE}-mirror-net-apply"
NET_UNIT="/etc/systemd/system/${SITE}-mirror-net.service"
RESOLVED_DROPIN="/etc/systemd/resolved.conf.d/${SITE}-mirror.conf"
SYNC_HELPER="/usr/local/sbin/${SITE}-mirror-sync"
SYNC_UNIT="/etc/systemd/system/${SITE}-mirror-sync.service"
SYNC_TIMER="/etc/systemd/system/${SITE}-mirror-sync.timer"
NGINX_SITE="/etc/nginx/sites-available/${SITE}-mirror"
NGINX_LINK="/etc/nginx/sites-enabled/${SITE}-mirror"
ARCHIVE_KEYRING="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
CLOUD_KEYRING="/usr/share/keyrings/ubuntu-cloud-keyring.gpg"
# ---------------------------------------------------------------------------
# Generated file contents (single source of truth for check AND install).
# ---------------------------------------------------------------------------
gen_helper() {
cat <<EOF
#!/usr/bin/env bash
# ${SITE}-mirror-net-apply -- generated by scripts/dc-mirror.sh; do not hand-edit.
# Adds the mirror LISTEN alias (D-134 utility .4) and the default route via the
# DC edge (D-107 mirror-sync egress). Bridge resolved from the libvirt network
# NAME at runtime (virbrN is auto-assigned, not stable identity).
set -euo pipefail
br_of() {
virsh -c qemu:///system net-dumpxml "\$1" | sed -n "s/.*bridge name='\\([^']*\\)'.*/\\1/p"
}
ip addr replace ${LISTEN_CIDR} dev "\$(br_of ${MIRROR_NET})"
ip route replace default via ${EDGE_GW}
EOF
}
gen_net_unit() {
cat <<EOF
[Unit]
Description=${SITE} mirror net (dc-mirror.sh; D-135 -- LISTEN alias + edge default route)
After=libvirtd.service ${SITE}-rack-legs.service network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${HELPER}
[Install]
WantedBy=multi-user.target
EOF
}
gen_resolved_dropin() {
cat <<EOF
# ${SITE} mirror DNS (generated by scripts/dc-mirror.sh; D-135). System DNS =
# the DC edge resolver (MEASURED answering from the rack, 2026-07-23). The
# MAAS rack agent ignores resolv.conf (D-131), so node DNS is unaffected.
[Resolve]
DNS=${EDGE_GW}
EOF
}
gen_sync_helper() {
cat <<EOF
#!/usr/bin/env bash
# ${SITE}-mirror-sync -- generated by scripts/dc-mirror.sh (D-135 item 1);
# do not hand-edit. Jammy amd64 binary mirror + UCA caracal, GPG-verified.
set -uo pipefail
STATUS="${MIRROR_ROOT}/last-sync.status"
echo "RUNNING \$(date -u +%FT%TZ)" > "\$STATUS"
mkdir -p ${MIRROR_ROOT}/ubuntu ${MIRROR_ROOT}/cloud-archive
debmirror ${MIRROR_ROOT}/ubuntu \\
--host=archive.ubuntu.com --root=ubuntu --method=http \\
--dist=jammy,jammy-updates,jammy-security \\
--section=main,restricted,universe,multiverse \\
--arch=amd64 --nosource --diff=none \\
--keyring=${ARCHIVE_KEYRING} --progress
RC1=\$?
debmirror ${MIRROR_ROOT}/cloud-archive \\
--host=ubuntu-cloud.archive.canonical.com --root=ubuntu --method=http \\
--dist=jammy-updates/caracal --section=main \\
--arch=amd64 --nosource --diff=none \\
--keyring=${CLOUD_KEYRING} --progress
RC2=\$?
if [ "\$RC1" -eq 0 ] && [ "\$RC2" -eq 0 ]; then
echo "OK \$(date -u +%FT%TZ) ubuntu=0 uca=0" > "\$STATUS"; exit 0
fi
echo "FAIL \$(date -u +%FT%TZ) ubuntu=\$RC1 uca=\$RC2" > "\$STATUS"; exit 1
EOF
}
gen_sync_unit() {
cat <<EOF
[Unit]
Description=${SITE} mirror upstream sync (dc-mirror.sh; D-135 item 1: apt+UCA)
After=${SITE}-mirror-net.service network-online.target
Requires=${SITE}-mirror-net.service
[Service]
Type=oneshot
ExecStart=${SYNC_HELPER}
Nice=10
IOSchedulingClass=idle
EOF
}
gen_sync_timer() {
cat <<EOF
[Unit]
Description=${SITE} mirror daily upstream re-sync (dc-mirror.sh; D-135)
[Timer]
OnCalendar=daily
RandomizedDelaySec=3600
Persistent=true
[Install]
WantedBy=timers.target
EOF
}
gen_nginx_site() {
cat <<EOF
# ${SITE} mirror vhost (generated by scripts/dc-mirror.sh; D-135). Serves the
# per-DC apt+UCA mirror on the D-134 utility address. Stage-5 wiring:
# juju model-config apt-mirror=http://${LISTEN}/ubuntu (per DC model).
server {
listen ${LISTEN}:80;
root ${MIRROR_ROOT};
autoindex on;
access_log /var/log/nginx/${SITE}-mirror.access.log;
}
EOF
}
# ---------------------------------------------------------------------------
# check -- mutate NOTHING; report and exit 0 ok / 1 incomplete.
# ---------------------------------------------------------------------------
FAILED=0
say(){ echo " $1"; }
miss(){ echo " MISS $1"; FAILED=1; }
check_file() { # path gen_fn
if [ ! -f "$1" ]; then miss "$1 absent"; return; fi
if diff -q <("$2") "$1" >/dev/null 2>&1; then say "OK $1 matches"; else
miss "$1 DIFFERS from generated content"; fi
}
do_check() {
check_file "$HELPER" gen_helper
check_file "$NET_UNIT" gen_net_unit
check_file "$RESOLVED_DROPIN" gen_resolved_dropin
check_file "$SYNC_HELPER" gen_sync_helper
check_file "$SYNC_UNIT" gen_sync_unit
check_file "$SYNC_TIMER" gen_sync_timer
check_file "$NGINX_SITE" gen_nginx_site
[ -L "$NGINX_LINK" ] && say "OK $NGINX_LINK enabled" || miss "$NGINX_LINK absent"
for u in "${SITE}-mirror-net.service" "${SITE}-mirror-sync.timer"; do
[ "$(systemctl is-enabled "$u" 2>/dev/null)" = "enabled" ] \
&& say "OK $u enabled" || miss "$u not enabled"
done
br="$(virsh -c qemu:///system net-dumpxml "$MIRROR_NET" 2>/dev/null \
| sed -n "s/.*bridge name='\([^']*\)'.*/\1/p")"
if [ -n "$br" ] && ip -4 -o addr show dev "$br" 2>/dev/null | grep -q " ${LISTEN}/"; then
say "OK ${LISTEN_CIDR} on $br ($MIRROR_NET)"
else miss "${LISTEN_CIDR} absent on metal-admin bridge"; fi
ip route show default 2>/dev/null | grep -q "via ${EDGE_GW}" \
&& say "OK default route via edge ${EDGE_GW}" \
|| miss "default route via ${EDGE_GW} absent"
systemctl is-active --quiet nginx \
&& say "OK nginx active" || miss "nginx not active"
if command -v curl >/dev/null 2>&1; then
HTTP="$(curl -s -o /dev/null -w '%{http_code}' -m 5 "http://${LISTEN}/" 2>/dev/null)"
[ "$HTTP" = "200" ] && say "OK mirror answers http://${LISTEN}/ (200)" \
|| miss "mirror not answering on http://${LISTEN}/ (got '$HTTP')"
fi
[ -f "${MIRROR_ROOT}/last-sync.status" ] \
&& say "OK last-sync: $(cat "${MIRROR_ROOT}/last-sync.status")" \
|| say "note last-sync.status absent (no sync attempted yet)"
[ "$FAILED" -eq 0 ] && { echo "dc-mirror check ($SITE): PASS"; exit 0; }
echo "dc-mirror check ($SITE): FAIL"; exit 1
}
# ---------------------------------------------------------------------------
# install -- idempotent; requires root. Net layer first (the rack has no
# egress until it is up), then fail loud on missing packages.
# ---------------------------------------------------------------------------
do_install() {
[ "$(id -u)" = "0" ] || { echo "FAIL: install requires root" >&2; exit 4; }
umask 022
mkdir -p /etc/systemd/resolved.conf.d "$MIRROR_ROOT" || exit 4
gen_helper > "$HELPER" && chmod 755 "$HELPER" || exit 4
gen_net_unit > "$NET_UNIT" || exit 4
gen_resolved_dropin > "$RESOLVED_DROPIN" || exit 4
systemctl daemon-reload || exit 4
systemctl enable --now "${SITE}-mirror-net.service" >/dev/null 2>&1 || exit 4
systemctl try-restart systemd-resolved >/dev/null 2>&1
MISSING=""
command -v debmirror >/dev/null 2>&1 || MISSING="$MISSING debmirror"
command -v nginx >/dev/null 2>&1 || MISSING="$MISSING nginx"
[ -f "$ARCHIVE_KEYRING" ] || MISSING="$MISSING ubuntu-keyring"
[ -f "$CLOUD_KEYRING" ] || MISSING="$MISSING ubuntu-cloud-keyring"
if [ -n "$MISSING" ]; then
echo "FAIL: missing packages:$MISSING" >&2
echo " net layer is up -- run: apt-get update && apt-get install -y$MISSING" >&2
echo " then re-run: dc-mirror.sh install $SITE" >&2
exit 4
fi
gen_sync_helper > "$SYNC_HELPER" && chmod 755 "$SYNC_HELPER" || exit 4
gen_sync_unit > "$SYNC_UNIT" || exit 4
gen_sync_timer > "$SYNC_TIMER" || exit 4
gen_nginx_site > "$NGINX_SITE" || exit 4
ln -sf "$NGINX_SITE" "$NGINX_LINK" || exit 4
systemctl daemon-reload || exit 4
systemctl enable "${SITE}-mirror-sync.timer" >/dev/null 2>&1 || exit 4
systemctl start "${SITE}-mirror-sync.timer" >/dev/null 2>&1 || exit 4
nginx -t >/dev/null 2>&1 || { echo "FAIL: nginx config test" >&2; exit 4; }
systemctl enable --now nginx >/dev/null 2>&1
systemctl reload nginx || exit 4
echo "dc-mirror install ($SITE): done -- running check:"
echo " (initial sync NOT started -- run: dc-mirror.sh sync $SITE)"
do_check
}
# ---------------------------------------------------------------------------
# sync -- start the systemd sync job (async; hours for the initial pull).
# ---------------------------------------------------------------------------
do_sync() {
[ "$(id -u)" = "0" ] || { echo "FAIL: sync requires root" >&2; exit 4; }
systemctl start --no-block "${SITE}-mirror-sync.service" || exit 4
echo "dc-mirror sync ($SITE): started (async). Watch:"
echo " systemctl status ${SITE}-mirror-sync.service"
echo " cat ${MIRROR_ROOT}/last-sync.status"
}
case "$MODE" in
check) do_check ;;
install) do_install ;;
sync) do_sync ;;
esac
