#!/usr/bin/env bash
# scripts/dc-mirror.sh <check|install|sync> <site> -- per-DC artifact mirror,
# D-135 item 1 (apt + Ubuntu Cloud Archive; the D-107 build-out, staged).
# Part of DC-standup definition-of-done from Stage 4 on.
#
# RUNS ON THE DC RACK HOST (e.g. vvr1-dc0), not on vcloud/voffice1. Invoke
# from a repo host over ssh, piping the script (no repo clone needed there):
#
#   check:    ssh -i <dc-key> -J voffice1 <user>@<rack-transit-ip> \
#                 'sudo bash -s' -- check dc0 < scripts/dc-mirror.sh
#   install:  same with 'install dc0' (idempotent; safe to re-run).
#   sync:     same with 'sync dc0' -- starts the systemd sync job (async;
#             the initial jammy+UCA pull is several hundred GB, hours).
#
# WHAT IT OWNS (site-keyed table below; every value MEASURED or RULED, cited):
#   1. <site>-mirror-net.service -- oneshot, After=<site>-rack-legs.service:
#      (a) adds the mirror LISTEN alias (the FIRST D-134 utility-band address,
#      .4 on metal-admin) and (b) sets the rack's default route via the DC
#      edge's LAN gateway -- the D-107 "mirror's upstream sync" egress path,
#      MEASURED WORKING 2026-07-23 (rack->edge->internet ping 0% loss, curl
#      HTTP 301 -- the D-125 signature; as-executed log stage4-carve). Bridge
#      devices are resolved FROM LIBVIRT NETWORK NAMES at runtime (virbrN is
#      auto-assigned, a drifting ID -- hard rule 3).
#   2. /etc/systemd/resolved.conf.d/<site>-mirror.conf -- system DNS = the DC
#      edge's resolver (MEASURED 2026-07-23: edge Unbound answers
#      archive.ubuntu.com from the rack). Additive: the rack previously had NO
#      working system DNS; the MAAS rack agent ignores resolv.conf entirely
#      (measured, D-131), so this cannot regress node DNS.
#   3. /usr/local/sbin/<site>-mirror-sync -- debmirror runner: Ubuntu jammy
#      amd64 BINARY mirror (main/restricted/universe/multiverse; jammy,
#      jammy-updates, jammy-security; no sources) into
#      /var/lib/dc-mirror/ubuntu, plus Ubuntu Cloud Archive
#      jammy-updates/caracal into /var/lib/dc-mirror/cloud-archive. GPG
#      verified against the ubuntu-archive / ubuntu-cloud keyrings. Status in
#      /var/lib/dc-mirror/last-sync.status.
#   4. <site>-mirror-sync.service (oneshot) + <site>-mirror-sync.timer (daily,
#      randomized) -- the upstream re-sync.
#   5. /etc/nginx/sites-available/<site>-mirror (+ sites-enabled symlink) --
#      serves /var/lib/dc-mirror on http://<LISTEN>/ (autoindex). Stage-5
#      wiring consumes this via `juju model-config apt-mirror=` per DC model.
#
# PREREQ PACKAGES (install FAILS LOUD if absent; run the printed apt line
# after the net unit is up -- the rack has no egress until it is):
#   debmirror nginx ubuntu-keyring ubuntu-cloud-keyring
#
# UPSTREAM HOST NOTE (operator, 2026-07-25 -- future PREFERENCE, NOT yet applied):
#   The MAIN ubuntu archive pulls from --host=archive.ubuntu.com (the generated
#   <site>-mirror-sync, item 3). PREFER http://mirror.uoregon.edu/ubuntu/ -- the
#   closest/fastest mirror for this deployment: switch that debmirror line's
#   `--host=archive.ubuntu.com` -> `--host=mirror.uoregon.edu` (--root stays `ubuntu`).
#   SCOPE = MAIN archive ONLY. The UCA (cloud-archive) debmirror uses a SEPARATE host
#   (ubuntu-cloud.archive.canonical.com) that uoregon does NOT mirror -- leave it (or
#   find a UCA-carrying mirror). Same preference applies to dc-cache-proxy.sh's upstream
#   and the Roosevelt dedicated mirror. VERIFY uoregon carries jammy{,-updates,-security}
#   main/restricted/universe/multiverse amd64 before flipping.
#
# EXIT: 0 ok | 1 check failed | 2 bad args/unknown site | 4 install failed.
# ASCII + LF only.
set -uo pipefail

MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in check|install|sync) ;; *)
  echo "usage: dc-mirror.sh <check|install|sync> <site>" >&2; exit 2 ;; esac

# ---------------------------------------------------------------------------
# Site table. ADD A SITE ONLY WITH MEASURED/RULED VALUES (hard rule 2); the
# harness rejects rows without a MEASURED tag.
# ---------------------------------------------------------------------------
case "$SITE" in
  dc0)
    # MEASURED/RULED 2026-07-23 (Stage 4, D-134 amendment + D-135; egress
    # probe in as-executed log stage4-carve):
    #   LISTEN 10.12.8.4/22 on vr1-dc0-metal-admin -- FIRST utility-band
    #     address (D-134 .4-.9), mirror service address (D-135).
    #   EDGE_GW 10.12.4.1 -- dc0 edge LAN gw (lib-net vr1-dc0 PLANE_GW;
    #     ruled D-124/D-113 addressing), reached via the rack's existing
    #     provider-public leg 10.12.4.2/22 (dc-rack-net.sh dc0 row).
    MIRROR_NET="vr1-dc0-metal-admin"
    LISTEN_CIDR="10.12.8.4/22"
    EDGE_GW="10.12.4.1"
    ;;
  dc1)
    # !! dc1 IS NOT A MIRROR SITE ANY MORE (D-135 AMENDMENT 2026-07-27). Operator:
    # "DC0 is the test of a full mirror. DC1 is the test of the mirror proxy." dc1's
    # partial mirror, its units, timer, sync helper, nginx vhost, resolved drop-in and
    # the debmirror package were ALL REMOVED that day
    # (docs/audit/dc1-mirror-teardown-20260727.txt). dc1's artifact path is
    # scripts/dc-cache-proxy.sh (apt-cacher-ng on .4:3142).
    #   -> `check dc1` FAILS BY DESIGN. That is not a regression; use dc-cache-proxy.sh.
    #   -> `install dc1` would SILENTLY REBUILD the removed apparatus, including an
    #      ENABLED daily debmirror timer that would start a fresh ~950G pull on a rack
    #      whose ruled path is the proxy. Only do that as a deliberate, ruled decision
    #      to change dc1's strategy back -- not as a "repair".
    # The row is RETAINED (not deleted) so the measured values survive for exactly that
    # deliberate rebuild, and so this warning has somewhere to live. A runtime guard is a
    # QUEUED finding, not built here (hard rule 1).
    # MEASURED/RULED 2026-07-23 (same session; dc1 values from the D-124
    # amendment plane map + lib-net vr1-dc1 arm):
    #   LISTEN 10.12.68.4/22 on vr1-dc1-metal-admin -- utility-band .4.
    #   EDGE_GW 10.12.64.1 -- dc1 edge LAN gw (PLANE_GW, ruled provider
    #     gateway), via the rack's provider-public leg 10.12.64.2/22.
    MIRROR_NET="vr1-dc1-metal-admin"
    LISTEN_CIDR="10.12.68.4/22"
    EDGE_GW="10.12.64.1"
    ;;
  *) echo "FAIL: unknown site '$SITE' -- add a MEASURED row block first" >&2; exit 2 ;;
esac
LISTEN="${LISTEN_CIDR%%/*}"

MIRROR_ROOT="/var/lib/dc-mirror"
HELPER="/usr/local/sbin/${SITE}-mirror-net-apply"
NET_UNIT="/etc/systemd/system/${SITE}-mirror-net.service"
RESOLVED_DROPIN="/etc/systemd/resolved.conf.d/${SITE}-mirror.conf"
SYNC_HELPER="/usr/local/sbin/${SITE}-mirror-sync"
SYNC_UNIT="/etc/systemd/system/${SITE}-mirror-sync.service"
SYNC_TIMER="/etc/systemd/system/${SITE}-mirror-sync.timer"
NGINX_SITE="/etc/nginx/sites-available/${SITE}-mirror"
NGINX_LINK="/etc/nginx/sites-enabled/${SITE}-mirror"
ARCHIVE_KEYRING="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
CLOUD_KEYRING="/usr/share/keyrings/ubuntu-cloud-keyring.gpg"

# ---------------------------------------------------------------------------
# Generated file contents (single source of truth for check AND install).
# ---------------------------------------------------------------------------
gen_helper() {
  cat <<EOF
#!/usr/bin/env bash
# ${SITE}-mirror-net-apply -- generated by scripts/dc-mirror.sh; do not hand-edit.
# Adds the mirror LISTEN alias (D-134 utility .4) and the default route via the
# DC edge (D-107 mirror-sync egress). Bridge resolved from the libvirt network
# NAME at runtime (virbrN is auto-assigned, not stable identity).
set -euo pipefail
br_of() {
  virsh -c qemu:///system net-dumpxml "\$1" | sed -n "s/.*bridge name='\\([^']*\\)'.*/\\1/p"
}
ip addr replace ${LISTEN_CIDR} dev "\$(br_of ${MIRROR_NET})"
ip route replace default via ${EDGE_GW}
EOF
}

gen_net_unit() {
  cat <<EOF
[Unit]
Description=${SITE} mirror net (dc-mirror.sh; D-135 -- LISTEN alias + edge default route)
After=libvirtd.service ${SITE}-rack-legs.service network-online.target
Wants=network-online.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${HELPER}

[Install]
WantedBy=multi-user.target
EOF
}

gen_resolved_dropin() {
  cat <<EOF
# ${SITE} mirror DNS (generated by scripts/dc-mirror.sh; D-135). System DNS =
# the DC edge resolver (MEASURED answering from the rack, 2026-07-23). The
# MAAS rack agent ignores resolv.conf (D-131), so node DNS is unaffected.
[Resolve]
DNS=${EDGE_GW}
EOF
}

gen_sync_helper() {
  cat <<EOF
#!/usr/bin/env bash
# ${SITE}-mirror-sync -- generated by scripts/dc-mirror.sh (D-135 item 1);
# do not hand-edit. Jammy amd64 binary mirror + UCA caracal, GPG-verified.
# jammy-backports ADDED by the D-135 AMENDMENT 2026-07-31 (operator: "Sync the
# backports into the mirror"). It is NOT optional: the MAAS jammy image ships
# jammy-backports in its sources.list, and juju's apt-mirror model-config rewrites
# EVERY suite to this mirror -- so a mirror without it 404s and every charm's
# install hook fails on 'does not have a Release file'. That cost 22 units on the
# 2026-07-31 dc0 deploy. The mirror's suite list is a CONTRACT with the node image.
set -uo pipefail
STATUS="${MIRROR_ROOT}/last-sync.status"
echo "RUNNING \$(date -u +%FT%TZ)" > "\$STATUS"
mkdir -p ${MIRROR_ROOT}/ubuntu ${MIRROR_ROOT}/cloud-archive
debmirror ${MIRROR_ROOT}/ubuntu \\
  --host=archive.ubuntu.com --root=ubuntu --method=http \\
  --dist=jammy,jammy-updates,jammy-security,jammy-backports \\
  --section=main,restricted,universe,multiverse \\
  --arch=amd64 --nosource --diff=none \\
  --keyring=${ARCHIVE_KEYRING} --progress
RC1=\$?
debmirror ${MIRROR_ROOT}/cloud-archive \\
  --host=ubuntu-cloud.archive.canonical.com --root=ubuntu --method=http \\
  --dist=jammy-updates/caracal --section=main \\
  --arch=amd64 --nosource --diff=none \\
  --keyring=${CLOUD_KEYRING} --progress
RC2=\$?
if [ "\$RC1" -eq 0 ] && [ "\$RC2" -eq 0 ]; then
  echo "OK \$(date -u +%FT%TZ) ubuntu=0 uca=0" > "\$STATUS"; exit 0
fi
echo "FAIL \$(date -u +%FT%TZ) ubuntu=\$RC1 uca=\$RC2" > "\$STATUS"; exit 1
EOF
}

gen_sync_unit() {
  cat <<EOF
[Unit]
Description=${SITE} mirror upstream sync (dc-mirror.sh; D-135 item 1: apt+UCA)
After=${SITE}-mirror-net.service network-online.target
Requires=${SITE}-mirror-net.service

[Service]
Type=oneshot
# debmirror dies with "HOME not defined in environment!" under systemd's
# empty environment (it needs a GPG home) -- measured on the first dc0 run.
Environment=HOME=/root
ExecStart=${SYNC_HELPER}
Nice=10
IOSchedulingClass=idle
EOF
}

gen_sync_timer() {
  cat <<EOF
[Unit]
Description=${SITE} mirror daily upstream re-sync (dc-mirror.sh; D-135)

[Timer]
OnCalendar=daily
RandomizedDelaySec=3600
Persistent=true

[Install]
WantedBy=timers.target
EOF
}

gen_nginx_site() {
  cat <<EOF
# ${SITE} mirror vhost (generated by scripts/dc-mirror.sh; D-135). Serves the
# per-DC apt+UCA mirror on the D-134 utility address. Stage-5 wiring:
# juju model-config apt-mirror=http://${LISTEN}/ubuntu (per DC model).
server {
    listen ${LISTEN}:80;
    root ${MIRROR_ROOT};
    autoindex on;
    access_log /var/log/nginx/${SITE}-mirror.access.log;
}
EOF
}

# ---------------------------------------------------------------------------
# check -- mutate NOTHING; report and exit 0 ok / 1 incomplete.
# ---------------------------------------------------------------------------
FAILED=0
say(){ echo "  $1"; }
miss(){ echo "  MISS $1"; FAILED=1; }

check_file() { # path gen_fn
  if [ ! -f "$1" ]; then miss "$1 absent"; return; fi
  if diff -q <("$2") "$1" >/dev/null 2>&1; then say "OK   $1 matches"; else
    miss "$1 DIFFERS from generated content"; fi
}

do_check() {
  check_file "$HELPER" gen_helper
  check_file "$NET_UNIT" gen_net_unit
  check_file "$RESOLVED_DROPIN" gen_resolved_dropin
  check_file "$SYNC_HELPER" gen_sync_helper
  check_file "$SYNC_UNIT" gen_sync_unit
  check_file "$SYNC_TIMER" gen_sync_timer
  check_file "$NGINX_SITE" gen_nginx_site
  [ -L "$NGINX_LINK" ] && say "OK   $NGINX_LINK enabled" || miss "$NGINX_LINK absent"
  for u in "${SITE}-mirror-net.service" "${SITE}-mirror-sync.timer"; do
    [ "$(systemctl is-enabled "$u" 2>/dev/null)" = "enabled" ] \
      && say "OK   $u enabled" || miss "$u not enabled"
  done
  br="$(virsh -c qemu:///system net-dumpxml "$MIRROR_NET" 2>/dev/null \
        | sed -n "s/.*bridge name='\([^']*\)'.*/\1/p")"
  if [ -n "$br" ] && ip -4 -o addr show dev "$br" 2>/dev/null | grep -q " ${LISTEN}/"; then
    say "OK   ${LISTEN_CIDR} on $br ($MIRROR_NET)"
  else miss "${LISTEN_CIDR} absent on metal-admin bridge"; fi
  ip route show default 2>/dev/null | grep -q "via ${EDGE_GW}" \
    && say "OK   default route via edge ${EDGE_GW}" \
    || miss "default route via ${EDGE_GW} absent"
  systemctl is-active --quiet nginx \
    && say "OK   nginx active" || miss "nginx not active"
  if command -v curl >/dev/null 2>&1; then
    HTTP="$(curl -s -o /dev/null -w '%{http_code}' -m 5 "http://${LISTEN}/" 2>/dev/null)"
    [ "$HTTP" = "200" ] && say "OK   mirror answers http://${LISTEN}/ (200)" \
      || miss "mirror not answering on http://${LISTEN}/ (got '$HTTP')"
  fi
  # ASSERT ON THE STATUS CONTENT, not merely on the file existing. The original form here
  # tested `[ -f ]` and printed whatever the file said behind an unconditional "OK", so a
  # sync that FAILED and a sync that never finished both read as green and the check still
  # exited PASS. MEASURED 2026-07-27, both racks, at the Stage 4 mirror gate: dc0 read
  # "FAIL 2026-07-27T00:54:54Z ubuntu=255 uca=0" (a transient upstream 500 on
  # dists/jammy/Release) and dc1 read "RUNNING 2026-07-23T21:49:35Z" -- a marker four days
  # stale, left by the debmirror the D-135 amendment PAUSED with a signal, so it never wrote
  # a terminal state. Both printed OK. A gate that cannot distinguish "synced" from "failed"
  # or "died mid-run" is not a gate, and this is the check the stage closes on (GA-R6).
  #
  # RUNNING is deliberately NOT a pass and NOT a plain failure: it is either a sync genuinely
  # in flight or a corpse, and the two are indistinguishable from the file alone -- so it is
  # reported as an explicit UNKNOWN that the caller must resolve, never silently accepted.
  # The cross-check is the unit: a RUNNING marker with no active unit is a corpse.
  LSS="${MIRROR_ROOT}/last-sync.status"
  if [ ! -f "$LSS" ]; then
    miss "last-sync.status absent -- no sync has ever completed, so the mirror content is not attested"
  else
    LSTXT="$(tr -d '\r' < "$LSS" | head -1)"
    case "$LSTXT" in
      OK*)
        say "OK   last-sync: $LSTXT" ;;
      RUNNING*)
        if systemctl is-active --quiet "${SITE}-mirror-sync.service" 2>/dev/null; then
          miss "last-sync UNKNOWN: a sync is IN FLIGHT ($LSTXT) -- re-run this check once it finishes; the mirror is not yet attested complete"
        else
          miss "last-sync UNKNOWN: status says RUNNING ($LSTXT) but ${SITE}-mirror-sync.service is NOT active -- the sync died without writing a terminal state (stale marker, not a success)"
        fi ;;
      FAIL*)
        miss "last-sync FAILED: $LSTXT -- the mirror content is NOT attested current; investigate before any gate cites this check" ;;
      *)
        miss "last-sync UNRECOGNISED: '$LSTXT' -- refusing to interpret an unknown status word as success" ;;
    esac
  fi
  [ "$FAILED" -eq 0 ] && { echo "dc-mirror check ($SITE): PASS"; exit 0; }
  echo "dc-mirror check ($SITE): FAIL"; exit 1
}

# ---------------------------------------------------------------------------
# install -- idempotent; requires root. Net layer first (the rack has no
# egress until it is up), then fail loud on missing packages.
# ---------------------------------------------------------------------------
do_install() {
  [ "$(id -u)" = "0" ] || { echo "FAIL: install requires root" >&2; exit 4; }
  umask 022
  mkdir -p /etc/systemd/resolved.conf.d "$MIRROR_ROOT" || exit 4
  gen_helper          > "$HELPER"          && chmod 755 "$HELPER" || exit 4
  gen_net_unit        > "$NET_UNIT"        || exit 4
  gen_resolved_dropin > "$RESOLVED_DROPIN" || exit 4
  systemctl daemon-reload || exit 4
  systemctl enable --now "${SITE}-mirror-net.service" >/dev/null 2>&1 || exit 4
  systemctl try-restart systemd-resolved >/dev/null 2>&1
  MISSING=""
  command -v debmirror >/dev/null 2>&1 || MISSING="$MISSING debmirror"
  command -v nginx     >/dev/null 2>&1 || MISSING="$MISSING nginx"
  [ -f "$ARCHIVE_KEYRING" ] || MISSING="$MISSING ubuntu-keyring"
  [ -f "$CLOUD_KEYRING" ]   || MISSING="$MISSING ubuntu-cloud-keyring"
  if [ -n "$MISSING" ]; then
    echo "FAIL: missing packages:$MISSING" >&2
    echo "  net layer is up -- run: apt-get update && apt-get install -y$MISSING" >&2
    echo "  then re-run: dc-mirror.sh install $SITE" >&2
    exit 4
  fi
  gen_sync_helper > "$SYNC_HELPER" && chmod 755 "$SYNC_HELPER" || exit 4
  gen_sync_unit   > "$SYNC_UNIT"   || exit 4
  gen_sync_timer  > "$SYNC_TIMER"  || exit 4
  gen_nginx_site  > "$NGINX_SITE"  || exit 4
  ln -sf "$NGINX_SITE" "$NGINX_LINK" || exit 4
  systemctl daemon-reload || exit 4
  systemctl enable "${SITE}-mirror-sync.timer" >/dev/null 2>&1 || exit 4
  systemctl start "${SITE}-mirror-sync.timer" >/dev/null 2>&1 || exit 4
  nginx -t >/dev/null 2>&1 || { echo "FAIL: nginx config test" >&2; exit 4; }
  systemctl enable --now nginx >/dev/null 2>&1
  systemctl reload nginx || exit 4
  echo "dc-mirror install ($SITE): done -- running check:"
  echo "  (initial sync NOT started -- run: dc-mirror.sh sync $SITE)"
  do_check
}

# ---------------------------------------------------------------------------
# sync -- start the systemd sync job (async; hours for the initial pull).
# ---------------------------------------------------------------------------
do_sync() {
  [ "$(id -u)" = "0" ] || { echo "FAIL: sync requires root" >&2; exit 4; }
  systemctl start --no-block "${SITE}-mirror-sync.service" || exit 4
  echo "dc-mirror sync ($SITE): started (async). Watch:"
  echo "  systemctl status ${SITE}-mirror-sync.service"
  echo "  cat ${MIRROR_ROOT}/last-sync.status"
}

case "$MODE" in
  check)   do_check ;;
  install) do_install ;;
  sync)    do_sync ;;
esac
