Newer
Older
openstack-caracal-ipv4 / scripts / cloud-snapshot.sh
#!/usr/bin/env bash
# scripts/cloud-snapshot.sh [--with-controller-backup] [MODEL]
#
# Pre-change baseline snapshot (DOCFIX-102; session-ledger script-backlog item 7).
# Captures the restorable/diffable point-in-time baseline that D-070 KEPT when it
# retired the KVM-snapshot restore path: the juju export + inventory captures.
# Run it BEFORE consequential changes (tenant onboarding, maintenance windows,
# charm refreshes) to pin the pre-change state. It complements, not replaces,
# `bash scripts/cloud-assert.sh --capture` (DOCFIX-075): cloud-assert's BOM is
# the behavioral-sweep-attached deploy baseline; this is the fast juju-layer
# capture that needs NO OpenStack admin scope and no unit ssh.
#
# Captures into asbuilt/<UTC YYYYMMDD-HHMMSS>/ (the D-070 committed-baseline
# location; commit the directory -- it is the diffable pre-change baseline):
#   bundle-exported.yaml   juju export-bundle -m <MODEL>        (D-070 "juju export")
#   juju-status.json       juju status -m <MODEL> --format=json (asbuilt BOM prior art)
#   model-config.yaml      juju model-config -m <MODEL>         (inventory)
#   manifest.txt           UTC stamp, model, juju client version, per-file
#                          sha256 + byte counts, redaction report  (inventory)
#
# Secret hygiene: exported bundles and model config CAN embed credential-shaped
# option values. Every captured text file is scrubbed by scripts/snapshot_scrub.py
# (credential-shaped keys get their values replaced with a REDACTED marker) and
# any redaction downgrades the verdict to WARN -- review before committing.
#
# --with-controller-backup: additionally runs `juju create-backup -m
# admin/controller` (EXISTS on juju 3.6 -- D-071 AMENDMENT 2026-07-05 /
# DOCFIX-088; the controller model on this cloud is admin/controller). The
# archive is ~900MB-class and slow, hence flag-gated and never default. It is
# written to ~/openstack-baseline/ on the jumphost (snap cannot use /tmp; and
# per ops-update-procedure 2.1 the archive is secret-adjacent and NEVER
# committed) -- the manifest records its path/size/sha256 only.
#
# Mutates NOTHING in the cloud except the flag-gated create-backup call (which
# writes a backup archive controller-side, then downloads it). Repo-side it
# writes only under asbuilt/; jumphost-side only under ~/openstack-baseline/
# (flag path). CLOUD_SNAPSHOT_ROOT overrides the repo root for the offline
# harness ONLY.
#
# Exit (cloud-assert prior art): 0 snapshot complete | 1 HELD/FAIL (juju
# unreachable, or any capture failed -- partial captures are counted and
# reported) | 2 WARN (snapshot complete but redactions applied -- review
# before commit).  ASCII + LF.

set -uo pipefail
shopt -s inherit_errexit 2>/dev/null || true
IFS=$'\n\t'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$SCRIPT_DIR/.." && pwd)"
ROOT="${CLOUD_SNAPSHOT_ROOT:-$REPO}"

MODEL="openstack"; WITH_BACKUP=0
for a in "$@"; do
  case "$a" in
    --with-controller-backup) WITH_BACKUP=1 ;;
    -*) echo "FAIL: unknown flag $a"; exit 1 ;;
    *) MODEL="$a" ;;
  esac
done

# Structured captures take stdout ONLY -- merged stderr lands at char 0 and
# kills the parse / corrupts the capture (DOCFIX-094). stderr goes to a
# tempfile and is surfaced on failure, never discarded.
JERR="$(mktemp)"; trap 'rm -f "$JERR"' EXIT
J() { juju "$@" </dev/null 2>"$JERR"; }
jerr() { head -1 "$JERR" 2>/dev/null || true; }

WORST=0
fail() { echo "  [FAIL] $*"; WORST=1; }
warn() { echo "  [WARN] $*"; [ "$WORST" -ne 1 ] && WORST=2; }
ok()   { echo "  [ok]   $*"; }

command -v juju    >/dev/null 2>&1 || { echo "FAIL: juju not found"; exit 1; }
command -v jq      >/dev/null 2>&1 || { echo "FAIL: jq required"; exit 1; }
command -v python3 >/dev/null 2>&1 || { echo "FAIL: python3 required"; exit 1; }
SCRUB="$SCRIPT_DIR/snapshot_scrub.py"
[ -f "$SCRUB" ] || { echo "FAIL: helper missing: scripts/snapshot_scrub.py"; exit 2; }

echo "================ S0: juju reachability (nothing written until this passes) ================"
ST=$(J status -m "$MODEL" --format=json || true)
if ! jq -e .applications >/dev/null 2>&1 <<<"$ST"; then
  fail "cannot read juju status for model '$MODEL': $(jerr)"
  echo; echo "CLOUD-SNAPSHOT: HELD (juju unreachable -- nothing captured)"; exit 1
fi
ok "model '$MODEL' reachable"

TS="$(date -u +%Y%m%d-%H%M%S)"
DIR="$ROOT/asbuilt/$TS"
mkdir -p "$DIR"
JVER=$(J version || true)

echo "================ S1: captures -> asbuilt/$TS/ ================"
CAP_TOTAL=0; CAP_OK=0
capture() { # capture <filename> <validator-regex> <label> -- <juju args...>
  local fn="$1" rx="$2" label="$3"; shift 3; shift   # drop the -- separator
  CAP_TOTAL=$((CAP_TOTAL+1))
  local out
  out=$(J "$@" || true)
  if [ -n "$out" ] && grep -qE "$rx" <<<"$out"; then
    printf '%s\n' "$out" > "$DIR/$fn"
    CAP_OK=$((CAP_OK+1)); ok "$label -> $fn"
  else
    fail "$label FAILED: $(jerr)"
  fi
}
# juju-status.json: already captured and shape-validated at S0
printf '%s\n' "$ST" > "$DIR/juju-status.json"
CAP_TOTAL=$((CAP_TOTAL+1)); CAP_OK=$((CAP_OK+1)); ok "juju status -> juju-status.json"
capture "bundle-exported.yaml" '^applications:|^  [a-z]' "juju export-bundle" -- export-bundle -m "$MODEL"
capture "model-config.yaml"    ':'                       "juju model-config"  -- model-config -m "$MODEL" --format=yaml

echo "================ S2: secret scrub (credential-shaped values) ================"
SCRUB_FILES=()
for f in juju-status.json bundle-exported.yaml model-config.yaml; do
  [ -f "$DIR/$f" ] && SCRUB_FILES+=("$DIR/$f")
done
RED=$(python3 "$SCRUB" "${SCRUB_FILES[@]}" 2>"$JERR"); SRC=$?
[ "$SRC" -eq 0 ] || fail "scrub helper errored (rc $SRC): $(jerr)"
NRED=$(grep -c '^REDACTED ' <<<"$RED" || true)
if [ "${NRED:-0}" -gt 0 ]; then
  while IFS= read -r l; do [ -n "$l" ] && warn "$l"; done <<<"$RED"
  # sed-is-not-a-verifier: assert the redactions actually stuck
  LEFT=$(python3 "$SCRUB" --check "${SCRUB_FILES[@]}" 2>/dev/null || true)
  if grep -q '^REDACTED ' <<<"$LEFT"; then
    fail "scrub did not stick -- credential-shaped values remain; DO NOT COMMIT $DIR"
  else
    warn "$NRED credential-shaped value(s) REDACTED -- review the capture before committing"
  fi
else
  ok "no credential-shaped values found"
fi

if [ "$WITH_BACKUP" -eq 1 ]; then
  echo "================ S3: controller backup (flag-gated; D-071 amendment / DOCFIX-088) ================"
  CAP_TOTAL=$((CAP_TOTAL+1))
  BDIR="$HOME/openstack-baseline"; mkdir -p "$BDIR"
  BFILE="$BDIR/juju-controller-backup-$TS.tar.gz"
  # controller model is admin/controller on this cloud (ops-update-procedure 2.1).
  # ~900MB-class, ~35s measured 2026-07-05; archive stays on the jumphost.
  BOUT=$(J create-backup -m admin/controller --filename "$BFILE" "cloud-snapshot $TS baseline" || true)
  if [ -s "$BFILE" ]; then
    CAP_OK=$((CAP_OK+1))
    BSUM=$(sha256sum "$BFILE" | awk '{print $1}')
    BSIZE=$(stat -c %s "$BFILE" 2>/dev/null || wc -c < "$BFILE")
    ok "controller backup -> $BFILE ($BSIZE bytes)"
    BACKUP_LINE="controller-backup: $BFILE bytes=$BSIZE sha256=$BSUM (jumphost-local; secret-adjacent; NEVER commit)"
  else
    fail "controller backup FAILED: $(head -1 <<<"$BOUT")$(jerr)"
    BACKUP_LINE="controller-backup: FAILED"
  fi
else
  BACKUP_LINE="controller-backup: not requested (use --with-controller-backup; ~900MB-class, slow)"
fi

echo "================ S4: inventory manifest ================"
{
  echo "# cloud-snapshot manifest (D-070 export/inventory baseline; DOCFIX-102)"
  echo "utc: $TS"
  echo "model: $MODEL"
  echo "juju-client: ${JVER:-unknown}"
  echo "captured-by: scripts/cloud-snapshot.sh"
  echo "captured: $CAP_OK/$CAP_TOTAL"
  echo "redactions: ${NRED:-0}"
  echo "$BACKUP_LINE"
  echo "files:"
  ( cd "$DIR" && for f in *; do
      [ "$f" = "manifest.txt" ] && continue
      printf '%s  %s  %s\n' "$(sha256sum "$f" | awk '{print $1}')" "$(wc -c < "$f" | tr -d ' ')" "$f"
    done )
} > "$DIR/manifest.txt"
ok "manifest -> manifest.txt"

echo
if [ "$CAP_OK" -lt "$CAP_TOTAL" ]; then WORST=1; fi
case "$WORST" in
  0) echo "CLOUD-SNAPSHOT: PASS (captured $CAP_OK/$CAP_TOTAL) -- commit asbuilt/$TS (D-070 baseline)" ;;
  2) echo "CLOUD-SNAPSHOT: WARN (captured $CAP_OK/$CAP_TOTAL, ${NRED:-0} redaction(s)) -- review asbuilt/$TS before committing" ;;
  *) echo "CLOUD-SNAPSHOT: FAIL (captured $CAP_OK/$CAP_TOTAL) -- partial baseline in asbuilt/$TS; do not treat as a baseline" ;;
esac
exit "$WORST"