Newer
Older
openstack-caracal-dc-dc / tests / render-baseline / run-tests.sh
#!/usr/bin/env bash
# tests/render-baseline/run-tests.sh
#
# Integrity harness for the frozen renderer reproduction fixtures (see README.md).
#
# ASSERTS: the fixtures still are what they were when frozen.
# DOES NOT ASSERT: that they match the live overlays/ or bundle.yaml. They are a
# historical snapshot and live is SUPPOSED to diverge from them once the R2/R11
# reconciliation lands. Repointing these at live files would turn this harness red
# for the exact change it exists to support -- the tests/creds-matrix T24 trap.
set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FIX="$HERE/fixtures"
PASS=0; FAIL=0

ok()  { PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
bad() { FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; }

t_eq() { # label expected actual
  if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi
}

echo "== render-baseline: frozen reproduction fixture integrity =="

# T1 -- the fixture directory and both fixtures exist.
for f in vr1-dc1-vips.v4-11app.yaml bundle-vips.v4-11app.txt; do
  if [ -f "$FIX/$f" ]; then ok "T1 fixture present: $f"
  else bad "T1 fixture MISSING: $f -- a frozen baseline cannot be regenerated later; recover from git"; fi
done

# T2 -- SHA256SUMS exists and every fixture matches it. This is the core assertion:
#       silent edits to a baseline make every future renderer diff meaningless.
if [ -f "$HERE/SHA256SUMS" ]; then
  ok "T2a SHA256SUMS present"
  if ( cd "$HERE" && sha256sum -c --status SHA256SUMS ); then
    ok "T2b every fixture matches its pinned hash"
  else
    bad "T2b FIXTURE HASH MISMATCH -- a frozen baseline was modified. Do NOT re-pin to make this green; recover the original from git and add a NEW dated fixture instead"
  fi
else
  bad "T2a SHA256SUMS MISSING -- fixtures are unpinned and cannot be trusted as a baseline"
fi

# T3 -- SHA256SUMS covers EVERY file in fixtures/ (a fixture added without a hash line
#       would pass T2 vacuously -- `sha256sum -c` only checks what is listed).
if [ -f "$HERE/SHA256SUMS" ]; then
  n_fix="$(find "$FIX" -maxdepth 1 -type f | wc -l | tr -d ' ')"
  n_pin="$(grep -c . "$HERE/SHA256SUMS" 2>/dev/null || echo 0)"
  t_eq "T3 SHA256SUMS covers every fixture (no unpinned file)" "$n_fix" "$n_pin"
fi

# T4 -- the YAML fixture still parses and still carries the 11-application v4 shape it
#       was frozen for. Structure, not just bytes: a valid-but-wrong file would pass T2
#       if someone re-pinned it, so assert the CONTENT claim the filename makes.
if command -v python3 >/dev/null 2>&1; then
  read -r n_apps n_bad3 <<<"$(python3 - "$FIX/vr1-dc1-vips.v4-11app.yaml" <<'PY'
import sys, yaml
d = yaml.safe_load(open(sys.argv[1])) or {}
apps = d.get('applications') or {}
n = 0; bad3 = 0
for _, spec in apps.items():
    v = (spec.get('options') or {}).get('vip')
    if not v:
        continue
    n += 1
    if len(v.split()) != 3:
        bad3 += 1
print(n, bad3)
PY
)"
  t_eq "T4a YAML fixture carries 11 vip application(s)" "11" "$n_apps"
  t_eq "T4b every frozen vip is a v4 TRIPLE (pre-dual-stack)" "0" "$n_bad3"
else
  bad "T4 python3 absent -- cannot verify fixture structure ('could not look' is not 'nothing there')"
fi

# T5 -- the extracted bundle fixture still declares 11 rows and its rows are tab-shaped.
#       (Instrument note: this file is TAB-separated by construction. The creds-matrix.tsv
#       lesson -- a space-aligned file parsed with -F'\t' returns zero rows and looks like
#       a real result -- is why this asserts the count rather than trusting the parse.)
n_rows="$(grep -vc '^#' "$FIX/bundle-vips.v4-11app.txt" 2>/dev/null || echo 0)"
t_eq "T5a bundle fixture carries 11 app rows" "11" "$n_rows"
n_tab="$(grep -v '^#' "$FIX/bundle-vips.v4-11app.txt" | grep -Pc '\t' 2>/dev/null || echo 0)"
t_eq "T5b every bundle fixture row is tab-separated" "11" "$n_tab"

# T6 -- THE REPRODUCTION CASE the renderer was frozen for (added 2026-07-29 when
#       scripts/render-dc-overlays.py landed; README:71-72 says ADD this, and do
#       NOT repoint T1-T5 at live files).
#
#       It compares against the FIXTURE, deliberately NOT overlays/vr1-dc1-vips.yaml:
#       the live overlay is about to be rewritten by the R2/R11 reconciliation and
#       the fixture is not, so pointing this at live would turn the harness red for
#       the very change it exists to support -- the tests/creds-matrix T24 trap this
#       whole directory was built to avoid.
#
#       SKIPS (does not fail) if the renderer is absent, so this file remains valid
#       at any commit before the renderer landed. A skip is REPORTED, never silent.
REN="$(cd "$HERE/../.." && pwd)/scripts/render-dc-overlays.py"
if [ ! -f "$REN" ]; then
  echo "  SKIP  T6 renderer not present at this commit ($REN)"
else
  _t6="$(mktemp -d)"
  if python3 "$REN" derive --dc vr1-dc1 --from-overlay "$FIX/vr1-dc1-vips.v4-11app.yaml" \
       --output overlays/vr1-dc1-vips.yaml -o "$_t6/v.yaml" >/dev/null 2>&1; then
    got="$(python3 "$REN" render "$_t6/v.yaml" 2>/dev/null | sha256sum | cut -d' ' -f1)"
    want="$(sha256sum "$FIX/vr1-dc1-vips.v4-11app.yaml" | cut -d' ' -f1)"
    t_eq "T6 renderer reproduces the frozen fixture byte-for-byte" "$want" "$got"
  else
    bad "T6 renderer could not derive a values file for vr1-dc1"
  fi
  rm -rf "$_t6"
fi

echo
echo "RESULT: PASS=$PASS FAIL=$FAIL"
[ "$FAIL" -eq 0 ] || exit 1