Newer
Older
openstack-caracal-dc-dc / scripts / run-tests-all.sh
#!/usr/bin/env bash
# scripts/run-tests-all.sh [filter]
#
# The harness gauntlet: discovers and runs every tests/*/run-tests.sh, one
# summary line each, worst exit wins. This is the TOOLING gate -- run it after
# any script/harness change (the change-delivery loop) and before trusting the
# gate scripts in a deploy. Deliberately NOT folded into preflight.sh:
# preflight gates the DEPLOY TARGET, this gates the TOOLS -- different cadence,
# different failure meaning.
#
# [filter] runs only harnesses whose directory name contains the substring
# (e.g. `bash scripts/run-tests-all.sh phase-06`).
# Mutates NOTHING (harnesses confine themselves to mktemp dirs).
# Exit: 0 all green | 1 any harness failed | 2 no harnesses matched.
# ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/.." && pwd)"
FILTER="${1:-}"
FAILED=(); RAN=0
# ---- R15(2): the gauntlet pins a MANIFEST of harness NAMES, not a count ---------
# Ruled 2026-07-27 ("All three, with a harness MANIFEST rather than a count"), OPS.
# The zero-floor below (RAN -eq 0 -> exit 2) already existed; what was ABSENT was any
# pin on WHICH harnesses ran. The count `81` lived only as prose in CURRENT-STATE, so
# a renamed or deleted harness was neither run nor failed and the gauntlet still
# printed ALL GREEN. A bare count would also miss the add-one/remove-one case; a
# manifest of names does not.
# Mirrors two proven in-repo patterns: clientdocs/sweep-receipt.txt hash-pinning and
# the D-137 creds-manifests derive-plus-drift gate.
# Seeded 2026-07-27 from a tree verified in the same session (gauntlet ALL GREEN,
# repo-lint 0 fail) -- the ruling explicitly warns a manifest must not be seeded from
# whatever happens to be on disk.
# Re-record deliberately, and say why in the commit:  bash scripts/run-tests-all.sh --record-manifest
MANIFEST="$REPO/tests/HARNESS-MANIFEST"
if [ "${1:-}" = "--record-manifest" ]; then
  ls -1 "$REPO"/tests/*/run-tests.sh 2>/dev/null | xargs -n1 dirname | xargs -n1 basename \
    | sort > "$MANIFEST"
  echo "recorded $(wc -l < "$MANIFEST") harness name(s) to tests/HARNESS-MANIFEST"
  exit 0
fi
shopt -s nullglob
for t in "$REPO"/tests/*/run-tests.sh; do
  name="$(basename "$(dirname "$t")")"
  [ -n "$FILTER" ] && case "$name" in *"$FILTER"*) ;; *) continue ;; esac
  RAN=$((RAN+1))
  OUT="$(bash "$t" 2>&1)"; RC=$?
  TAIL="$(printf '%s\n' "$OUT" | tail -1)"
  printf '  %-38s %s\n' "$name" "$TAIL"
  if [ "$RC" -ne 0 ]; then
    FAILED+=("$name")
    printf '%s\n' "$OUT" | grep -E '^\s*(FAIL|\[XX\]|MISS|LEAK|COUNT)' | head -4 | sed 's/^/      /'
  fi
done
echo
if [ "$RAN" -eq 0 ]; then echo "GAUNTLET: no harnesses matched '${FILTER}'"; exit 2; fi

# R15(2) drift gate -- only meaningful on a FULL run; a filtered run legitimately
# executes a subset, so comparing it to the manifest would be a false red.
if [ -z "$FILTER" ]; then
  if [ ! -f "$MANIFEST" ]; then
    echo "GAUNTLET: FAIL -- tests/HARNESS-MANIFEST is MISSING. The gauntlet cannot"
    echo "  attest WHICH harnesses ran, so ALL GREEN would be unfalsifiable. Restore"
    echo "  it from git, or re-record deliberately with --record-manifest."
    exit 1
  fi
  ACTUAL="$(ls -1 "$REPO"/tests/*/run-tests.sh 2>/dev/null | xargs -n1 dirname \
            | xargs -n1 basename | sort)"
  if ! DIFF="$(diff <(cat "$MANIFEST") <(printf '%s\n' "$ACTUAL") 2>&1)"; then
    echo "GAUNTLET: FAIL -- harness set DRIFTED from tests/HARNESS-MANIFEST."
    echo "  '<' = pinned but MISSING (renamed or deleted -- it was neither run nor failed)"
    echo "  '>' = present but UNPINNED (added without recording)"
    printf '%s\n' "$DIFF" | sed 's/^/    /'
    echo "  If the change is intended: bash scripts/run-tests-all.sh --record-manifest"
    exit 1
  fi
fi
if [ "${#FAILED[@]}" -eq 0 ]; then
  echo "GAUNTLET: ALL GREEN ($RAN harnesses)"; exit 0
fi
echo "GAUNTLET: ${#FAILED[@]}/$RAN FAILED -- ${FAILED[*]}"
exit 1