diff --git a/docs/changelog-20260727-stage5-phase0.md b/docs/changelog-20260727-stage5-phase0.md index 1c092ec..9ef2a2a 100644 --- a/docs/changelog-20260727-stage5-phase0.md +++ b/docs/changelog-20260727-stage5-phase0.md @@ -236,3 +236,30 @@ **Revert.** `git revert `. Preflight returns to ignoring every sub-gate exit code that is not 1 or 2 -- a crashed gate again reads as clear-to-deploy. + +## 8. R15(2) -- the gauntlet pins a harness MANIFEST, not a count + +**What.** `tests/HARNESS-MANIFEST` (82 names, sorted) + a drift gate in +`scripts/run-tests-all.sh`, with `--record-manifest` to re-record deliberately. + +**Why.** R15(2), ruled 2026-07-27. `run-tests-all.sh:35` already had a ZERO-floor +(`RAN -eq 0` -> exit 2); what was absent was any pin on WHICH harnesses ran. The count +`81` existed only as prose in CURRENT-STATE, so a renamed or deleted harness was neither +run nor failed and the gauntlet still printed ALL GREEN. + +**Proven able to fail, on both cases -- including the one a bare count cannot catch.** +(a) renamed `render-baseline` -> drift reported, exit 1. (b) renamed it AND added a decoy +so the **count held at exactly 82** -> still caught, naming both the missing and the +unpinned entry. That second case is precisely why the ruling chose a manifest over a +count. + +**Scope note.** The drift gate runs only on a FULL gauntlet; a filtered run +(`run-tests-all.sh `) legitimately executes a subset, so comparing it would be a +false red. A MISSING manifest is itself a FAIL -- without it, ALL GREEN is unfalsifiable. + +**Seeding.** Seeded from a tree verified in the same session (gauntlet ALL GREEN, +repo-lint 0 fail), per the ruling's explicit warning not to seed from whatever happens to +be on disk. This is why R15(2) was sequenced LAST of the three. + +**Revert.** `git revert ` and delete `tests/HARNESS-MANIFEST`. The gauntlet +returns to reporting ALL GREEN over whatever harnesses happen to exist. diff --git a/scripts/run-tests-all.sh b/scripts/run-tests-all.sh index cfcfe97..d30c68d 100644 --- a/scripts/run-tests-all.sh +++ b/scripts/run-tests-all.sh @@ -18,6 +18,26 @@ REPO="$(cd "$HERE/.." && pwd)" FILTER="${1:-}" FAILED=(); RAN=0 +# ---- R15(2): the gauntlet pins a MANIFEST of harness NAMES, not a count --------- +# Ruled 2026-07-27 ("All three, with a harness MANIFEST rather than a count"), OPS. +# The zero-floor below (RAN -eq 0 -> exit 2) already existed; what was ABSENT was any +# pin on WHICH harnesses ran. The count `81` lived only as prose in CURRENT-STATE, so +# a renamed or deleted harness was neither run nor failed and the gauntlet still +# printed ALL GREEN. A bare count would also miss the add-one/remove-one case; a +# manifest of names does not. +# Mirrors two proven in-repo patterns: clientdocs/sweep-receipt.txt hash-pinning and +# the D-137 creds-manifests derive-plus-drift gate. +# Seeded 2026-07-27 from a tree verified in the same session (gauntlet ALL GREEN, +# repo-lint 0 fail) -- the ruling explicitly warns a manifest must not be seeded from +# whatever happens to be on disk. +# Re-record deliberately, and say why in the commit: bash scripts/run-tests-all.sh --record-manifest +MANIFEST="$REPO/tests/HARNESS-MANIFEST" +if [ "${1:-}" = "--record-manifest" ]; then + ls -1 "$REPO"/tests/*/run-tests.sh 2>/dev/null | xargs -n1 dirname | xargs -n1 basename \ + | sort > "$MANIFEST" + echo "recorded $(wc -l < "$MANIFEST") harness name(s) to tests/HARNESS-MANIFEST" + exit 0 +fi shopt -s nullglob for t in "$REPO"/tests/*/run-tests.sh; do name="$(basename "$(dirname "$t")")" @@ -33,6 +53,27 @@ done echo if [ "$RAN" -eq 0 ]; then echo "GAUNTLET: no harnesses matched '${FILTER}'"; exit 2; fi + +# R15(2) drift gate -- only meaningful on a FULL run; a filtered run legitimately +# executes a subset, so comparing it to the manifest would be a false red. +if [ -z "$FILTER" ]; then + if [ ! -f "$MANIFEST" ]; then + echo "GAUNTLET: FAIL -- tests/HARNESS-MANIFEST is MISSING. The gauntlet cannot" + echo " attest WHICH harnesses ran, so ALL GREEN would be unfalsifiable. Restore" + echo " it from git, or re-record deliberately with --record-manifest." + exit 1 + fi + ACTUAL="$(ls -1 "$REPO"/tests/*/run-tests.sh 2>/dev/null | xargs -n1 dirname \ + | xargs -n1 basename | sort)" + if ! DIFF="$(diff <(cat "$MANIFEST") <(printf '%s\n' "$ACTUAL") 2>&1)"; then + echo "GAUNTLET: FAIL -- harness set DRIFTED from tests/HARNESS-MANIFEST." + echo " '<' = pinned but MISSING (renamed or deleted -- it was neither run nor failed)" + echo " '>' = present but UNPINNED (added without recording)" + printf '%s\n' "$DIFF" | sed 's/^/ /' + echo " If the change is intended: bash scripts/run-tests-all.sh --record-manifest" + exit 1 + fi +fi if [ "${#FAILED[@]}" -eq 0 ]; then echo "GAUNTLET: ALL GREEN ($RAN harnesses)"; exit 0 fi diff --git a/tests/HARNESS-MANIFEST b/tests/HARNESS-MANIFEST new file mode 100644 index 0000000..869dc63 --- /dev/null +++ b/tests/HARNESS-MANIFEST @@ -0,0 +1,82 @@ +carve-host-interfaces +checks +claude-guard +clientdocs-scripts +clientdocs-skill +cloud-assert +cloudinit-vm +cloud-snapshot +creds-audit +creds-matrix +creds-probe +d063-apply +d115-office-carve +d120-compose-bands +d124-transit-seed +dc-cache-proxy +dc-dc-ceph-disk-budget +dc-dc-dr-drill +dc-dc-mtu-geneve-budget +dc-dc-prefixes-import +dc-dc-radosgw-multisite +dc-dc-rbd-mirror +dc-dc-whole-host-budget +dc-edge-wan-import +dc-mirror +dc-rack-mgmt-import +dc-rack-net +dc-selector +juju-spaces-check +keystone-policy-drift +ledger-scan +lib-validate +maas-node-power +netem-link +netem-sudoers +node-vm +office1-record-dump +opentofu-validate +opnsense-api +opnsense-bootstrap-apikey +opnsense-plugins +opnsense-prep-image +opnsense-set-interface-v4 +opnsense-set-interface-v6 +phase-00-maas-standup +phase-00-teardown-d061 +phase-02 +phase-03 +phase-03-adminrc +phase-04 +phase-04-create +phase-04-internal-cert-san +phase-05 +phase-05-amphora +phase-06-bootstrap +phase-06-capi-stack +phase-06-k8s-bootstrap +phase-06-kubeconfig-gate +phase-06-mgmt-vm +phase-06-net-setup +phase-07-conductor-graft +preflight +prereqs +provider-bundle-check +record-audit +reenroll-hosts +render-baseline +repo-lint +roles-aggregates-import +sandbox-fidelity-check +sandbox-seed +site-baseleg +site-forward +site-headend-install +site-ssh-config +tenant-acceptance +tenant-assert +tenant-offboard +tenant-onboard +trust_filter +validate +vault-kv-health