diff --git a/docs/changelog-20260727-stage5-phase0.md b/docs/changelog-20260727-stage5-phase0.md index a77ad47..1c092ec 100644 --- a/docs/changelog-20260727-stage5-phase0.md +++ b/docs/changelog-20260727-stage5-phase0.md @@ -202,3 +202,37 @@ **Revert.** `git revert `. The lint reverts to accepting any unknown flag as its root -- i.e. back to a gate that cannot distinguish a clean repo from an unexamined one. + +## 7. R15(3) -- preflight no longer FAILS OPEN + +**What.** `scripts/preflight.sh`: `note()` now takes a gate label and a per-gate +policy for rc=2, treats ANY unexpected exit code as FAIL, and names the gate in the +message. All 7 call sites updated. + +**Why.** R15(3), ruled 2026-07-27. Reproduced before fixing, in isolation, using the +exact `note()` from `:27`: sub-gate exits **127, 126, 130 and 3 ALL left +`PREFLIGHT: PASS -- clear to add-model / deploy`**. This is the gate that authorises +the deploy. + +**Measured departure from R15's stated rationale, and why.** R15 says rc=2 "is how +these checkers signal 'I could not evaluate anything'" and should be remapped. Measured +per checker, that is only PARTLY true: +`repo-lint.sh` documents rc2 as "warnings only"; `pre-flight-checks.sh` documents rc2 as +"warning"; but `provider-bundle-check.py` returns 2 only on could-not-evaluate (PyYAML +missing `:32`, cannot parse `:108`, cannot merge overlay `:113`) and `creds-matrix.py` +likewise (`:116`, `:1037`). `channel_assert.py` uses 2 for BOTH (`sys.exit(2)` missing +PyYAML at `:19`; `return 2 if warns` at `:55`). +**A blanket 2 -> FAIL would have turned the standing L1 legacy-ASCII warn into a deploy +blocker.** So rc=2 is per-gate: FAIL for P2/P5, WARN for P1/P4. P3 is ambiguous IN THE +CHECKER and kept WARN -- its missing-PyYAML root cause is caught anyway by P2, which +shares it and is strict. Making P3's rc2 unambiguous belongs in `channel_assert.py`; +flagged, not guessed. + +**Verification.** Harness `tests/preflight` **16/16** (was 10). T11-T13 lock the +127/126/3 fail-open cases; T14/T15 lock could-not-evaluate -> FAIL; **T16 locks the +NON-over-correction** (P1 rc=2 stays WARN). Gauntlet ALL GREEN (82); repo-lint 0 fail. +Real repo: preflight still exits 1 for the same known reasons, with no new +UNEXPECTED/COULD-NOT-EVALUATE lines -- i.e. the fix did not newly break it. + +**Revert.** `git revert `. Preflight returns to ignoring every sub-gate +exit code that is not 1 or 2 -- a crashed gate again reads as clear-to-deploy. diff --git a/scripts/preflight.sh b/scripts/preflight.sh index dce908a..30f48d3 100644 --- a/scripts/preflight.sh +++ b/scripts/preflight.sh @@ -24,22 +24,70 @@ REPO="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" cd "$REPO" || { echo "FAIL: cannot cd to repo root $REPO"; exit 1; } WORST=0 -note() { local rc=$1; [ "$rc" -eq 1 ] && WORST=1; [ "$rc" -eq 2 ] && [ "$WORST" -ne 1 ] && WORST=2; } +# ---- R15(3): preflight must not FAIL OPEN (ruled 2026-07-27, OPS) -------------- +# MEASURED DEFECT: note() tested ONLY rc 1 and 2, so a sub-gate exiting 127 (command +# not found), 126 (not executable), 130 (interrupted) or any rc>=3 left WORST +# untouched and this printed "PREFLIGHT: PASS -- clear to add-model / deploy". +# Reproduced for all four. This is the gate that AUTHORISES the deploy. +# +# rc=2 IS PER-GATE, and R15's blanket framing ("rc=2 is how these checkers signal +# 'I could not evaluate anything'") is only PARTLY right -- measured per checker: +# P1 repo-lint.sh rc2 = "warnings only" (documented header) -> WARN +# P2 provider-bundle-check rc2 = could-not-evaluate (PyYAML missing at :32, +# cannot parse :108, cannot merge overlay :113) -> FAIL +# P3 channel_assert.py rc2 = BOTH: sys.exit(2) on missing PyYAML at :19, but +# `return 2 if warns` at :55 is a LEGITIMATE warn. +# Ambiguous IN THE CHECKER, so preflight cannot +# resolve it -- kept WARN because the offline/ +# charmhub-unreachable warn is the common path, and +# the missing-PyYAML root cause is caught anyway by +# P2, which shares it and is strict. Flagged, not +# guessed: making rc2 unambiguous belongs in +# channel_assert.py, not here. +# P4 pre-flight-checks.sh rc2 = "warning" (documented header) -> WARN +# P5 creds-matrix.py rc2 = could-not-evaluate (bad --dc at :1037, +# unreadable matrix at :116) -> FAIL +# Blanket-remapping 2 to FAIL would have turned the standing L1 legacy-ASCII WARN +# into a deploy blocker. Measured before implementing. +# +# note +note() { + local rc="$1" gate="$2" two="${3:-warn}" + case "$rc" in + 0) ;; + 1) WORST=1 ;; + 2) + if [ "$two" = "fail" ]; then + echo " [FAIL] $gate exited 2 = COULD NOT EVALUATE (not 'no findings'). A gate" + echo " that could not look must never read as a gate that found nothing." + WORST=1 + else + [ "$WORST" -ne 1 ] && WORST=2 + fi + ;; + *) + echo " [FAIL] $gate exited $rc -- UNEXPECTED exit code (127=command not found," + echo " 126=not executable, 130=interrupted, >=3=undefined). Treated as" + echo " FAIL: this used to leave 'PREFLIGHT: PASS -- clear to deploy'." + WORST=1 + ;; + esac +} echo "================ P1: repo lint ================" -bash scripts/repo-lint.sh "$REPO"; note $? +bash scripts/repo-lint.sh "$REPO"; note $? "P1 repo lint" warn echo "================ P2: bundle invariants ================" -python3 scripts/provider-bundle-check.py bundle.yaml; note $? +python3 scripts/provider-bundle-check.py bundle.yaml; note $? "P2 bundle invariants" fail echo "================ P3: channel assert (charmhub) ================" -python3 scripts/channel_assert.py bundle.yaml; note $? +python3 scripts/channel_assert.py bundle.yaml; note $? "P3 channel assert" warn echo "================ P4: live pre-flight (MAAS/overlay/nodes) ================" if [ -x scripts/pre-flight-checks.sh ] || [ -f scripts/pre-flight-checks.sh ]; then - bash scripts/pre-flight-checks.sh; note $? + bash scripts/pre-flight-checks.sh; note $? "P4 live pre-flight" warn else - echo " [FAIL] scripts/pre-flight-checks.sh missing"; note 1 + echo " [FAIL] scripts/pre-flight-checks.sh missing"; note 1 "P4 live pre-flight" fail fi echo "================ P5: credential matrix (D-137 tier 1 + tier 2 local) ================" @@ -68,9 +116,9 @@ # it to a warning instead of stopping the deploy. A gate that vanishes must never read as # a pass (harness T9 encodes this). if [ -f scripts/creds-matrix.py ]; then - python3 scripts/creds-matrix.py --tier2; note $? + python3 scripts/creds-matrix.py --tier2; note $? "P5 credential matrix" fail else - echo " [FAIL] scripts/creds-matrix.py missing"; note 1 + echo " [FAIL] scripts/creds-matrix.py missing"; note 1 "P5 credential matrix" fail fi echo "================ P6: stage-2 reminders (NOT run here) ================" diff --git a/tests/preflight/run-tests.sh b/tests/preflight/run-tests.sh index 2bcec9d..fc4cec3 100644 --- a/tests/preflight/run-tests.sh +++ b/tests/preflight/run-tests.sh @@ -83,5 +83,25 @@ echo " FAIL T10 credential gate ordering"; FAIL=$((FAIL+1)) fi +# ---- T11-T16 -- R15(3): PREFLIGHT MUST NOT FAIL OPEN (ruled 2026-07-27) ---- +# MEASURED DEFECT: note() tested only rc 1 and 2, so a sub-gate exiting 127/126/130 +# or any rc>=3 left WORST untouched and preflight printed +# "PREFLIGHT: PASS -- clear to add-model / deploy" +# Reproduced for all four before the fix. This is the gate that AUTHORISES the deploy. +run 1 'PREFLIGHT: FAIL' "T11 R15(3) sub-gate exit 127 (command not found) -> FAIL, was PASS" "$(mkfix rc127 127 0 1 0)" +run 1 'PREFLIGHT: FAIL' "T12 R15(3) sub-gate exit 126 (not executable) -> FAIL, was PASS" "$(mkfix rc126 0 126 1 0)" +run 1 'PREFLIGHT: FAIL' "T13 R15(3) sub-gate exit 3 (undefined) -> FAIL, was PASS" "$(mkfix rc3 0 0 1 3)" + +# rc=2 is PER-GATE and was measured per checker rather than blanket-remapped. +# P2/P5 rc2 means "could not evaluate" -> FAIL. A gate that could not look must never +# read as a gate that found nothing. +run 1 'COULD NOT EVALUATE' "T14 R15(3) P2 rc=2 (could-not-evaluate) -> FAIL" "$(mkfix p2two 0 2 1 0)" +run 1 'COULD NOT EVALUATE' "T15 R15(3) P5 rc=2 (could-not-evaluate) -> FAIL" "$(mkfix p5two 0 0 1 2)" + +# P1 rc2 is "warnings only" by its own documented contract. It must STAY a warning -- +# a blanket 2->FAIL would have turned the standing L1 legacy-ASCII warn into a deploy +# blocker. This case is the regression lock on NOT over-correcting. +run 2 'PREFLIGHT: WARN' "T16 R15(3) P1 rc=2 (warnings only) stays WARN, not FAIL" "$(mkfix p1two 2 0 1 0)" + echo; echo "RESULT: PASS=$PASS FAIL=$FAIL" [[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1