#!/usr/bin/env bash
# tests/creds-matrix/run-tests.sh -- harness for scripts/creds-matrix.py (D-137 tier 1).
#
# One case per invariant, each named for the REAL defect it encodes. Fixtures are
# mutations of a minimal valid artifact set, and every case asserts BOTH the exit code and
# an output regex -- a check that fires with the wrong message is a check nobody can act on.
#
# Gauntlet contract (scripts/run-tests-all.sh:22 registers by glob and shows only tail -1):
# the LAST line self-describes, and failure detail lines lead with a bare FAIL so the
# runner's `^\s*(FAIL|\[XX\]|MISS|LEAK|COUNT)` excerpt matches them.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
CHECK="$REPO/scripts/creds-matrix.py"
TMPROOT="${TMPDIR:-/tmp}/creds-matrix-tests.$$"
mkdir -p "$TMPROOT"
trap 'rm -rf "$TMPROOT"' EXIT
P=0; F=0
ok() { P=$((P+1)); echo " [ok] $1"; }
no() { F=$((F+1)); echo "FAIL $1"; }
# ---- minimal VALID fixture set -------------------------------------------------------
mkenv() {
local d; d="$(mktemp -d "$TMPROOT/env.XXXXXX")"
mkdir -p "$d/creds-manifests"
cat > "$d/matrix.tsv" <<'EOF'
alpha-key per-site vr1-office1 jumphost alpha_ed25519 ssh service consolidated stage2 operator-terminal - n-alpha
alpha-key per-site vr1-office1 jumphost alpha_ed25519.pub none service consolidated stage2 operator-terminal - n-alpha
EOF
cat > "$d/creds-manifests/vr1-office1.manifest" <<'EOF'
alpha_ed25519 600 local
alpha_ed25519.pub 644 local
EOF
cat > "$d/notes.md" <<'EOF'
## n-alpha
The alpha service keypair.
EOF
echo "$d"
}
run() { # run <env-dir> [extra args...] -> sets RC, writes $1/out
local d="$1"; shift
python3 "$CHECK" --matrix "$d/matrix.tsv" --manifest-dir "$d/creds-manifests" \
--notes "$d/notes.md" --repo "$REPO" "$@" > "$d/out" 2>&1
RC=$?
}
row() { printf '%s\n' "$1" >> "$2/matrix.tsv"; }
note() { printf '\n## %s\nfixture note.\n' "$1" >> "$2/notes.md"; }
echo "=== creds-matrix harness ==="
# T1 -- the baseline must actually PASS, or every other case proves nothing.
D=$(mkenv); run "$D"
[ "$RC" = 0 ] && grep -q '^PASS: creds-matrix' "$D/out" \
&& ok "T1 a consistent matrix/manifest/notes set -> PASS, exit 0" \
|| { no "T1 the valid baseline must pass (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T2 -- a field containing whitespace silently shifts every later column.
D=$(mkenv); row 'beta pass singleton vr1-office1 jumphost beta.txt api service consolidated stage2 operator-terminal - -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q 'field(s), want 12' "$D/out" \
&& ok "T2 a row with a whitespace-bearing field -> field-count FAIL" \
|| { no "T2 whitespace in a field must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T3 -- bare `dc0` is the drifting form that has meant two different things per DC.
D=$(mkenv); row 'beta per-DC dc0 jumphost beta.txt api service off-manifest-known stage3 operator-terminal - -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q 'not region-qualified' "$D/out" \
&& ok "T3 a bare 'dc0' site-key -> REJECTED (region-qualified discipline)" \
|| { no "T3 bare dc0 must be rejected (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T4 -- an invented custody token would silently escape every custody-conditioned rule.
D=$(mkenv); row 'beta singleton vr1-office1 jumphost beta.txt api service probably-fine stage2 operator-terminal - -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q "custody 'probably-fine' invalid" "$D/out" \
&& ok "T4 an unknown custody token -> enum FAIL" \
|| { no "T4 custody enum must be closed (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T5 -- the SAME identity declared twice at one location is a real duplication error.
D=$(mkenv); row 'alpha-key per-site vr1-office1 jumphost alpha_ed25519 ssh service consolidated stage2 operator-terminal - n-alpha' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q 'duplicate (id,site,host-role,filename)' "$D/out" \
&& ok "T5 one identity declared twice at one location -> duplicate FAIL" \
|| { no "T5 same-identity duplication must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T6 -- REGRESSION: two DIFFERENT identities may share one file (the Vault init artifact
# holds both the unseal shares and the root token). Keying dedup on location alone
# false-failed this.
D=$(mkenv)
row 'vault-shares singleton - jumphost vault-init none service off-manifest-known vr0-phase02 operator-terminal - -' "$D"
row 'vault-root singleton - jumphost vault-init none service off-manifest-known vr0-phase02 operator-terminal - -' "$D"
run "$D"
[ "$RC" = 0 ] \
&& ok "T6 two identities sharing one file -> allowed (Vault init regression)" \
|| { no "T6 distinct identities may share a file (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T7 -- SEC-021's class: the register expects a credential the manifest never declared.
D=$(mkenv); row 'gamma per-site vr1-office1 jumphost gamma.txt api service consolidated stage2 operator-terminal SEC-021 -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q "EXPECTED-BUT-ABSENT: 'gamma.txt'" "$D/out" \
&& ok "T7 expected by the matrix, absent from the manifest -> EXPECTED-BUT-ABSENT (SEC-021 class)" \
|| { no "T7 expected-but-absent must fire (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T8 -- the undeclared-secret class SEC-009/-020 kept missing, now from the other side.
D=$(mkenv); printf 'delta.txt 600 local\n' >> "$D/creds-manifests/vr1-office1.manifest"
run "$D"
[ "$RC" = 1 ] && grep -q "UNEXPECTED-EXTRA: manifest declares 'delta.txt'" "$D/out" \
&& ok "T8 declared in the manifest, unknown to the matrix -> UNEXPECTED-EXTRA" \
|| { no "T8 unexpected-extra must fire (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T9 -- THE FALSE GREEN. An empty matrix examines no site, so a site-scoped loop finds
# nothing undeclared and reports CLEAN while every declared credential sits unregistered.
# Same defect netbox/sandbox-fidelity-check.py:131-143 records: an upper bound
# masquerading as an assertion. The site set must be the UNION of both sides.
D=$(mkenv); printf '# every row removed\n' > "$D/matrix.tsv"
run "$D"
[ "$RC" = 1 ] && grep -q 'UNEXPECTED-EXTRA' "$D/out" \
&& ok "T9 an EMPTY matrix against populated manifests -> FAILS (the false-green regression)" \
|| { no "T9 an empty matrix must not read clean (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T10 -- a mint-ref that points past EOF is stale provenance, the drift class tier 3 exists for.
D=$(mkenv); row 'beta singleton vr1-office1 headend beta.txt api service off-manifest-known stage2 script:scripts/creds-audit.sh:99999 - -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q 'points past EOF' "$D/out" \
&& ok "T10 a mint-ref line past EOF -> stale-provenance FAIL" \
|| { no "T10 out-of-range mint-ref must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T11 -- a mint-ref naming a file that no longer exists (the rename/retire class).
D=$(mkenv); row 'beta singleton vr1-office1 headend beta.txt api service off-manifest-known stage2 script:scripts/no-such-script.sh:1 - -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q 'mint-ref target does not exist' "$D/out" \
&& ok "T11 a mint-ref to a deleted file -> FAIL" \
|| { no "T11 missing mint-ref target must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T12 -- research FINDING 1: ssh-keygen has ZERO hits repo-wide, so 12 declared secrets
# have no mint command. Those rows are the DEBT the register exists to surface -- they
# must be ADMITTED and reported, never treated as parse errors.
D=$(mkenv); run "$D"
[ "$RC" = 0 ] && grep -q 'provenance debt: 2 row(s) are mint-ref=operator-terminal' "$D/out" \
&& ok "T12 operator-terminal rows -> admitted and reported as debt, not parse errors" \
|| { no "T12 operator-terminal must be admitted + counted (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T13 -- a per-DC credential existing at one DC only (the SEC-021 divergence shape).
D=$(mkenv)
row 'dc0-power per-DC vr1-dc0 jumphost vr1-dc0-power_ed25519 ssh service consolidated stage3 operator-terminal - -' "$D"
mkdir -p "$D/creds-manifests"; printf 'vr1-dc0-power_ed25519 600 local\n' > "$D/creds-manifests/vr1-dc0.manifest"
run "$D"
[ "$RC" = 1 ] && grep -q 'S5 ASYMMETRY' "$D/out" \
&& ok "T13 a per-DC credential at one DC only -> ASYMMETRY FAIL (SEC-021 shape)" \
|| { no "T13 per-DC asymmetry must fire (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T14 -- narrowing to one DC cannot silently 'pass' a check that needs both.
D=$(mkenv); run "$D" --dc vr1-dc1
[ "$RC" = 0 ] && grep -q 'symmetry needs BOTH DCs, so the check is SKIPPED' "$D/out" \
&& ok "T14 --dc narrows -> symmetry SKIPPED explicitly, never silently passed" \
|| { no "T14 --dc must skip symmetry out loud (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T15 -- D-137 ruling 5: ONE IDENTITY SERVES ONE PRINCIPAL TYPE. This is the live SEC-020
# conflation (`admin` is both the human GUI login and the automation identity).
D=$(mkenv)
row 'shared-ident singleton vr1-office1 headend shared.pass gui human off-manifest-known stage2 operator-terminal - -' "$D"
row 'shared-ident singleton vr1-office1 headend shared.apikey api service off-manifest-known stage2 operator-terminal - -' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q 'IDENTITY CONFLATION' "$D/out" \
&& ok "T15 one identity serving human AND service -> ruling-5 CONFLATION FAIL" \
|| { no "T15 ruling-5 invariant must fire (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T16 -- a non-credential index file (principal '-') must not fault the ruling-5 check.
D=$(mkenv)
row 'folder-readme per-site vr1-office1 jumphost README none - consolidated - - - -' "$D"
printf 'README 600 local\n' >> "$D/creds-manifests/vr1-office1.manifest"
run "$D"
[ "$RC" = 0 ] \
&& ok "T16 a principal='-' index file -> skipped by ruling 5, not faulted" \
|| { no "T16 principal '-' must be skipped (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T17 -- a notes-ref with no entry means ruling-2 governance prose was silently dropped.
D=$(mkenv); row 'beta singleton vr1-office1 headend beta.txt api service off-manifest-known stage2 operator-terminal - n-nonexistent' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q "notes-ref 'n-nonexistent' has no" "$D/out" \
&& ok "T17 a dangling notes-ref -> FAIL (ruling-2 prose must not be droppable)" \
|| { no "T17 dangling notes-ref must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T18 -- an orphan note is prose that survived the row it documented (the other direction).
D=$(mkenv); note 'n-orphan' "$D"
run "$D"
[ "$RC" = 1 ] && grep -q "orphan note 'n-orphan'" "$D/out" \
&& ok "T18 a note referenced by no row -> orphan FAIL" \
|| { no "T18 orphan notes must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T19 -- the mode column is DERIVED (.pub -> 644, else 600), so a hand-edited manifest
# mode is drift. This is what makes manifests generated artifacts per ruling 2.
D=$(mkenv); sed -i 's/^alpha_ed25519.pub 644/alpha_ed25519.pub 600/' "$D/creds-manifests/vr1-office1.manifest"
run "$D"
[ "$RC" = 1 ] && grep -qE 'FIELD DRIFT|render drift' "$D/out" \
&& ok "T19 a hand-edited manifest mode -> render/field DRIFT FAIL" \
|| { no "T19 mode drift must be caught (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T20 -- rows with no host artifact (filename '-') must skip existence-shaped checks
# rather than fault: the juju-<dc> superuser password is stored NOWHERE by ruling.
D=$(mkenv)
row 'nowhere-user singleton vr1-office1 - - api service off-manifest-known stage5 operator-terminal - -' "$D"
run "$D"
[ "$RC" = 0 ] \
&& ok "T20 a filename='-' row (credential with no host artifact) -> skipped, not faulted" \
|| { no "T20 no-artifact rows must be skipped (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T21 -- an unbuilt tier must announce itself. A silent pass is the exact false-green
# this design exists to prevent.
D=$(mkenv); run "$D" --all
[ "$RC" = 0 ] && grep -q 'tier 2 (EXISTENCE) NOT BUILT' "$D/out" \
&& grep -q 'tier 3 (VALIDITY) NOT BUILT' "$D/out" \
&& ok "T21 --all names tiers 2 and 3 as NOT BUILT -> no silent pass" \
|| { no "T21 unbuilt tiers must self-announce (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T22 -- METADATA ONLY. The checker must never gain a content-reading verb; it inspects
# existence, mode, and declared provenance. Mirrors tests/creds-audit/run-tests.sh:71-73.
grep -qE '\b(cat|head|tail|less|more|od|xxd|hexdump)\b' "$CHECK" \
&& no "T22 the checker contains a content-reading verb -- it must inspect metadata only" \
|| ok "T22 metadata-only: no cat/head/tail/od/xxd in the checker"
# T23 -- --render must emit a GENERATED banner, so nobody hand-edits a derived manifest.
D=$(mkenv); run "$D" --render
[ "$RC" = 0 ] && grep -q 'GENERATED from creds-matrix.tsv' "$D/out" \
&& grep -qE 'alpha_ed25519\.pub +644 +local' "$D/out" \
&& ok "T23 --render emits a GENERATED banner and the derived modes" \
|| { no "T23 --render output shape (rc=$RC)"; sed 's/^/ /' "$D/out"; }
# T24 -- the REAL tree must be RED, and red for the RIGHT reasons. A checker that goes
# green on today's tree is wrong (plan: EXPECTED RED BY DESIGN). Guards against someone
# "fixing" the register by deleting the rows that fail.
python3 "$CHECK" > "$TMPROOT/real.out" 2>&1; RRC=$?
[ "$RRC" = 1 ] && grep -q 'IDENTITY CONFLATION' "$TMPROOT/real.out" \
&& grep -q 'EXPECTED-BUT-ABSENT' "$TMPROOT/real.out" \
&& ok "T24 the real tree is RED with the ruling-5 conflation + SEC-021 absences intact" \
|| { no "T24 the real tree must stay RED by design (rc=$RRC)"; sed 's/^/ /' "$TMPROOT/real.out"; }
echo
if [ "$F" = 0 ]; then echo "creds-matrix: $P/$P PASS"; exit 0; fi
echo "creds-matrix: FAILURES: $F (passed $P)"; exit 1