#!/usr/bin/env bash
# tests/tenant-onboard/run-tests.sh -- focused offline harness for guard behaviors:
# - stage4 duplicate-CIDR guard (2026-07-03 sweep; was fail-open under SIGPIPE)
# - stage3 H2 re-run guards (2026-07-06): app-cred/keypair exists + local secret
# intact -> SKIP rc0; exists WITHOUT usable local secret -> STOP rc1 with
# operator guidance (no implicit credential rotation on a re-run)
# - stage7 H3 canary (2026-07-06): happy path CANARY PASS + teardown rc0;
# ssh-dead path CANARY FAIL rc1 AND teardown still runs (trap EXIT)
# Fake openstack/ssh replay fixtures; fake sleep compresses wait loops; fake HOMEs
# provide stub admin-openrc + CA. Not a full-stage harness -- assertions target
# the guard/canary behaviors only.
# Exit: 0 all pass | 1 any case failed. ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0
BIN="$TMP/bin"; H="$TMP/home"; mkdir -p "$BIN" "$H"
printf 'export OS_AUTH_URL=https://fake:5000/v3\nexport OS_CACERT=$HOME/vault-init/vault-ca-root.pem\n' > "$H/admin-openrc"
mkdir -p "$H/vault-init"
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -subj /CN=fake-ca \
-keyout "$H/vault-init/ca.key" -out "$H/vault-init/vault-ca-root.pem" -days 2 >/dev/null 2>&1
DOMID="d0000000000000000000000000000001"
PID="b0000000000000000000000000000001"
ACID="e0000000000000000000000000000001"
CLUID="c0000000000000000000000000000002"
SVCID="c0000000000000000000000000000003"
cat > "$BIN/openstack" <<FB
#!/usr/bin/env bash
case "\$*" in
*"subnet list"*) cat "\$FIXDIR/subnets.txt" ;;
"token issue -f value -c domain_id") echo "$DOMID" ;;
"project create --domain $DOMID"*) : ;;
"project show testco-prod --domain $DOMID -f value -c id") echo "$PID" ;;
"user create --domain $DOMID"*) : ;;
"user show testco-cluster --domain $DOMID -f value -c id") echo "$CLUID" ;;
"user show testco-svc --domain $DOMID -f value -c id") echo "$SVCID" ;;
"role add --project $PID"*) : ;;
"role assignment list --project $PID --user $SVCID --names -f value -c Role")
RARC=\$(cat "\$FIXDIR/roleassign-rc" 2>/dev/null || echo 0)
if [ "\$RARC" != 0 ]; then echo "An unexpected error prevented the server from fulfilling your request. (HTTP 500)" >&2; exit "\$RARC"; fi
cat "\$FIXDIR/roleassign.txt" ;;
"subnet show capi-mgmt-subnet -f value -c cidr") cat "\$FIXDIR/capi-cidr.txt" ;;
"domain show testco -f value -c id") echo "$DOMID" ;;
"token issue -f value -c project_id") echo "$PID" ;;
"application credential show testco-svc-cred -f value -c id") cat "\$FIXDIR/appcred-id.txt" ;;
"keypair show testco-key -f value -c name") cat "\$FIXDIR/keypair-name.txt" ;;
"image list --public -f json") cat "\$FIXDIR/images.json" ;;
"security group create testco-canary-sg"*) : ;;
"security group rule create testco-canary-sg"*) : ;;
"server create testco-canary"*) : ;;
"server show testco-canary -f value -c status") cat "\$FIXDIR/status.txt" ;;
"floating ip create provider-ext -f value -c floating_ip_address") echo "10.99.0.5" ;;
"server add floating ip testco-canary 10.99.0.5") : ;;
"server delete --wait testco-canary") echo "deleted" >> "\$FIXDIR/teardown.log" ;;
"floating ip delete 10.99.0.5") echo "fip-del" >> "\$FIXDIR/teardown.log" ;;
"security group delete testco-canary-sg") echo "sg-del" >> "\$FIXDIR/teardown.log" ;;
*) echo "fake-openstack: unmatched: \$*" >&2; exit 1 ;;
esac
FB
chmod +x "$BIN/openstack"
cat > "$BIN/ssh" <<'FB'
#!/usr/bin/env bash
exit "$(cat "$FIXDIR/sshrc" 2>/dev/null || echo 1)"
FB
chmod +x "$BIN/ssh"
cat > "$BIN/sleep" <<'FB'
#!/usr/bin/env bash
exit 0
FB
chmod +x "$BIN/sleep"
mk_case_home(){ # mk_case_home <dir> [seed: full|noac|nokey|none]
local h="$1" seed="${2:-full}"
mkdir -p "$h"; cp "$H/admin-openrc" "$h/"; cp -r "$H/vault-init" "$h/"
local d="$h/tenant-testco"; mkdir -p "$d"; chmod 700 "$d"
printf 'username=testco-svc\nproject_id=%s\npassword=x\n' "$PID" > "$d/testco-svc-cred.txt"
printf 'username=testco-cluster\nproject_id=%s\npassword=x\n' "$PID" > "$d/testco-cluster-cred.txt"
printf 'domain=testco\ndomain_id=%s\nusername=testco-domain-admin\npassword=x\n' "$DOMID" > "$d/testco-domain-admin-cred.txt"
if [ "$seed" = "full" ] || [ "$seed" = "nokey" ]; then
printf 'id="%s"\nsecret="fake-secret-value"\n' "$ACID" > "$d/testco-svc-appcred.txt"
fi
if [ "$seed" = "full" ] || [ "$seed" = "noac" ]; then
printf -- '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n' > "$d/testco-key.pem"
fi
chmod 600 "$d"/* 2>/dev/null || true
}
run_stage(){ # run_stage <home> <fixdir> <stage> [cidr] -> OUT/RC
OUT=$( cd "$TMP" && env HOME="$1" FIXDIR="$2" PATH="$BIN:$PATH" \
TENANT_CIDR="${4:-10.20.24.0/24}" \
bash "$REPO/scripts/tenant-onboard.sh" testco "$3" 2>&1 ); RC=$?
}
check(){ # check <want_rc|-> <want:regex> <must_not:regex|-> <label>
local wrc="$1" want="$2" not="$3" label="$4"
local ok=1
[ "$wrc" = "-" ] || [ "$RC" = "$wrc" ] || ok=0
grep -qE "$want" <<<"$OUT" || ok=0
[ "$not" = "-" ] || { grep -qE "$not" <<<"$OUT" && ok=0; }
if [ "$ok" = 1 ]; then echo " PASS $label"; PASS=$((PASS+1))
else echo " FAIL $label (rc=$RC want=$wrc)"; echo "$OUT" | tail -5 | sed 's/^/ /'; FAIL=$((FAIL+1)); fi
}
# ---- stage4 CIDR guard (D-074: reserved-ranges-only, overlap math) ----
mkdir -p "$TMP/fx-collide" "$TMP/fx-clear" "$TMP/fx-capifail"
printf '10.20.7.0/24\n10.20.24.0/24\n10.20.9.0/24\n' > "$TMP/fx-collide/subnets.txt"
printf '10.20.7.0/24\n10.20.9.0/24\n' > "$TMP/fx-clear/subnets.txt"
printf '10.20.0.0/24\n' > "$TMP/fx-collide/capi-cidr.txt"
printf '10.20.0.0/24\n' > "$TMP/fx-clear/capi-cidr.txt"
printf 'No Subnet found for capi-mgmt-subnet\n' > "$TMP/fx-capifail/capi-cidr.txt"
# T1 (D-074): tenant-vs-tenant EXACT overlap is now ALLOWED -- another tenant
# already holds 10.20.24.0/24 in the fixture, and the guard must NOT die.
run_stage "$H" "$TMP/fx-collide" stage4
check - '== stage4:' 'rejected|in use' "T1 tenant-vs-tenant exact overlap ALLOWED (D-074)"
# T2: disjoint RFC1918 range passes (no false die)
run_stage "$H" "$TMP/fx-clear" stage4 "10.100.0.0/24"
check - '== stage4:' 'rejected|in use' "T2 disjoint CIDR passes the guard"
# T3: PARTIAL overlap against a reserved plane range -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "10.12.5.0/24"
check - 'rejected: RESERVED-OVERLAP 10.12.4.0/22' '-' "T3 partial overlap vs reserved plane rejected"
[ "$RC" -ne 0 ] || { echo " FAIL T3b guard must exit non-zero"; FAIL=$((FAIL+1)); }
# T4: EXACT match on a reserved plane range -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "10.12.4.0/22"
check - 'rejected: RESERVED-OVERLAP 10.12.4.0/22' '-' "T4 exact reserved match rejected"
# T5: partial overlap against the live-resolved capi-mgmt net -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "10.20.0.128/25"
check - 'rejected: RESERVED-OVERLAP 10.20.0.0/24' '-' "T5 capi-mgmt overlap rejected (dynamic lookup)"
# T6: non-RFC1918 (TEST-NET, which python is_private would wrongly accept) -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "192.0.2.0/24"
check - 'rejected: NOT-RFC1918' '-' "T6 non-RFC1918 rejected"
# T7: capi-mgmt lookup failure -> guard fails CLOSED
run_stage "$H" "$TMP/fx-capifail" stage4 "10.100.0.0/24"
check - 'capi-mgmt-subnet CIDR lookup failed' '-' "T7 capi lookup miss fails closed"
[ "$RC" -ne 0 ] || { echo " FAIL T7b fail-closed must exit non-zero"; FAIL=$((FAIL+1)); }
# ---- stage3 H2 re-run guards ----
# T3 both exist remotely + both local files intact -> double SKIP, rc0
mk_case_home "$TMP/h3" full; mkdir -p "$TMP/fx3"
echo "$ACID" > "$TMP/fx3/appcred-id.txt"
echo "testco-key" > "$TMP/fx3/keypair-name.txt"
run_stage "$TMP/h3" "$TMP/fx3" stage3
check 0 'app cred testco-svc-cred exists \+ local secret intact -- SKIP' '-' "T3a app-cred re-run SKIPs"
check 0 'keypair testco-key exists \+ local private key intact -- SKIP' '-' "T3b keypair re-run SKIPs"
# T4 app cred exists remotely, local secret file MISSING -> STOP rc1 with guidance
mk_case_home "$TMP/h4" noac; mkdir -p "$TMP/fx4"
echo "$ACID" > "$TMP/fx4/appcred-id.txt"
echo "testco-key" > "$TMP/fx4/keypair-name.txt"
run_stage "$TMP/h4" "$TMP/fx4" stage3
check 1 'OPERATOR DECISION required \(rotates a live credential\)' '-' "T4 lost-secret re-run STOPs with guidance"
# T5 keypair exists remotely, local key MISSING -> STOP rc1 with guidance
mk_case_home "$TMP/h5" nokey; mkdir -p "$TMP/fx5"
printf 'x\n' > "$TMP/fx5/appcred-id.txt" # app cred absent -> would create; but skip via intact file
echo "$ACID" > "$TMP/fx5/appcred-id.txt"
echo "testco-key" > "$TMP/fx5/keypair-name.txt"
run_stage "$TMP/h5" "$TMP/fx5" stage3
check 1 'OPERATOR DECISION required \(breaks existing node access\)' '-' "T5 lost-private-key re-run STOPs with guidance"
# ---- stage2 anti-escalation guard (capture-then-test; was a raced live pipe) ----
# T8a happy: no admin row -> anti-escalation OK, stage2 completes rc0
mk_case_home "$TMP/h8a" full; mkdir -p "$TMP/fx8a"
printf 'member\nload-balancer_member\n' > "$TMP/fx8a/roleassign.txt"
run_stage "$TMP/h8a" "$TMP/fx8a" stage2
check 0 'anti-escalation OK \(admin grant denied\)' '\*\*\* ESCALATION' "T8a stage2 happy: anti-escalation OK"
# T8b admin row present -> ESCALATION-STOP rc1 (pre-fix: SIGPIPE on the match
# could kill the pipe and print OK -- the guard failed OPEN)
mk_case_home "$TMP/h8b" full; mkdir -p "$TMP/fx8b"
printf 'admin\nmember\nload-balancer_member\n' > "$TMP/fx8b/roleassign.txt"
run_stage "$TMP/h8b" "$TMP/fx8b" stage2
check 1 '\*\*\* ESCALATION: manager granted admin -- STOP \*\*\*' 'anti-escalation OK' "T8b admin row triggers ESCALATION-STOP"
# T8c role-assignment list FAILS -> guard fails CLOSED (never prints OK)
mk_case_home "$TMP/h8c" full; mkdir -p "$TMP/fx8c"
printf 'member\n' > "$TMP/fx8c/roleassign.txt"; echo 1 > "$TMP/fx8c/roleassign-rc"
run_stage "$TMP/h8c" "$TMP/fx8c" stage2
check 1 'anti-escalation check UNVERIFIABLE .*HTTP 500' 'anti-escalation OK|\*\*\* ESCALATION: manager granted' "T8c failed list fails CLOSED, no OK"
# ---- stage7 H3 canary ----
mk_fix7(){ local d="$1"; mkdir -p "$d"
printf '[{"Name":"fedora-coreos-kube-1.28","ID":"11111111-2222-3333-4444-555555555555"}]\n' > "$d/images.json"
echo "ACTIVE" > "$d/status.txt"; : > "$d/teardown.log"
}
# T6 happy path: ACTIVE + ssh ok -> CANARY PASS, teardown ran, rc0
mk_case_home "$TMP/h6" full; mk_fix7 "$TMP/fx6"; echo 0 > "$TMP/fx6/sshrc"
run_stage "$TMP/h6" "$TMP/fx6" stage7
check 0 'CANARY PASS' '-' "T6a canary happy path passes"
check 0 'teardown done' '-' "T6b teardown ran on success"
grep -q 'deleted' "$TMP/fx6/teardown.log" && grep -q 'sg-del' "$TMP/fx6/teardown.log" \
&& { echo " PASS T6c teardown actually deleted server+SG"; PASS=$((PASS+1)); } \
|| { echo " FAIL T6c teardown log incomplete"; FAIL=$((FAIL+1)); }
# T7 ssh dead: CANARY FAIL rc1 AND teardown still ran (trap EXIT)
mk_case_home "$TMP/h7" full; mk_fix7 "$TMP/fx7"; echo 1 > "$TMP/fx7/sshrc"
run_stage "$TMP/h7" "$TMP/fx7" stage7
check 1 'CANARY FAIL: no SSH' '-' "T7a dead SSH fails the canary"
check 1 'teardown done' '-' "T7b teardown still ran on failure"
grep -q 'deleted' "$TMP/fx7/teardown.log" \
&& { echo " PASS T7c server deleted despite failure"; PASS=$((PASS+1)); } \
|| { echo " FAIL T7c no server delete on failure path"; FAIL=$((FAIL+1)); }
echo; echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1