#!/usr/bin/env bash
# tests/tenant-offboard/run-tests.sh -- mock harness for scripts/tenant-offboard.sh
# Scenarios: blocklist=20, missing-domain=20, audit=0 (ZERO mutations), tty-mismatch=24,
#            apply-happy=0 (+ Phase E0 explicit app-cred inventory line, DOCFIX-100),
#            cluster-DELETE_FAILED=21, sweep-failure-injection=22 (DOCFIX-082),
#            e0-skip: re-run against already-cleaned tenant SKIPs Phase E0 cleanly (rc 0),
#            --sweep-magnum-orphans (DOCFIX-099): audit classification (ORPHAN /
#            PROTECTED / UNPROVEN, zero mutations), UNPROVEN fail-closed (alive project
#            + cluster-show 404 is NOT an orphan), confirmation-mismatch=24 with zero
#            mutations, gated apply deletes ONLY the proven orphan (+its trust), and an
#            idempotent re-run is a clean no-op (rc 0). Requires no cloud access.
set -uo pipefail
SD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; SCRIPT="$SD/../../scripts/tenant-offboard.sh"
W=$(mktemp -d); trap 'rm -rf "$W"' EXIT
mkdir -p "$W/bin" "$W/vault-init"; touch "$W/admin-openrc" "$W/vault-init/vault-ca-root.pem"
printf 'username=acme-svc\nproject_id=%s\npassword=pw\nauth_url=https://10.77.77.77:5000/v3\n' "$(python3 -c "print('a'*32)")" > "$W/acme-svc-cred.txt"
printf '#!/usr/bin/env bash\nexit 0\n' > "$W/bin/sleep"
cat > "$W/bin/openstack" <<'OSMOCK'
#!/usr/bin/env bash
S="${MOCK_SCEN:-happy}"; ST="${MOCK_STATE:-/home/claude/mock6/state}"; mkdir -p "$ST"
mut(){ echo "$*" >> "$ST/mutations"; }
# R10 observability: record the auth URL every tenant-scoped (coe) call runs under
case "$*" in "coe cluster"*) echo "${OS_AUTH_URL:-unset}" >> "$ST/authurl";; esac
# warnstderr scenario: truthful CLI -- list commands emit a deprecation-style
# notice ON STDERR (exercises the stdout-only inventory captures). coe excluded:
# the Phase A cluster capture still merges stderr (known, out of this batch).
if [ "$S" = warnstderr ]; then case "$*" in "coe "*) : ;; *" list"*) echo "WARNING: a deprecated-option notice (fake, stderr)" >&2 ;; esac; fi
DOM=$(python3 -c "print('d'*32)"); U1=$(python3 -c "print('1'*32)"); U2=$(python3 -c "print('2'*32)")
P1=$(python3 -c "print('a'*32)")
# neutron/octavia/nova resources are dashed-uuid shaped (truthful to the real CLI)
LBU="aaaa1111-1111-4111-8111-111111111111"
FIPU="bbbb2222-2222-4222-8222-222222222222"
RTU="cccc3333-3333-4333-8333-333333333333"
SNU="dddd4444-4444-4444-8444-444444444444"
NWU="eeee5555-5555-4555-8555-555555555555"
# sweep-mode fixtures: magnum trustee domain population
MDOM=$(python3 -c "print('e'*32)"); MADM=$(python3 -c "print('9'*32)")
UO=$(python3 -c "print('4'*32)"); UP=$(python3 -c "print('5'*32)")
UU=$(python3 -c "print('6'*32)"); UV=$(python3 -c "print('7'*32)"); UW=$(python3 -c "print('8'*32)")
PGONE=$(python3 -c "print('f'*32)"); PB=$(python3 -c "print('b'*32)")
NO="11111111-1111-4111-8111-111111111111_${PGONE}"   # orphan: project gone
NP="22222222-2222-4222-8222-222222222222_${P1}"      # protected: project+cluster alive
NU="33333333-3333-4333-8333-333333333333_${PB}"      # unproven: project lookup ambiguous
NV="44444444-4444-4444-8444-444444444444_${P1}"      # unproven: project alive, cluster 404
case "$*" in
  "domain show acme"*)
    [ -f "$ST/domgone" ] && { echo "No domain with a name or ID of 'acme' exists."; exit 1; }
    echo "$DOM";;
  "domain show magnum"*) echo "$MDOM";;
  "domain show "*) echo "No domain found"; exit 1;;
  "user show magnum_domain_admin"*) echo "$MADM";;
  "user show $UO"*)
    [ -f "$ST/orphandel" ] && { echo "No user with a name or ID of '$UO' exists."; exit 1; }
    echo "$UO";;
  "user list --domain $MDOM"*)
    printf '%s magnum_domain_admin\n' "$MADM"
    [ -f "$ST/orphandel" ] || printf '%s %s\n' "$UO" "$NO"
    printf '%s %s\n%s %s\n%s %s\n%s capi-helper\n' "$UP" "$NP" "$UU" "$NU" "$UV" "$NV" "$UW";;
  "user list --domain"*)
    [ "$S" = cleaned ] && exit 0
    printf '%s acme-svc\n%s acme-domain-admin\n' "$U1" "$U2";;
  "project list --domain"*)
    [ "$S" = cleaned ] && exit 0
    printf '%s acme-prod\n' "$P1";;
  "project show $P1"*) echo "$P1";;
  "project show $PGONE"*) echo "No project with a name or ID of '$PGONE' exists."; exit 1;;
  "project show $PB"*) echo "Gateway Timeout (HTTP 504)"; exit 1;;
  "coe cluster show 22222222-2222-4222-8222-222222222222"*) echo "22222222-2222-4222-8222-222222222222";;
  "coe cluster show "*) echo "Cluster 44444444-4444-4444-8444-444444444444 could not be found (HTTP 404)"; exit 1;;
  "coe cluster list"*)
    [ "$S" = cleaned ] && exit 0
    if [ -f "$ST/cdel" ]; then
      N=$(cat "$ST/cn" 2>/dev/null || echo 0); N=$((N+1)); echo "$N" > "$ST/cn"
      if [ "$S" = delfail ]; then echo "cu1 acme-cluster DELETE_FAILED"
      elif [ "$N" -le 2 ]; then echo "cu1 acme-cluster DELETE_IN_PROGRESS"; fi
    else echo "cu1 acme-cluster CREATE_FAILED"; fi;;
  "coe cluster delete"*) mut "$*"; touch "$ST/cdel";;
  "trust list"*)
    if [ "$S" = sweep ]; then
      if [ -f "$ST/strdel" ] || [ -f "$ST/orphandel" ]; then echo "[]"
      else printf '[{"ID":"str1","Trustor User ID":"tor","Trustee User ID":"%s"}]\n' "$UO"; fi
    elif [ -f "$ST/tdel" ]; then echo "[]"
    else printf '[{"ID":"tr1","Trustor User ID":"%s","Trustee User ID":"tee"}]\n' "$U1"; fi;;
  "trust show tr1"*) [ -f "$ST/tdel" ] && exit 1; echo "tr1";;
  "trust delete str1"*) mut "$*"; touch "$ST/strdel";;
  "trust delete"*)
    if [ "$S" = trustdelfail ]; then echo "Unable to delete trust (HTTP 500)"; exit 1; fi
    mut "$*"; touch "$ST/tdel";;
  "application credential list --user $U1"*) python3 -c "print('c'*32)";;
  "application credential list --user"*) : ;;
  "loadbalancer list --project "*"-c provisioning_status") [ -f "$ST/lbdel" ] || echo "$LBU old-lb ERROR";;
  "loadbalancer list --project "*"-c id")
    if [ "$S" = lbstuck ]; then echo "$LBU"; else [ -f "$ST/lbdel" ] || echo "$LBU"; fi;;
  "loadbalancer delete"*) mut "$*"; touch "$ST/lbdel";;
  "floating ip list --project "*"Floating IP Address") [ -f "$ST/fipdel" ] || echo "$FIPU 10.12.5.99";;
  "floating ip list --project "*"-c ID") [ -f "$ST/fipdel" ] || echo "$FIPU";;
  "floating ip delete"*) mut "$*"; touch "$ST/fipdel";;
  "server list"*) : ;;
  "router list --project "*"-c Name") [ -f "$ST/rdel" ] || echo "$RTU acme-router";;
  "router list --project "*"-c ID") [ -f "$ST/rdel" ] || echo "$RTU";;
  "router remove subnet"*|"router unset"*) mut "$*";;
  "router delete"*) mut "$*"; touch "$ST/rdel";;
  "subnet list --project "*"-c Name") [ -f "$ST/sndel" ] || echo "$SNU acme-subnet";;
  "subnet list --project "*"-c ID") [ -f "$ST/sndel" ] || echo "$SNU";;
  "subnet delete"*) mut "$*"; touch "$ST/sndel";;
  "network list --project "*"-c Name") [ -f "$ST/nwdel" ] || echo "$NWU acme-net";;
  "network list --project "*"-c ID")
    if [ "$S" = garbagerow ]; then echo "please retry later"
    else [ -f "$ST/nwdel" ] || echo "$NWU"; fi;;
  "network delete"*)
    if [ "${MOCK_SCEN:-}" = sweepfail ]; then echo "Conflict: one or more ports still in use (HTTP 409)"; exit 1; fi
    mut "$*"; touch "$ST/nwdel";;
  "user delete $UO"*) mut "$*"; touch "$ST/orphandel";;
  "user delete"*|"project delete"*) mut "$*";;
  "domain set --disable"*) mut "$*";;
  "domain delete"*) mut "$*"; touch "$ST/domgone";;
esac
OSMOCK
cat > "$W/ptyrun.py" <<'PTY'
import pty, os, sys
cmd = sys.argv[1:-1]; feed = sys.argv[-1]
pid, fd = pty.fork()
if pid == 0:
    os.execvp(cmd[0], cmd)
os.write(fd, (feed + "\n").encode())
out = b""
while True:
    try:
        d = os.read(fd, 4096)
    except OSError:
        break
    if not d:
        break
    out += d
_, st = os.waitpid(pid, 0)
sys.stdout.write(out.decode(errors="replace"))
sys.exit(os.waitstatus_to_exitcode(st))
PTY
chmod +x "$W/bin/"*
P=0; F=0
chk(){ if [ "$2" = "$3" ]; then echo "PASS: $1 (exit $2)"; P=$((P+1)); else echo "FAIL: $1 (exit $2, want $3)"; F=$((F+1)); fi; }
export MOCK_STATE="$W/state"
UO32=$(python3 -c "print('4'*32)")
HOME="$W" PATH="$W/bin:$PATH" bash "$SCRIPT" magnum --apply >/dev/null 2>&1; chk blocklist $? 20
rm -rf "$W/state"; HOME="$W" PATH="$W/bin:$PATH" bash "$SCRIPT" ghost >/dev/null 2>&1; chk missing-domain $? 20
rm -rf "$W/state"; HOME="$W" PATH="$W/bin:$PATH" bash "$SCRIPT" acme --audit >/dev/null 2>&1; RC=$?
MUT=$(cat "$W/state/mutations" 2>/dev/null | wc -l)
if [ "$RC" = 0 ] && [ "$MUT" = 0 ]; then echo "PASS: audit (exit 0, 0 mutations)"; P=$((P+1)); else echo "FAIL: audit (exit $RC, mutations $MUT)"; F=$((F+1)); fi
# R10: tenant_env must take OS_AUTH_URL from the cred file's auth_url= line
if grep -q 'https://10.77.77.77:5000/v3' "$W/state/authurl" 2>/dev/null; then
  echo "PASS: auth-url-from-cred (tenant calls used cred auth_url)"; P=$((P+1))
else echo "FAIL: auth-url-from-cred (authurl: $(cat "$W/state/authurl" 2>/dev/null | sort -u | tr '\n' ';'))"; F=$((F+1)); fi
# R10 fallback: cred without auth_url= -> KEYSTONE_VIP-derived URL
W2=$(mktemp -d); mkdir -p "$W2/vault-init"; touch "$W2/admin-openrc" "$W2/vault-init/vault-ca-root.pem"
printf 'username=acme-svc\nproject_id=%s\npassword=pw\n' "$(python3 -c "print('a'*32)")" > "$W2/acme-svc-cred.txt"
rm -rf "$W/state"; HOME="$W2" KEYSTONE_VIP=10.55.55.55 PATH="$W/bin:$PATH" bash "$SCRIPT" acme --audit >/dev/null 2>&1; RC=$?
if [ "$RC" = 0 ] && grep -q 'https://10.55.55.55:5000/v3' "$W/state/authurl" 2>/dev/null; then
  echo "PASS: auth-url-fallback (KEYSTONE_VIP override honored)"; P=$((P+1))
else echo "FAIL: auth-url-fallback (exit $RC, authurl: $(cat "$W/state/authurl" 2>/dev/null | sort -u | tr '\n' ';'))"; F=$((F+1)); fi
rm -rf "$W2"
rm -rf "$W/state"; HOME="$W" PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply WRONG >/dev/null 2>&1; chk tty-mismatch $? 24
rm -rf "$W/state"; HOME="$W" PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/happy.out" 2>&1; chk apply-happy $? 0
# DOCFIX-100 Phase E0: apply must EXPLICITLY inventory the app cred it removes via cascade
if grep -q "app cred $(python3 -c "print('c'*32)") (owner $(python3 -c "print('1'*32)")) -- removed via owner delete (cascade)" "$W/happy.out"; then
  echo "PASS: apply-e0-appcred-explicit (cascade inventory line present)"; P=$((P+1))
else echo "FAIL: apply-e0-appcred-explicit (no cascade inventory line)"; F=$((F+1)); fi
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=delfail PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme >/dev/null 2>&1; chk delete-failed $? 21
# DOCFIX-082 failure injection: a mid-sweep resource-delete failure must surface as exit 22,
# never as silent success (the pre-082 script lost mutation exit codes in display pipelines).
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=sweepfail PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/sweepfail.out" 2>&1; RC=$?
if [ "$RC" = 22 ] && grep -q '\^ FAILED: openstack network delete' "$W/sweepfail.out"; then
  echo "PASS: sweep-failure-injection (exit 22, failure line printed)"; P=$((P+1))
else echo "FAIL: sweep-failure-injection (exit $RC, want 22 + FAILED line)"; F=$((F+1)); fi
# DOCFIX-100 Phase E0 re-run guard: already-cleaned tenant (users gone, domain still
# present -- e.g. a prior apply died at the domain step) must SKIP the app-cred stage
# cleanly and complete with rc 0, never error.
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=cleaned PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/e0.out" 2>&1; RC=$?
if [ "$RC" = 0 ] && grep -q "app-cred stage: no users remain in domain -- SKIP (idempotent re-run)" "$W/e0.out"; then
  echo "PASS: e0-skip-idempotent (exit 0, SKIP line)"; P=$((P+1))
else echo "FAIL: e0-skip-idempotent (exit $RC; want 0 + SKIP line)"; F=$((F+1)); fi
# ---- stderr-separation + row-validation hardening ----
# stderr warnings on list commands must be SURFACED as NOTEs, never become
# delete argv (pre-fix: merged 2>&1 word-split "WARNING: ..." into the loops).
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=warnstderr PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/warn.out" 2>&1; RC=$?
if [ "$RC" = 0 ] && ! grep -qiE 'delete.*(WARNING|deprecated)' "$W/state/mutations" 2>/dev/null \
   && grep -q 'NOTE: .*stderr: WARNING' "$W/warn.out"; then
  echo "PASS: stderr-not-argv (exit 0, warnings surfaced as NOTE, none deleted)"; P=$((P+1))
else echo "FAIL: stderr-not-argv (exit $RC; muts: $(grep -icE 'WARNING|deprecated' "$W/state/mutations" 2>/dev/null))"; F=$((F+1)); fi
# a non-id stdout row (API error prose) must be SKIPped and counted -> exit 22
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=garbagerow PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/garb.out" 2>&1; RC=$?
if [ "$RC" = 22 ] && grep -q 'SKIP non-id network row' "$W/garb.out" && ! grep -q 'network delete please' "$W/state/mutations" 2>/dev/null; then
  echo "PASS: non-id-row-skipped (exit 22, row never deleted)"; P=$((P+1))
else echo "FAIL: non-id-row-skipped (exit $RC)"; F=$((F+1)); fi
# LB wait-budget exhaustion must FAIL the phase loudly (exit 22), not fall through silently
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=lbstuck PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/lbstuck.out" 2>&1; RC=$?
if [ "$RC" = 22 ] && grep -q 'LBs not confirmed gone in budget' "$W/lbstuck.out"; then
  echo "PASS: lb-wait-budget-fails-loud (exit 22)"; P=$((P+1))
else echo "FAIL: lb-wait-budget-fails-loud (exit $RC)"; F=$((F+1)); fi
# Phase B counter: a failed residual-trust delete must surface as exit 22
# (pre-fix: the printf|while subshell ate the SWEEP_FAIL increment -> exit 0)
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=trustdelfail PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" acme --apply acme > "$W/tdf.out" 2>&1; RC=$?
if [ "$RC" = 22 ] && grep -q '\^ FAILED: openstack trust delete' "$W/tdf.out"; then
  echo "PASS: phaseB-counter-survives (exit 22, failed trust delete counted)"; P=$((P+1))
else echo "FAIL: phaseB-counter-survives (exit $RC, want 22)"; F=$((F+1)); fi
# ---- DOCFIX-099 --sweep-magnum-orphans ----
# audit: read-only classification -- exactly 1 ORPHAN, 2 PROTECTED (domain admin +
# alive cluster), 3 UNPROVEN (ambiguous project, alive-project/404-cluster, odd name);
# ZERO mutations.
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=sweep PATH="$W/bin:$PATH" bash "$SCRIPT" --sweep-magnum-orphans --audit > "$W/sw-audit.out" 2>&1; RC=$?
MUT=$(cat "$W/state/mutations" 2>/dev/null | wc -l)
NORP=$(grep -c '^ORPHAN' "$W/sw-audit.out"); NPRO=$(grep -c '^PROTECTED' "$W/sw-audit.out"); NUNP=$(grep -c '^UNPROVEN' "$W/sw-audit.out")
if [ "$RC" = 0 ] && [ "$MUT" = 0 ] && [ "$NORP" = 1 ] && [ "$NPRO" = 2 ] && [ "$NUNP" = 3 ] && grep -q '^ORPHAN.*11111111-1111' "$W/sw-audit.out"; then
  echo "PASS: sweep-audit (exit 0, 0 mutations, ORPHAN=1 PROTECTED=2 UNPROVEN=3)"; P=$((P+1))
else echo "FAIL: sweep-audit (exit $RC, mut $MUT, O=$NORP P=$NPRO U=$NUNP)"; F=$((F+1)); fi
# fail-closed: alive project + cluster-show 404 must classify UNPROVEN, never ORPHAN
if grep '^UNPROVEN' "$W/sw-audit.out" | grep -q '44444444-4444' && ! grep '^ORPHAN' "$W/sw-audit.out" | grep -q '44444444-4444'; then
  echo "PASS: sweep-unproven-failclosed (404-cluster stays UNPROVEN)"; P=$((P+1))
else echo "FAIL: sweep-unproven-failclosed"; F=$((F+1)); fi
# gated apply: confirmation mismatch refuses with 24 and zero mutations
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=sweep PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" --sweep-magnum-orphans --apply WRONG >/dev/null 2>&1; RC=$?
MUT=$(cat "$W/state/mutations" 2>/dev/null | wc -l)
if [ "$RC" = 24 ] && [ "$MUT" = 0 ]; then echo "PASS: sweep-confirm-mismatch (exit 24, 0 mutations)"; P=$((P+1))
else echo "FAIL: sweep-confirm-mismatch (exit $RC, mut $MUT)"; F=$((F+1)); fi
# gated apply: deletes ONLY the proven orphan (its trust + its user), nothing else
rm -rf "$W/state"; HOME="$W" MOCK_SCEN=sweep PATH="$W/bin:$PATH" python3 "$W/ptyrun.py" bash "$SCRIPT" --sweep-magnum-orphans --apply magnum-orphans > "$W/sw-apply.out" 2>&1; RC=$?
MUTF="$W/state/mutations"; NMUT=$(cat "$MUTF" 2>/dev/null | wc -l); NUDEL=$(grep -c 'user delete' "$MUTF" 2>/dev/null)
if [ "$RC" = 0 ] && [ "$NMUT" = 2 ] && [ "$NUDEL" = 1 ] && grep -q "user delete $UO32" "$MUTF" && grep -q "trust delete str1" "$MUTF"; then
  echo "PASS: sweep-apply-gated (exit 0; exactly orphan user + its trust deleted)"; P=$((P+1))
else echo "FAIL: sweep-apply-gated (exit $RC, mutations: $(cat "$MUTF" 2>/dev/null | tr '\n' ';'))"; F=$((F+1)); fi
# idempotent re-run: same state, orphan already gone -> clean no-op rc 0, no new mutations
HOME="$W" MOCK_SCEN=sweep PATH="$W/bin:$PATH" bash "$SCRIPT" --sweep-magnum-orphans --apply > "$W/sw-rerun.out" 2>&1; RC=$?
NMUT2=$(cat "$MUTF" 2>/dev/null | wc -l)
if [ "$RC" = 0 ] && grep -q "NOTHING PROVEN ORPHANED" "$W/sw-rerun.out" && [ "$NMUT2" = "$NMUT" ]; then
  echo "PASS: sweep-rerun-idempotent (exit 0, no-op)"; P=$((P+1))
else echo "FAIL: sweep-rerun-idempotent (exit $RC, mut $NMUT->$NMUT2)"; F=$((F+1)); fi
echo; T=$((P+F)); [ "$F" = 0 ] && { echo "ALL PASS ($P/$T)"; exit 0; } || { echo "FAILURES: $F ($P/$T)"; exit 1; }