#!/usr/bin/env bash
# tests/checks/run-tests.sh -- unit tests for scripts/checks/d011-*.sh (grows per batch).
# Batch 1: d011-01-charms (mock juju), d011-06-vault-unseal (mock ledger).
# Also an INTEGRATION assertion: run both via the real orchestrator against mocks.
set -u
SD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; REPO="$(cd "$SD/../.." && pwd)"
CHK="$REPO/scripts/checks"; VAL="$REPO/scripts/validate.sh"
P=0; F=0; ok(){ echo "PASS: $1"; P=$((P+1)); }; no(){ echo "FAIL: $1"; F=$((F+1)); }
chk(){ [ "$2" = "$3" ] && ok "$1" || no "$1 (got '$2' want '$3')"; }
W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT; mkdir -p "$W/bin"

# --- mock juju (scenario via MOCK_JUJU) ---
cat > "$W/bin/juju" <<'JM'
#!/usr/bin/env bash
case "${MOCK_JUJU:-ok}" in
  unreachable) echo "ERROR connection refused" >&2; exit 1 ;;
  notjson)     echo "not json at all" ;;
  blocked)     cat <<'J'
{"applications":{"nova":{"units":{"nova/0":{"workload-status":{"current":"blocked"},"juju-status":{"current":"idle"}}}}}}
J
  ;;
  gss)         cat <<'J'
{"applications":{"gss":{"units":{"gss/0":{"workload-status":{"current":"unknown"},"juju-status":{"current":"idle"}}}},
  "vault":{"units":{"vault/0":{"workload-status":{"current":"active"},"juju-status":{"current":"idle"},
   "subordinates":{"vault-mysql-router/0":{"workload-status":{"current":"active"},"juju-status":{"current":"idle"}}}}}}}}
J
  ;;
  *)           cat <<'J'
{"applications":{"vault":{"units":{"vault/0":{"workload-status":{"current":"active"},"juju-status":{"current":"idle"}}}}}}
J
  ;;
esac
JM
chmod +x "$W/bin/juju"; command -v jq >/dev/null || { echo "SKIP: jq absent"; exit 0; }

runchk(){ PATH="$W/bin:$PATH" bash "$CHK/d011-01-charms.sh" 2>&1; }

# d011-01-charms
OUT="$(MOCK_JUJU=ok runchk)"; chk "charms all-active PASS" "$?" 0
grep -qE '^RESULT d011-01-charms PASS 0 ' <<<"$OUT" && ok "charms PASS line" || no "charms PASS line"
OUT="$(MOCK_JUJU=blocked runchk)"; chk "charms blocked FAIL" "$?" 1
grep -q 'nova/0 workload=blocked' <<<"$OUT" && ok "charms names the bad unit" || no "charms names bad unit"
OUT="$(MOCK_JUJU=gss runchk)"; chk "charms gss-unknown tolerated PASS" "$?" 0
grep -q 'tolerate: gss/0' <<<"$OUT" && ok "charms tolerates gss" || no "charms tolerates gss"
OUT="$(MOCK_JUJU=unreachable runchk)"; chk "charms unreachable HOLD" "$?" 2
OUT="$(MOCK_JUJU=notjson runchk)"; chk "charms notjson HOLD" "$?" 2

# d011-06-vault-unseal (mock ledger via VR_LEDGER)
mkl="$(mktemp)"
mkledger(){ printf '| id | date | item | src | owner | status |\n|---|---|---|---|---|---|\n| SEC-003 | 2026-07-03 | unseal custody | D-069 | operator | %s |\n' "$1" > "$mkl"; }
runvault(){ VR_LEDGER="$mkl" bash "$CHK/d011-06-vault-unseal.sh" 2>&1; }
mkledger "OPEN -- assign custodians + rehearse"; OUT="$(runvault)"; chk "vault OPEN -> MANUAL(3)" "$?" 3
grep -qE '^RESULT d011-06-vault-unseal PASS_PENDING_MANUAL 3 ' <<<"$OUT" && ok "vault MANUAL line" || no "vault MANUAL line"
mkledger "CLOSED 2026-07-10 -- rehearsed by A.Jones (second person)"; OUT="$(runvault)"; chk "vault CLOSED -> PASS(0)" "$?" 0
mkledger "REHEARSED 2026-07-10"; OUT="$(runvault)"; chk "vault REHEARSED -> PASS(0)" "$?" 0
mkledger "banana"; OUT="$(runvault)"; chk "vault unknown-status -> HOLD(2)" "$?" 2
printf 'no sec-003 row here\n' > "$mkl"; OUT="$(runvault)"; chk "vault missing-row -> HOLD(2)" "$?" 2
OUT="$(VR_LEDGER=/nonexistent bash "$CHK/d011-06-vault-unseal.sh" 2>&1)"; chk "vault missing-file -> HOLD(2)" "$?" 2

# INTEGRATION: real orchestrator runs both checks against mocks, from the REAL checks dir
# (checks resolve lib-validate.sh relative to their own location, so they must run from
# scripts/checks/ with the library sibling present -- do NOT copy them to an isolated temp).
mkledger "OPEN -- outstanding"
OUT="$(MOCK_JUJU=ok VR_LEDGER="$mkl" VR_CHECKDIR="$CHK" PATH="$W/bin:$PATH" bash "$VAL" --checks d011-01-charms,d011-06-vault-unseal 2>&1)"; RC=$?
chk "orchestrator overall = PASS_PENDING_MANUAL(3)" "$RC" 3
grep -q 'MANUAL=1' <<<"$OUT" && ok "orchestrator counts manual" || no "orchestrator counts manual"
grep -q 'PASS=1' <<<"$OUT" && ok "orchestrator counts charms pass" || no "orchestrator counts charms pass"


# ============================ BATCH 2 ============================
command -v jq >/dev/null || { echo "SKIP batch2: jq absent"; echo; [ "$F" = 0 ] && { echo "ALL PASS ($P checks)"; exit 0; } || { echo "FAILURES: $F"; exit 1; }; }

# ---- d011-02-vip-jumphost: mock openstack (endpoint list) + mock curl ----
B2="$(mktemp -d)"; mkdir -p "$B2/bin" "$B2/home/vault-init"
printf 'export OS_AUTH_URL=https://ks:5000/v3\nexport OS_CACERT=%s/home/vault-init/ca.pem\nexport OS_USERNAME=admin\nexport OS_PASSWORD=x\nexport OS_PROJECT_NAME=admin\n' "$B2" > "$B2/home/admin-openrc"
touch "$B2/home/vault-init/ca.pem"
cat > "$B2/bin/openstack" <<'OM'
#!/usr/bin/env bash
case "$*" in
  *"endpoint list"*"--interface public"*)
    if [ "${MOCK_EP:-ok}" = empty ]; then echo "[]"; else
    cat <<'J'
[{"Service Name":"keystone","URL":"https://ks.corp:5000/v3","Interface":"public"},
 {"Service Name":"nova","URL":"https://nova.corp:8774/v2.1","Interface":"public"}]
J
    fi ;;
esac
OM
cat > "$B2/bin/curl" <<'CM'
#!/usr/bin/env bash
k=0; url=""
for a in "$@"; do [ "$a" = "-k" ] && k=1; case "$a" in https://*) url="$a";; esac; done
case "${MOCK_CURL:-healthy}" in
  healthy)     echo 300; exit 0 ;;
  fivexx)      echo 500; exit 0 ;;
  unreachable) exit 7 ;;                                  # both verified and -k fail
  tlsbroken)   if [ "$k" = 1 ]; then echo 200; exit 0; else exit 60; fi ;;
esac
CM
chmod +x "$B2/bin/openstack" "$B2/bin/curl"
runvip2(){ HOME="$B2/home" PATH="$B2/bin:$PATH" bash "$CHK/d011-02-vip-jumphost.sh" 2>&1; }

OUT="$(MOCK_CURL=healthy runvip2)"; chk "vip-jumphost all-healthy PASS" "$?" 0
grep -qE '^RESULT d011-02-vip-jumphost PASS 0 ' <<<"$OUT" && ok "vip2 PASS line" || no "vip2 PASS line"
grep -q 'OK  https://ks.corp:5000' <<<"$OUT" && ok "vip2 lists healthy origin" || no "vip2 lists healthy origin"
OUT="$(MOCK_CURL=fivexx runvip2)"; chk "vip-jumphost 5xx FAIL" "$?" 1
grep -q 'reachable but 5xx' <<<"$OUT" && ok "vip2 flags 5xx" || no "vip2 flags 5xx"
OUT="$(MOCK_CURL=unreachable runvip2)"; chk "vip-jumphost unreachable FAIL" "$?" 1
grep -q 'UNREACHABLE' <<<"$OUT" && ok "vip2 classifies unreachable" || no "vip2 classifies unreachable"
OUT="$(MOCK_CURL=tlsbroken runvip2)"; chk "vip-jumphost tls-broken FAIL" "$?" 1
grep -q 'TLS-verify-FAILED' <<<"$OUT" && ok "vip2 classifies tls-broken (reachable w/-k)" || no "vip2 classifies tls-broken"
OUT="$(MOCK_EP=empty MOCK_CURL=healthy runvip2)"; chk "vip-jumphost no-endpoints HOLD" "$?" 2
OUT="$(HOME=/nonexistent PATH="$B2/bin:$PATH" bash "$CHK/d011-02-vip-jumphost.sh" 2>&1)"; chk "vip-jumphost no-scope HOLD" "$?" 2

# ---- d011-03-vip-tenant: mock kubectl + openstack + tenant cred/kubeconfig ----
B3="$(mktemp -d)"; mkdir -p "$B3/bin" "$B3/home/tenant-beta/kube" "$B3/home/vault-init"
touch "$B3/home/vault-init/vault-ca-root.pem"
printf 'auth_url=https://10.12.4.50:5000/v3\nusername=beta-cluster\nuser_domain_id=%s\nproject_id=%s\npassword=pw\n' "$(python3 -c 'print("b"*32)')" "$(python3 -c 'print("a"*32)')" > "$B3/home/tenant-beta/beta-cluster-cred.txt"
printf 'apiVersion: v1\nkind: Config\n' > "$B3/home/tenant-beta/kube/config"
cat > "$B3/bin/kubectl" <<'KM'
#!/usr/bin/env bash
case "$*" in
  "version --client"*) exit 0 ;;
  "run "*)   echo "pod/${2} created" ;;
  "get pod"*"jsonpath={.status.phase}"*)
    case "${MOCK_POD:-succeeded}" in succeeded) echo Succeeded;; blocked) echo Failed;; hang) echo Running;; esac ;;
  "logs "*)
    case "${MOCK_POD:-succeeded}" in blocked) echo "TIMEOUT";; succeeded) echo "CONNECTED to 10.12.4.50:5000";; *) echo "";; esac ;;
  "delete pod"*) : ;;
esac
KM
cat > "$B3/bin/openstack" <<'OM'
#!/usr/bin/env bash
case "$*" in
  *"coe cluster config"*)
    # write a fake kubeconfig into --dir
    d=""; prev=""; for a in "$@"; do [ "$prev" = "--dir" ] && d="$a"; prev="$a"; done
    [ -n "$d" ] && printf 'apiVersion: v1\nkind: Config\n' > "$d/config"; echo "config written" ;;
esac
OM
chmod +x "$B3/bin/kubectl" "$B3/bin/openstack"
runvip3(){ HOME="$B3/home" PATH="$B3/bin:$PATH" bash "$CHK/d011-03-vip-tenant.sh" 2>&1; }

OUT="$(MOCK_POD=succeeded runvip3)"; chk "vip-tenant reachable PASS" "$?" 0
grep -qE '^RESULT d011-03-vip-tenant PASS 0 ' <<<"$OUT" && ok "vip3 PASS line" || no "vip3 PASS line"
grep -q 'target keystone VIP: 10.12.4.50:5000' <<<"$OUT" && ok "vip3 derives VIP from auth_url" || no "vip3 derives VIP"
OUT="$(MOCK_POD=blocked runvip3)"; chk "vip-tenant blocked FAIL" "$?" 1
grep -q 'BLOCKED' <<<"$OUT" && ok "vip3 reports blocked" || no "vip3 reports blocked"
# fetch path: remove cached kubeconfig, ensure it fetches then passes
rm -f "$B3/home/tenant-beta/kube/config"
OUT="$(MOCK_POD=succeeded runvip3)"; chk "vip-tenant fetch-kubeconfig PASS" "$?" 0
grep -q 'fetching via coe cluster config' <<<"$OUT" && ok "vip3 fetch path exercised" || no "vip3 fetch path"
# no tenant cred -> HOLD
B3b="$(mktemp -d)"; mkdir -p "$B3b/home"
OUT="$(HOME="$B3b/home" PATH="$B3/bin:$PATH" bash "$CHK/d011-03-vip-tenant.sh" 2>&1)"; chk "vip-tenant no-cred HOLD" "$?" 2

# ---- INTEGRATION: post-restart profile now has 3 real checks (01 pass, 02/03 mocked) ----
# (02/03 need their own env; just assert orchestrator resolves & runs 01 with the others reported)
OUT="$(VR_CHECKDIR="$CHK" bash "$VAL" --list 2>&1)"
grep -q 'd011-02-vip-jumphost' <<<"$OUT" && ok "orchestrator discovers d011-02" || no "orchestrator discovers d011-02"
grep -q 'd011-03-vip-tenant' <<<"$OUT" && ok "orchestrator discovers d011-03" || no "orchestrator discovers d011-03"


# ============================ BATCH 3 ============================

# ---- d011-05-magnum-e2e: mock tenant-acceptance.sh returning each code ----
B5="$(mktemp -d)"; mkdir -p "$B5/bin" "$B5/scripts/checks"
cp "$CHK/../lib-validate.sh" "$B5/scripts/lib-validate.sh"
cp "$CHK/d011-05-magnum-e2e.sh" "$B5/scripts/checks/"
mkta(){ printf '#!/usr/bin/env bash\n%s\nexit %s\n' "$1" "$2" > "$B5/scripts/tenant-acceptance.sh"; chmod +x "$B5/scripts/tenant-acceptance.sh"; }
printf '#!/usr/bin/env bash\nexit 0\n' > "$B5/bin/kubectl"; printf '#!/usr/bin/env bash\nexit 0\n' > "$B5/bin/openstack"; chmod +x "$B5/bin/"*
run05(){ PATH="$B5/bin:$PATH" bash "$B5/scripts/checks/d011-05-magnum-e2e.sh" 2>&1; }
mkta 'echo P0-3 all good' 0;  OUT="$(run05)"; chk "magnum-e2e PASS" "$?" 0
grep -qE '^RESULT d011-05-magnum-e2e PASS 0 ' <<<"$OUT" && ok "e2e PASS line+timing" || no "e2e PASS line"
mkta 'echo kube broke' 11;    OUT="$(run05)"; chk "magnum-e2e kube FAIL" "$?" 1
mkta 'echo lb broke' 12;      OUT="$(run05)"; chk "magnum-e2e LB FAIL" "$?" 1
mkta 'echo isolation!' 13;    OUT="$(run05)"; chk "magnum-e2e isolation FAIL" "$?" 1
grep -q 'CRITICAL' <<<"$OUT" && ok "e2e flags isolation critical" || no "e2e flags isolation critical"
mkta 'echo no foil app-cred file' 14; OUT="$(run05)"; chk "magnum-e2e no-foil HOLD" "$?" 2
grep -q 'onboard a 2nd tenant' <<<"$OUT" && ok "e2e surfaces foil dependency" || no "e2e surfaces foil dependency"
mkta 'echo kubectl absent' 14; OUT="$(run05)"; chk "magnum-e2e precond HOLD" "$?" 2

# ---- d011-04-octavia-lb: mock kubectl+openstack+curl (faithful: LB name from svc) ----
B4="$(mktemp -d)"; mkdir -p "$B4/bin" "$B4/home/tenant-beta/kube" "$B4/home/vault-init"
touch "$B4/home/vault-init/vault-ca-root.pem"
printf 'export OS_AUTH_URL=https://ks:5000/v3\nexport OS_CACERT=%s/home/vault-init/vault-ca-root.pem\nexport OS_USERNAME=admin\nexport OS_PASSWORD=x\n' "$B4" > "$B4/home/admin-openrc"
printf 'apiVersion: v1\nkind: Config\n' > "$B4/home/tenant-beta/kube/config"
export MST="$B4/state"; mkdir -p "$MST"
cat > "$B4/bin/kubectl" <<'KM'
#!/usr/bin/env bash
ST="${MST:?}"
case "$1 $2" in
  "create deployment") echo "$3" > "$ST/svc"; echo "deployment.apps/$3 created"; exit 0;;
  "expose deployment") echo exposed; exit 0;;
  "get svc") [ "${OCTAVIA_READY:-1}" = 1 ] && echo "10.12.6.50"; exit 0;;
  "get pods") echo "pod-a"; exit 0;;
  "delete pod") exit 0;; "delete svc") exit 0;; "delete deploy") exit 0;;
esac
case "$*" in "wait "*) exit 0;; esac
exit 0
KM
cat > "$B4/bin/openstack" <<'OM'
#!/usr/bin/env bash
ST="${MST:?}"; SVC="$(cat "$ST/svc" 2>/dev/null || echo unknown)"
case "$*" in
  "loadbalancer list"*)
    [ "${OCTAVIA_READY:-1}" = 1 ] || { echo "octavia down" >&2; exit 1; }
    printf '[{"id":"lb-1","name":"kube_service_beta_default_%s","vip_address":"10.12.6.50"}]\n' "$SVC";;
  "loadbalancer amphora list"*) echo '[{"compute_id":"cmp-1","role":"STANDALONE"}]';;
  "loadbalancer show"*) echo "${MOCK_LB_STATUS:-ACTIVE}";;
  "loadbalancer failover"*) echo "failover accepted";;
  "server show"*) echo '{"flavor":"m1.amphora","id":"cmp-1"}';;
  "flavor show"*) echo '{"vcpus":1,"ram":1024}';;
  "hypervisor list"*)
    if [ "${MOCK_HEADROOM:-ok}" = ok ]; then echo '[{"vcpus":16,"vcpus_used":4,"memory_mb":32768,"memory_mb_used":8192}]';
    else echo '[{"vcpus":4,"vcpus_used":4,"memory_mb":8192,"memory_mb_used":8192}]'; fi;;
esac
OM
cat > "$B4/bin/curl" <<'CM'
#!/usr/bin/env bash
ST="${MST:?}"; w=0; for a in "$@"; do [ "$a" = "-w" ] && w=1; done
if [ "$w" = 1 ]; then echo 200; exit 0; fi
n=$(cat "$ST/rr" 2>/dev/null || echo 0); n=$((n+1)); echo "$n" > "$ST/rr"
if [ "${MOCK_RR:-multi}" = multi ]; then [ $((n % 2)) -eq 0 ] && echo pod-a || echo pod-b; else echo pod-a; fi
CM
chmod +x "$B4/bin/"*
run04(){ rm -f "$MST/rr" "$MST/svc"; HOME="$B4/home" PATH="$B4/bin:$PATH" MST="$MST" bash "$CHK/d011-04-octavia-lb.sh" 2>&1; }
OUT="$(OCTAVIA_READY=0 run04)"; chk "octavia not-ready HOLD" "$?" 2
OUT="$(MOCK_RR=multi run04)"; chk "lb RR+member PASS (amphora skipped)" "$?" 0
grep -qE '^RESULT d011-04-octavia-lb PASS 0 ' <<<"$OUT" && ok "lb PASS line" || no "lb PASS line"
grep -q 'amphora failover SKIPPED' <<<"$OUT" && ok "lb notes amphora skipped" || no "lb notes amphora skipped"
grep -q 'round-robin: 2 distinct' <<<"$OUT" && ok "lb round-robin distinct" || no "lb round-robin distinct"
OUT="$(MOCK_RR=single run04)"; chk "lb RR single-backend FAIL" "$?" 1
rm -f "$B4/home/tenant-beta/kube/config"; OUT="$(MOCK_RR=multi run04)"; chk "lb no-kubeconfig HOLD" "$?" 2
printf 'apiVersion: v1\nkind: Config\n' > "$B4/home/tenant-beta/kube/config"
OUT="$(VR_DISRUPTIVE=1 MOCK_RR=multi MOCK_HEADROOM=no run04)"; chk "amphora headroom-NO HOLD" "$?" 2
grep -q 'headroom NO' <<<"$OUT" && ok "amphora headroom guard fires" || no "amphora headroom guard fires"
OUT="$(VR_DISRUPTIVE=1 MOCK_RR=multi MOCK_HEADROOM=ok MOCK_LB_STATUS=ACTIVE run04)"; chk "amphora failover PASS" "$?" 0
grep -q 'amphora failover recovered' <<<"$OUT" && ok "amphora failover recovery asserted" || no "amphora failover recovery asserted"
OUT="$(VR_DISRUPTIVE=1 MOCK_RR=multi MOCK_HEADROOM=ok MOCK_LB_STATUS=ERROR run04)"; chk "amphora failover ERROR FAIL" "$?" 1
OUT="$(VR_CHECKDIR="$CHK" bash "$VAL" --list 2>&1)"
grep -q 'd011-04-octavia-lb' <<<"$OUT" && ok "orchestrator discovers d011-04" || no "orchestrator discovers d011-04"
grep -q 'd011-05-magnum-e2e' <<<"$OUT" && ok "orchestrator discovers d011-05" || no "orchestrator discovers d011-05"

echo; [ "$F" = 0 ] && { echo "ALL PASS ($P checks)"; exit 0; } || { echo "FAILURES: $F"; exit 1; }