#!/usr/bin/env bash
# tenant-offboard.sh -- full tenant offboarding (Option-3 or legacy single-svc layout).
# Usage: tenant-offboard.sh <client> [--audit|--apply]
# tenant-offboard.sh --sweep-magnum-orphans [--audit|--apply] (DOCFIX-099)
# --audit (DEFAULT): read-only inventory of everything the offboard would remove.
# --apply : destructive. Requires typing the client name at the terminal
# (sweep mode: the literal token 'magnum-orphans').
# Order (dependency-driven): magnum clusters (as TENANT; cleans per-cluster trustee/trust/
# child app-cred) -> residual trust sweep (admin) -> LB/FIP/server sweep -> L3 unwind ->
# app creds (Phase E0: explicit inventory, idempotent -- they die with their user) ->
# users -> projects -> domain disable+delete -> local cred dir retired.
# --sweep-magnum-orphans: standalone mode; classifies every user in the 'magnum' trustee
# domain (D-046) as ORPHAN / PROTECTED / UNPROVEN and (--apply, gated) deletes ONLY
# proven orphans. Fail-closed: UNPROVEN is never touched. Uses exit codes 0/20/22/24.
# Exit: 0 ok | 20 precondition/blocklist | 21 cluster delete failed | 22 resource sweep failed
# | 23 identity teardown failed | 24 confirmation refused/unavailable
# ERROR REGIME (DOCFIX-082): set -uo pipefail; every mutation goes through run() which
# CAPTURES output then TESTS exit status (no pipeline on the mutating command itself).
# Sweep phases are best-effort continue-and-report: failures are printed, counted, and the
# script exits nonzero at the end (22 sweep / 23 identity) rather than stranding silently.
set -uo pipefail
SWEEP_FAIL=0; ID_FAIL=0
run(){ # run <failure-counter-name> <cmd...>: capture-then-test; count failures loudly
local _c="$1"; shift; local out
if out=$("$@" </dev/null 2>&1); then
[ -n "$out" ] && printf '%s\n' "$out" | sed 's/^/ /'
else
printf '%s\n' "$out" | sed 's/^/ /'
echo " ^ FAILED: $*"
eval "$_c=\$(( $_c + 1 ))"
fi
return 0
}
CLIENT="${1:-}"; MODE="${2:---audit}"
BLOCKLIST="default admin_domain service_domain magnum capi heat"
[ -n "$CLIENT" ] || { echo "usage: tenant-offboard.sh <client>|--sweep-magnum-orphans [--audit|--apply]"; exit 20; }
case "$MODE" in --audit|--apply) : ;; *) echo "bad mode: $MODE"; exit 20;; esac
admin_env(){ for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done; source "$HOME/admin-openrc"; }
is_id(){ [[ "${1:-}" =~ ^[0-9a-f]{32}$ ]]; }
# ==== DOCFIX-099: orphaned per-cluster trustee sweep (magnum trustee domain) ====
# Magnum mints one trustee user per cluster in the 'magnum' trustee domain (D-046),
# conventionally named <cluster_uuid>_<project_id>, plus a trust and a child app-cred
# (D-039). An unclean cluster/tenant teardown can strand the trustee. Classification is
# FAIL-CLOSED: ORPHAN requires the owning project to be DEFINITIVELY gone (exact
# keystone not-found message). If the project is alive, only a SUCCESSFUL admin-scope
# cluster show proves PROTECTED; any cluster-show failure is UNPROVEN (magnum is
# tenant-scoped -- an admin-scope 404 can mask a policy denial). magnum_domain_admin
# and any name not matching the trustee convention are never touched. UNPROVEN never
# deletes. Trust list runs in the ADMIN context only (D-073 posture).
if [ "$CLIENT" = "--sweep-magnum-orphans" ]; then
admin_env
MDOM=$(openstack domain show magnum -f value -c id </dev/null 2>&1) || true
is_id "$MDOM" || { echo "PRECOND: magnum trustee domain not found -- D-046 setup missing? (raw: $MDOM)"; exit 20; }
MADM=$(openstack user show magnum_domain_admin --domain "$MDOM" -f value -c id </dev/null 2>&1) || true
is_id "$MADM" || { echo "PRECOND: magnum_domain_admin not found in magnum domain (raw: $MADM)"; exit 20; }
echo "=== MAGNUM ORPHAN-TRUSTEE SWEEP $MODE: domain=magnum ($MDOM) ==="
MUSERS=$(openstack user list --domain "$MDOM" -f value -c ID -c Name </dev/null 2>&1) || true
grep -q "^$MADM " <<<"$MUSERS" || { echo "PRECOND: magnum-domain user list did not include magnum_domain_admin -- listing untrusted (raw: $(head -1 <<<"$MUSERS"))"; exit 20; }
# DOCFIX-094 pattern: stdout-only structured capture (merged stderr kills the parse)
TRERR=$(mktemp)
STR=$(openstack trust list -f json </dev/null 2>"$TRERR" | python3 -c "
import sys,json
try:
for t in json.load(sys.stdin):
print(t.get('ID'), t.get('Trustee User ID'))
except Exception as e: print('trust-parse-error', e)") || true
[ -s "$TRERR" ] && echo " NOTE: trust list stderr: $(head -1 "$TRERR")"
rm -f "$TRERR"
grep -q 'trust-parse-error' <<<"${STR:-}" && { echo "PRECOND: trust list unparseable -- sweep would be blind on trusts"; exit 20; }
UUID_RE='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
ORPHANS=""; NORPH=0; NPROT=0; NUNPR=0
while read -r TUID TNAME; do
[ -n "$TUID" ] || continue
if ! is_id "$TUID"; then
echo "UNPROVEN $TUID $TNAME -- not an id+name user row; listing suspect; not touching"; NUNPR=$((NUNPR+1)); continue
fi
if [ "$TUID" = "$MADM" ]; then
echo "PROTECTED $TUID $TNAME -- trustee-domain service identity (D-046)"; NPROT=$((NPROT+1)); continue
fi
CU="${TNAME%%_*}"; PJ="${TNAME#*_}"
if ! { [[ "$CU" =~ $UUID_RE ]] && [[ "$PJ" =~ ^[0-9a-f]{32}$ ]] && [ "$TNAME" = "${CU}_${PJ}" ]; }; then
echo "UNPROVEN $TUID $TNAME -- name does not match <cluster_uuid>_<project_id> trustee convention; not touching"; NUNPR=$((NUNPR+1)); continue
fi
PSHOW=$(openstack project show "$PJ" -f value -c id </dev/null 2>&1) || true
if [ "$PSHOW" = "$PJ" ]; then
CSHOW=$(openstack coe cluster show "$CU" -f value -c uuid </dev/null 2>&1) || true
if [ "$CSHOW" = "$CU" ]; then
echo "PROTECTED $TUID $TNAME -- cluster $CU alive in project $PJ"; NPROT=$((NPROT+1))
else
echo "UNPROVEN $TUID $TNAME -- project alive but cluster $CU not shown in admin scope (a 404 can mask a policy denial; verify in tenant scope); not touching"; NUNPR=$((NUNPR+1))
fi
elif grep -q "No project with a name or ID of '$PJ' exists" <<<"$PSHOW"; then
echo "ORPHAN $TUID $TNAME -- owning project $PJ definitively gone"
for T in $(awk -v u="$TUID" '$2==u{print $1}' <<<"${STR:-}"); do
echo " + trust $T (trustee is this user)"
done
ORPHANS="${ORPHANS}${TUID} ${TNAME}"$'\n'; NORPH=$((NORPH+1))
else
echo "UNPROVEN $TUID $TNAME -- project lookup ambiguous (raw: $(head -1 <<<"$PSHOW")); not touching"; NUNPR=$((NUNPR+1))
fi
done <<<"$MUSERS"
echo "--- sweep summary: ORPHAN=$NORPH PROTECTED=$NPROT UNPROVEN=$NUNPR (UNPROVEN is never deleted) ---"
if [ "$MODE" = "--audit" ]; then
echo "AUDIT COMPLETE (no changes). Re-run with: tenant-offboard.sh --sweep-magnum-orphans --apply"
exit 0
fi
if [ "$NORPH" = 0 ]; then echo "NOTHING PROVEN ORPHANED -- no action taken (re-run safe)."; exit 0; fi
if [ ! -r /dev/tty ]; then echo "REFUSED: --apply needs an interactive terminal"; exit 24; fi
printf 'Type magnum-orphans to confirm DELETING %d orphaned trustee user(s): ' "$NORPH" > /dev/tty
IFS= read -r CONF < /dev/tty || CONF=""
[ "$CONF" = "magnum-orphans" ] || { echo "REFUSED: confirmation mismatch"; exit 24; }
# NOT a pipeline loop: SWEEP_FAIL increments must survive (subshell would eat them)
while read -r OU ON; do
[ -n "$OU" ] || continue
for T in $(awk -v u="$OU" '$2==u{print $1}' <<<"${STR:-}"); do
echo " trust delete $T (trustee $ON)"; run SWEEP_FAIL openstack trust delete "$T"
done
echo " trustee user delete $OU ($ON)"; run SWEEP_FAIL openstack user delete "$OU"
G=$(openstack user show "$OU" -f value -c id </dev/null 2>&1) || true
[ "$G" = "$OU" ] && { echo " ^ STILL PRESENT after delete"; SWEEP_FAIL=$((SWEEP_FAIL+1)); }
done <<<"$ORPHANS"
if [ "$SWEEP_FAIL" -gt 0 ]; then echo "SWEEP FINISHED WITH $SWEEP_FAIL FAILURES"; exit 22; fi
echo "SWEEP COMPLETE: $NORPH orphaned trustee(s) removed."
exit 0
fi
CL=$(printf '%s' "$CLIENT" | tr 'A-Z' 'a-z')
for b in $BLOCKLIST; do [ "$CL" = "$b" ] && { echo "REFUSED: '$CLIENT' is a protected domain"; exit 20; }; done
admin_env
DOM=$(openstack domain show "$CLIENT" -f value -c id </dev/null 2>&1) || true
is_id "$DOM" || { echo "PRECOND: domain '$CLIENT' not found (raw: $DOM)"; exit 20; }
echo "=== OFFBOARD $MODE: client=$CLIENT domain=$DOM ==="
# ---- tenant password cred discovery (magnum is tenant-scoped) ----
TCF=""
for c in "$HOME/tenant-${CLIENT}/${CLIENT}-cluster-cred.txt" \
"$HOME/tenant-${CLIENT}/${CLIENT}-svc-cred.txt" \
"$HOME/${CLIENT}-svc-cred.txt"; do
[ -s "$c" ] && grep -qE '^password=' "$c" && { TCF="$c"; break; }
done
tenant_env(){ for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done
export OS_AUTH_URL="https://10.12.4.50:5000/v3" OS_IDENTITY_API_VERSION=3
export OS_CACERT="${OS_CACERT:-$HOME/vault-init/vault-ca-root.pem}"
export OS_USERNAME="$(awk -F= '/^username=/{print $2}' "$TCF")" OS_USER_DOMAIN_ID="$DOM"
export OS_PROJECT_ID="$(awk -F= '/^project_id=/{print $2}' "$TCF")"
export OS_PASSWORD="$(awk -F= '/^password=/{print $2}' "$TCF")"; }
# ---- inventory ----
USERS=$(openstack user list --domain "$DOM" -f value -c ID -c Name </dev/null 2>&1) || true
PROJS=$(openstack project list --domain "$DOM" -f value -c ID -c Name </dev/null 2>&1) || true
echo "--- users in domain ---"; printf '%s\n' "${USERS:-<none>}" | sed 's/^/ /'
echo "--- projects in domain ---"; printf '%s\n' "${PROJS:-<none>}" | sed 's/^/ /'
CLUSTERS=""
if [ -n "$TCF" ]; then
echo "--- clusters (tenant scope via $(basename "$TCF")) ---"
CLUSTERS=$( (tenant_env; openstack coe cluster list -f value -c uuid -c name -c status </dev/null 2>&1) || true )
printf '%s\n' "${CLUSTERS:-<none>}" | sed 's/^/ /'
else
echo "--- clusters: NO tenant password cred file found -- cannot enumerate (magnum is tenant-scoped) ---"
fi
UIDLIST=$(printf '%s\n' "$USERS" | awk '{print $1}')
# DOCFIX-094: stdout-only structured capture -- merged stderr (deprecation notices)
# used to land at char 0, kill the JSON parse, and silently EMPTY the trust sweep.
TRERR=$(mktemp)
TRUSTS=$( openstack trust list -f json </dev/null 2>"$TRERR" | python3 -c "
import sys,json
uids=set('''$UIDLIST'''.split())
try:
for t in json.load(sys.stdin):
if t.get('Trustor User ID') in uids:
print(t.get('ID'), t.get('Trustor User ID'), t.get('Trustee User ID'))
except Exception as e: print('trust-parse-error', e)" ) || true
[ -s "$TRERR" ] && echo " NOTE: trust list stderr: $(head -1 "$TRERR")"
rm -f "$TRERR"
grep -q 'trust-parse-error' <<<"${TRUSTS:-}" && { echo "PRECOND: trust list unparseable -- audit/sweep would be blind; fix before offboarding"; exit 20; }
echo "--- trusts with trustor in domain ---"; printf '%s\n' "${TRUSTS:-<none>}" | sed 's/^/ /'
for P in $(printf '%s\n' "$PROJS" | awk '{print $1}'); do
echo "--- project $P resources ---"
echo " LBs:"; openstack loadbalancer list --project "$P" -f value -c id -c name -c provisioning_status </dev/null 2>&1 | sed 's/^/ /'
echo " FIPs:"; openstack floating ip list --project "$P" -f value -c ID -c "Floating IP Address" </dev/null 2>&1 | sed 's/^/ /'
echo " servers:"; openstack server list --project "$P" -f value -c ID -c Name -c Status </dev/null 2>&1 | sed 's/^/ /'
echo " routers:"; openstack router list --project "$P" -f value -c ID -c Name </dev/null 2>&1 | sed 's/^/ /'
echo " subnets:"; openstack subnet list --project "$P" -f value -c ID -c Name </dev/null 2>&1 | sed 's/^/ /'
echo " networks:"; openstack network list --project "$P" -f value -c ID -c Name </dev/null 2>&1 | sed 's/^/ /'
done
echo "--- local cred artifacts ---"
ls -d "$HOME/tenant-${CLIENT}" 2>/dev/null | sed 's/^/ /'
ls "$HOME/${CLIENT}"-*.txt "$HOME/${CLIENT}"-*.pem 2>/dev/null | sed 's/^/ /'
if [ "$MODE" = "--audit" ]; then echo "AUDIT COMPLETE (no changes). Re-run with --apply to execute."; exit 0; fi
# ---- confirmation gate (typed at terminal; refuses if no tty) ----
if [ ! -r /dev/tty ]; then echo "REFUSED: --apply needs an interactive terminal"; exit 24; fi
printf 'Type the client name (%s) to confirm DESTRUCTIVE offboard: ' "$CLIENT" > /dev/tty
IFS= read -r CONF < /dev/tty || CONF=""
[ "$CONF" = "$CLIENT" ] || { echo "REFUSED: confirmation mismatch"; exit 24; }
# ---- Phase A: clusters (as tenant) ----
if [ -n "$CLUSTERS" ] && [ "$CLUSTERS" != "<none>" ]; then
[ -n "$TCF" ] || { echo "FATAL: clusters exist but no tenant cred"; exit 21; }
printf '%s\n' "$CLUSTERS" | while read -r CU CN CS; do
[ -n "$CU" ] || continue
echo "== deleting cluster $CN ($CU, was $CS) =="
(tenant_env; openstack coe cluster delete "$CU" </dev/null 2>&1) | sed 's/^/ /'
done
for i in $(seq 1 40); do
LEFT=$( (tenant_env; openstack coe cluster list -f value -c name -c status </dev/null 2>&1) || true )
printf '%s [%02d] remaining: %s\n' "$(date +%T)" "$i" "${LEFT:-<none>}"
[ -z "$LEFT" ] && break
grep -q 'DELETE_FAILED' <<<"$LEFT" && { echo "FATAL: DELETE_FAILED -- see appendix-A stuck-delete"; exit 21; }
sleep 20
done
LEFT=$( (tenant_env; openstack coe cluster list -f value -c name </dev/null 2>&1) || true )
[ -z "$LEFT" ] || { echo "FATAL: clusters not gone in budget"; exit 21; }
fi
admin_env
# ---- Phase B: residual trusts ----
printf '%s\n' "${TRUSTS:-}" | while read -r TID REST; do
[ -n "$TID" ] && [ "$TID" != "<none>" ] || continue
T2=$(openstack trust show "$TID" -f value -c id </dev/null 2>&1) || true
[ "$T2" = "$TID" ] && { echo " deleting residual trust $TID"; run SWEEP_FAIL openstack trust delete "$TID"; }
done
# ---- Phase C+D: per-project resource sweep + L3 unwind ----
for P in $(printf '%s\n' "$PROJS" | awk '{print $1}'); do
for LB in $(openstack loadbalancer list --project "$P" -f value -c id </dev/null 2>&1); do
is_id "${LB//-/}" 2>/dev/null || true
echo " LB delete --cascade $LB"; run SWEEP_FAIL openstack loadbalancer delete --cascade "$LB"
done
for i in $(seq 1 20); do
N=$(openstack loadbalancer list --project "$P" -f value -c id </dev/null 2>&1 | grep -c . || true)
[ "$N" = 0 ] && break; echo " waiting LBs gone ($N left)"; sleep 15
done
for F in $(openstack floating ip list --project "$P" -f value -c ID </dev/null 2>&1); do
echo " FIP delete $F"; run SWEEP_FAIL openstack floating ip delete "$F"; done
for S in $(openstack server list --project "$P" -f value -c ID </dev/null 2>&1); do
echo " server delete $S"; run SWEEP_FAIL openstack server delete "$S"; done
for R in $(openstack router list --project "$P" -f value -c ID </dev/null 2>&1); do
for SN in $(openstack subnet list --project "$P" -f value -c ID </dev/null 2>&1); do
openstack router remove subnet "$R" "$SN" </dev/null 2>&1 || true; done
openstack router unset --external-gateway "$R" </dev/null 2>&1 || true
echo " router delete $R"; run SWEEP_FAIL openstack router delete "$R"; done
for SN in $(openstack subnet list --project "$P" -f value -c ID </dev/null 2>&1); do
echo " subnet delete $SN"; run SWEEP_FAIL openstack subnet delete "$SN"; done
for NW in $(openstack network list --project "$P" -f value -c ID </dev/null 2>&1); do
echo " network delete $NW"; run SWEEP_FAIL openstack network delete "$NW"; done
done
# ---- Phase E0: tenant app creds (explicit + idempotent; H2-style guard, DOCFIX-100) ----
# App creds die with their owning user (keystone cascade); this stage makes that removal
# EXPLICIT (per-user inventory, so the offboard record names what the user delete takes
# with it) and RE-RUN SAFE: a partially-offboarded tenant whose users are already gone
# (e.g. a prior --apply cleared Phase E users but failed at the domain step) SKIPs
# cleanly instead of erroring -- same skip semantics as tenant-onboard.sh stage3 (H2).
UIDS_E=$(printf '%s\n' "${USERS:-}" | awk 'NF{print $1}')
if [ -z "$UIDS_E" ]; then
echo " app-cred stage: no users remain in domain -- SKIP (idempotent re-run)"
else
for U in $UIDS_E; do
ACOUT=$(openstack application credential list --user "$U" -f value -c ID </dev/null 2>&1); ACRC=$?
if [ "$ACRC" != 0 ]; then
if grep -q "No user with a name or ID" <<<"$ACOUT"; then
echo " app creds ($U): owner already gone -- SKIP (idempotent re-run)"
else
printf '%s\n' "$ACOUT" | sed 's/^/ /'
echo " ^ FAILED: application credential list --user $U"
ID_FAIL=$((ID_FAIL+1))
fi
continue
fi
if [ -z "$ACOUT" ]; then
echo " app creds ($U): none -- already clean"
else
for AC in $ACOUT; do echo " app cred $AC (owner $U) -- removed via owner delete (cascade)"; done
fi
done
fi
# ---- Phase E: identities ----
for U in $(printf '%s\n' "$USERS" | awk '{print $1}'); do
echo " user delete $U"; run ID_FAIL openstack user delete "$U"; done
for P in $(printf '%s\n' "$PROJS" | awk '{print $1}'); do
echo " project delete $P"; run ID_FAIL openstack project delete "$P"; done
run ID_FAIL openstack domain set --disable "$DOM"
echo " domain delete $DOM"; run ID_FAIL openstack domain delete "$DOM"
D2=$(openstack domain show "$CLIENT" -f value -c id </dev/null 2>&1) || true
is_id "$D2" && { echo "FATAL: domain still present"; exit 23; }
[ "$ID_FAIL" = 0 ] || { echo "FATAL: identity teardown had $ID_FAIL failures (domain gone; review output)"; exit 23; }
# ---- Phase F: retire local creds ----
if [ -d "$HOME/tenant-${CLIENT}" ]; then
DST="$HOME/tenant-${CLIENT}.offboarded-$(date -u +%Y%m%d-%H%M%S)"
mv "$HOME/tenant-${CLIENT}" "$DST" && chmod 700 "$DST" && echo " local creds retired -> $DST (shred manually when record no longer needed)"
fi
ls "$HOME/${CLIENT}"-*.txt "$HOME/${CLIENT}"-*.pem 2>/dev/null | sed 's/^/ LEGACY loose file (handle manually): /'
if [ "$SWEEP_FAIL" -gt 0 ]; then
echo "OFFBOARD FINISHED WITH $SWEEP_FAIL SWEEP FAILURES: $CLIENT (identities gone; orphaned resources above)"
exit 22
fi
echo "OFFBOARD COMPLETE: $CLIENT"
exit 0