Newer
Older
openstack-caracal-ipv4 / scripts / tenant-offboard.sh
@JANeumatrix JANeumatrix 24 minutes ago 9 KB Patch
#!/usr/bin/env bash
# tenant-offboard.sh -- full tenant offboarding (Option-3 or legacy single-svc layout).
# Usage: tenant-offboard.sh <client> [--audit|--apply]
#   --audit (DEFAULT): read-only inventory of everything the offboard would remove.
#   --apply          : destructive. Requires typing the client name at the terminal.
# Order (dependency-driven): magnum clusters (as TENANT; cleans per-cluster trustee/trust/
# child app-cred) -> residual trust sweep (admin) -> LB/FIP/server sweep -> L3 unwind ->
# users (app creds die with user) -> projects -> domain disable+delete -> local cred dir retired.
# Exit: 0 ok | 20 precondition/blocklist | 21 cluster delete failed | 22 resource sweep failed
#       | 23 identity teardown failed | 24 confirmation refused/unavailable
# ERROR REGIME (DOCFIX-082): set -uo pipefail; every mutation goes through run() which
# CAPTURES output then TESTS exit status (no pipeline on the mutating command itself).
# Sweep phases are best-effort continue-and-report: failures are printed, counted, and the
# script exits nonzero at the end (22 sweep / 23 identity) rather than stranding silently.
set -uo pipefail
SWEEP_FAIL=0; ID_FAIL=0
run(){ # run <failure-counter-name> <cmd...>: capture-then-test; count failures loudly
  local _c="$1"; shift; local out
  if out=$("$@" </dev/null 2>&1); then
    [ -n "$out" ] && printf '%s\n' "$out" | sed 's/^/    /'
  else
    printf '%s\n' "$out" | sed 's/^/    /'
    echo "    ^ FAILED: $*"
    eval "$_c=\$(( $_c + 1 ))"
  fi
  return 0
}
CLIENT="${1:-}"; MODE="${2:---audit}"
BLOCKLIST="default admin_domain service_domain magnum capi heat"
[ -n "$CLIENT" ] || { echo "usage: tenant-offboard.sh <client> [--audit|--apply]"; exit 20; }
case "$MODE" in --audit|--apply) : ;; *) echo "bad mode: $MODE"; exit 20;; esac
CL=$(printf '%s' "$CLIENT" | tr 'A-Z' 'a-z')
for b in $BLOCKLIST; do [ "$CL" = "$b" ] && { echo "REFUSED: '$CLIENT' is a protected domain"; exit 20; }; done

admin_env(){ for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done; source "$HOME/admin-openrc"; }
is_id(){ [[ "${1:-}" =~ ^[0-9a-f]{32}$ ]]; }

admin_env
DOM=$(openstack domain show "$CLIENT" -f value -c id </dev/null 2>&1) || true
is_id "$DOM" || { echo "PRECOND: domain '$CLIENT' not found (raw: $DOM)"; exit 20; }
echo "=== OFFBOARD $MODE: client=$CLIENT domain=$DOM ==="

# ---- tenant password cred discovery (magnum is tenant-scoped) ----
TCF=""
for c in "$HOME/tenant-${CLIENT}/${CLIENT}-cluster-cred.txt" \
         "$HOME/tenant-${CLIENT}/${CLIENT}-svc-cred.txt" \
         "$HOME/${CLIENT}-svc-cred.txt"; do
  [ -s "$c" ] && grep -qE '^password=' "$c" && { TCF="$c"; break; }
done
tenant_env(){ for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done
  export OS_AUTH_URL="https://10.12.4.50:5000/v3" OS_IDENTITY_API_VERSION=3
  export OS_CACERT="${OS_CACERT:-$HOME/vault-init/vault-ca-root.pem}"
  export OS_USERNAME="$(awk -F= '/^username=/{print $2}' "$TCF")" OS_USER_DOMAIN_ID="$DOM"
  export OS_PROJECT_ID="$(awk -F= '/^project_id=/{print $2}' "$TCF")"
  export OS_PASSWORD="$(awk -F= '/^password=/{print $2}' "$TCF")"; }

# ---- inventory ----
USERS=$(openstack user list --domain "$DOM" -f value -c ID -c Name </dev/null 2>&1) || true
PROJS=$(openstack project list --domain "$DOM" -f value -c ID -c Name </dev/null 2>&1) || true
echo "--- users in domain ---";    printf '%s\n' "${USERS:-<none>}" | sed 's/^/  /'
echo "--- projects in domain ---"; printf '%s\n' "${PROJS:-<none>}" | sed 's/^/  /'

CLUSTERS=""
if [ -n "$TCF" ]; then
  echo "--- clusters (tenant scope via $(basename "$TCF")) ---"
  CLUSTERS=$( (tenant_env; openstack coe cluster list -f value -c uuid -c name -c status </dev/null 2>&1) || true )
  printf '%s\n' "${CLUSTERS:-<none>}" | sed 's/^/  /'
else
  echo "--- clusters: NO tenant password cred file found -- cannot enumerate (magnum is tenant-scoped) ---"
fi

UIDLIST=$(printf '%s\n' "$USERS" | awk '{print $1}')
TRUSTS=$( openstack trust list -f json </dev/null 2>&1 | python3 -c "
import sys,json
uids=set('''$UIDLIST'''.split())
try:
    for t in json.load(sys.stdin):
        if t.get('Trustor User ID') in uids:
            print(t.get('ID'), t.get('Trustor User ID'), t.get('Trustee User ID'))
except Exception as e: print('trust-parse-error', e)" ) || true
echo "--- trusts with trustor in domain ---"; printf '%s\n' "${TRUSTS:-<none>}" | sed 's/^/  /'

for P in $(printf '%s\n' "$PROJS" | awk '{print $1}'); do
  echo "--- project $P resources ---"
  echo "  LBs:";     openstack loadbalancer list --project "$P" -f value -c id -c name -c provisioning_status </dev/null 2>&1 | sed 's/^/    /'
  echo "  FIPs:";    openstack floating ip list --project "$P" -f value -c ID -c "Floating IP Address" </dev/null 2>&1 | sed 's/^/    /'
  echo "  servers:"; openstack server list --project "$P" -f value -c ID -c Name -c Status </dev/null 2>&1 | sed 's/^/    /'
  echo "  routers:"; openstack router list --project "$P" -f value -c ID -c Name </dev/null 2>&1 | sed 's/^/    /'
  echo "  subnets:"; openstack subnet list --project "$P" -f value -c ID -c Name </dev/null 2>&1 | sed 's/^/    /'
  echo "  networks:"; openstack network list --project "$P" -f value -c ID -c Name </dev/null 2>&1 | sed 's/^/    /'
done
echo "--- local cred artifacts ---"
ls -d "$HOME/tenant-${CLIENT}" 2>/dev/null | sed 's/^/  /'
ls "$HOME/${CLIENT}"-*.txt "$HOME/${CLIENT}"-*.pem 2>/dev/null | sed 's/^/  /'

if [ "$MODE" = "--audit" ]; then echo "AUDIT COMPLETE (no changes). Re-run with --apply to execute."; exit 0; fi

# ---- confirmation gate (typed at terminal; refuses if no tty) ----
if [ ! -r /dev/tty ]; then echo "REFUSED: --apply needs an interactive terminal"; exit 24; fi
printf 'Type the client name (%s) to confirm DESTRUCTIVE offboard: ' "$CLIENT" > /dev/tty
IFS= read -r CONF < /dev/tty || CONF=""
[ "$CONF" = "$CLIENT" ] || { echo "REFUSED: confirmation mismatch"; exit 24; }

# ---- Phase A: clusters (as tenant) ----
if [ -n "$CLUSTERS" ] && [ "$CLUSTERS" != "<none>" ]; then
  [ -n "$TCF" ] || { echo "FATAL: clusters exist but no tenant cred"; exit 21; }
  printf '%s\n' "$CLUSTERS" | while read -r CU CN CS; do
    [ -n "$CU" ] || continue
    echo "== deleting cluster $CN ($CU, was $CS) =="
    (tenant_env; openstack coe cluster delete "$CU" </dev/null 2>&1) | sed 's/^/  /'
  done
  for i in $(seq 1 40); do
    LEFT=$( (tenant_env; openstack coe cluster list -f value -c name -c status </dev/null 2>&1) || true )
    printf '%s [%02d] remaining: %s\n' "$(date +%T)" "$i" "${LEFT:-<none>}"
    [ -z "$LEFT" ] && break
    grep -q 'DELETE_FAILED' <<<"$LEFT" && { echo "FATAL: DELETE_FAILED -- see appendix-A stuck-delete"; exit 21; }
    sleep 20
  done
  LEFT=$( (tenant_env; openstack coe cluster list -f value -c name </dev/null 2>&1) || true )
  [ -z "$LEFT" ] || { echo "FATAL: clusters not gone in budget"; exit 21; }
fi

admin_env
# ---- Phase B: residual trusts ----
printf '%s\n' "${TRUSTS:-}" | while read -r TID REST; do
  [ -n "$TID" ] && [ "$TID" != "<none>" ] || continue
  T2=$(openstack trust show "$TID" -f value -c id </dev/null 2>&1) || true
  [ "$T2" = "$TID" ] && { echo "  deleting residual trust $TID"; run SWEEP_FAIL openstack trust delete "$TID"; }
done

# ---- Phase C+D: per-project resource sweep + L3 unwind ----
for P in $(printf '%s\n' "$PROJS" | awk '{print $1}'); do
  for LB in $(openstack loadbalancer list --project "$P" -f value -c id </dev/null 2>&1); do
    is_id "${LB//-/}" 2>/dev/null || true
    echo "  LB delete --cascade $LB"; run SWEEP_FAIL openstack loadbalancer delete --cascade "$LB"
  done
  for i in $(seq 1 20); do
    N=$(openstack loadbalancer list --project "$P" -f value -c id </dev/null 2>&1 | grep -c . || true)
    [ "$N" = 0 ] && break; echo "  waiting LBs gone ($N left)"; sleep 15
  done
  for F in $(openstack floating ip list --project "$P" -f value -c ID </dev/null 2>&1); do
    echo "  FIP delete $F"; run SWEEP_FAIL openstack floating ip delete "$F"; done
  for S in $(openstack server list --project "$P" -f value -c ID </dev/null 2>&1); do
    echo "  server delete $S"; run SWEEP_FAIL openstack server delete "$S"; done
  for R in $(openstack router list --project "$P" -f value -c ID </dev/null 2>&1); do
    for SN in $(openstack subnet list --project "$P" -f value -c ID </dev/null 2>&1); do
      openstack router remove subnet "$R" "$SN" </dev/null 2>&1 || true; done
    openstack router unset --external-gateway "$R" </dev/null 2>&1 || true
    echo "  router delete $R"; run SWEEP_FAIL openstack router delete "$R"; done
  for SN in $(openstack subnet list --project "$P" -f value -c ID </dev/null 2>&1); do
    echo "  subnet delete $SN"; run SWEEP_FAIL openstack subnet delete "$SN"; done
  for NW in $(openstack network list --project "$P" -f value -c ID </dev/null 2>&1); do
    echo "  network delete $NW"; run SWEEP_FAIL openstack network delete "$NW"; done
done

# ---- Phase E: identities ----
for U in $(printf '%s\n' "$USERS" | awk '{print $1}'); do
  echo "  user delete $U"; run ID_FAIL openstack user delete "$U"; done
for P in $(printf '%s\n' "$PROJS" | awk '{print $1}'); do
  echo "  project delete $P"; run ID_FAIL openstack project delete "$P"; done
run ID_FAIL openstack domain set --disable "$DOM"
echo "  domain delete $DOM"; run ID_FAIL openstack domain delete "$DOM"
D2=$(openstack domain show "$CLIENT" -f value -c id </dev/null 2>&1) || true
is_id "$D2" && { echo "FATAL: domain still present"; exit 23; }
[ "$ID_FAIL" = 0 ] || { echo "FATAL: identity teardown had $ID_FAIL failures (domain gone; review output)"; exit 23; }

# ---- Phase F: retire local creds ----
if [ -d "$HOME/tenant-${CLIENT}" ]; then
  DST="$HOME/tenant-${CLIENT}.offboarded-$(date -u +%Y%m%d-%H%M%S)"
  mv "$HOME/tenant-${CLIENT}" "$DST" && chmod 700 "$DST" && echo "  local creds retired -> $DST (shred manually when record no longer needed)"
fi
ls "$HOME/${CLIENT}"-*.txt "$HOME/${CLIENT}"-*.pem 2>/dev/null | sed 's/^/  LEGACY loose file (handle manually): /'
if [ "$SWEEP_FAIL" -gt 0 ]; then
  echo "OFFBOARD FINISHED WITH $SWEEP_FAIL SWEEP FAILURES: $CLIENT (identities gone; orphaned resources above)"
  exit 22
fi
echo "OFFBOARD COMPLETE: $CLIENT"
exit 0