diff --git a/scripts/tenant-acceptance.sh b/scripts/tenant-acceptance.sh new file mode 100644 index 0000000..235125d --- /dev/null +++ b/scripts/tenant-acceptance.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# tenant-acceptance.sh -- tenant-facing acceptance test for an Option-3 tenant cluster. +# Usage: tenant-acceptance.sh [foil-appcred-file] +# tenant name; cluster -cluster, creds in ~/tenant-/ +# [foil-appcred-file] app-cred file (-f shell format) of a DIFFERENT tenant used as the +# isolation foil (default: $HOME/acme-svc-appcred.txt) +# P0 health_status (single-column) + trustee identity confirmation (dynamic) +# P1 kubeconfig via coe cluster config AS TENANT -> kubectl nodes/pods +# P2 k8s Service type=LoadBalancer via OCCM -> Octavia -> FIP -> curl from jumphost +# P3 isolation: the foil identity must see NOTHING of (any visibility = CRITICAL) +# Exit: 0 pass | 11 kube fail | 12 LB fail | 13 ISOLATION VIOLATION | 14 precondition +# health not-yet-HEALTHY is WARN-only (driver reconcile lag), never a gate failure. +# NOTE: P2 leaves the lbtest deployment + LB in place for inspection (cleanup is a separate gate). +# Provenance: gate t3-01 of the 2026-07-02 session (D-066/D-067 acceptance); mock-tested 4-branch. +set -u +CLIENT="${1:-}" +[ -n "$CLIENT" ] || { echo "usage: tenant-acceptance.sh [foil-appcred-file]"; exit 14; } +CF="$HOME/tenant-${CLIENT}/${CLIENT}-cluster-cred.txt" +FOIL_ACF="${2:-$HOME/acme-svc-appcred.txt}" +[ -s "$CF" ] || { echo "PRECOND: no $CF"; exit 14; } +[ -s "$FOIL_ACF" ] || { echo "PRECOND: no foil app-cred file $FOIL_ACF (isolation probe needs it)"; exit 14; } +command -v kubectl >/dev/null || { echo "PRECOND: kubectl absent on jumphost"; exit 14; } +CUID=$(awk -F= '/^user_id=/{print $2}' "$CF") + +tenant_env() { for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done + export OS_AUTH_URL="$(awk -F= '/^auth_url=/{print $2}' "$CF")" OS_IDENTITY_API_VERSION=3 + export OS_CACERT="${OS_CACERT:-$HOME/vault-init/vault-ca-root.pem}" + export OS_USERNAME="${CLIENT}-cluster" OS_USER_DOMAIN_ID="$(awk -F= '/^user_domain_id=/{print $2}' "$CF")" + export OS_PROJECT_ID="$(awk -F= '/^project_id=/{print $2}' "$CF")" OS_PASSWORD="$(awk -F= '/^password=/{print $2}' "$CF")"; } +admin_env() { for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done; source "$HOME/admin-openrc"; } +foil_env() { for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done + export OS_AUTH_TYPE=v3applicationcredential OS_AUTH_URL="https://10.12.4.50:5000/v3" OS_IDENTITY_API_VERSION=3 + export OS_CACERT="${OS_CACERT:-$HOME/vault-init/vault-ca-root.pem}" + export OS_APPLICATION_CREDENTIAL_ID="$(awk -F'"' '/^id=/{print $2}' "$FOIL_ACF")" + export OS_APPLICATION_CREDENTIAL_SECRET="$(awk -F'"' '/^secret=/{print $2}' "$FOIL_ACF")"; } + +echo "=== P0: health + trust identities ===" +H=$( (tenant_env; openstack coe cluster show "${CLIENT}-cluster" -f value -c health_status &1) || true ) +BUUID=$( (tenant_env; openstack coe cluster show "${CLIENT}-cluster" -f value -c uuid &1) || true ) +echo " health_status=$H uuid=$BUUID" +case "$H" in HEALTHY) echo " health: PASS";; *) echo " health: WARN (not HEALTHY yet -- reconcile lag or D-042-class issue; re-check later)";; esac +case "$BUUID" in *[!0-9a-f-]*|'') echo "PRECOND: bad beta uuid"; exit 14;; esac +TRUSTEE=$( (admin_env; openstack trust list -f json &1) | python3 -c " +import sys,json +try: + ts=[t for t in json.load(sys.stdin) if t.get('Trustor User ID')=='$CUID'] + print(ts[0]['Trustee User ID'] if ts else '') +except Exception: print('')" ) +if [ -n "$TRUSTEE" ]; then + TNAME=$( (admin_env; openstack user show "$TRUSTEE" -f value -c name &1) || true ) + echo " trustee: $TRUSTEE name=$TNAME (expect magnum_domain_admin)" +else echo " trustee: WARN could not resolve"; fi + +echo +echo "=== P1: kubeconfig (as tenant) + nodes/pods ===" +KDIR="$HOME/tenant-${CLIENT}/kube"; mkdir -p "$KDIR"; chmod 700 "$KDIR" +CFGOUT=$( (tenant_env; cd "$KDIR" && openstack coe cluster config "${CLIENT}-cluster" --dir "$KDIR" --force &1) || true ) +printf '%s\n' "$CFGOUT" | head -2 +[ -s "$KDIR/config" ] || { echo "FAIL: no kubeconfig written"; exit 11; } +chmod 600 "$KDIR/config"; export KUBECONFIG="$KDIR/config" +echo "--- nodes ---" +NODES=$(kubectl get nodes -o wide 2>&1 || true); printf '%s\n' "$NODES" | sed 's/^/ /' +READY=$(printf '%s\n' "$NODES" | awk '$2=="Ready"{c++} END{print c+0}') +[ "$READY" -ge 2 ] || { echo "FAIL: expected >=2 Ready nodes, got $READY"; exit 11; } +echo "--- pods not Running/Completed (want none) ---" +BAD=$(kubectl get pods -A --no-headers 2>&1 | awk '$4!="Running" && $4!="Completed"' || true) +if [ -n "$BAD" ]; then printf '%s\n' "$BAD" | sed 's/^/ /'; echo " WARN: non-Running pods above"; else echo " (none)"; fi +TOT=$(kubectl get pods -A --no-headers 2>/dev/null | wc -l) +echo " pods total=$TOT P1: PASS (>=2 Ready nodes, kubeconfig fetched by TENANT)" + +echo +echo "=== P2: tenant LB via OCCM (Service type=LoadBalancer) ===" +kubectl create deployment lbtest --image=registry.k8s.io/e2e-test-images/agnhost:2.47 -- /agnhost netexec --http-port=8080 2>&1 | sed 's/^/ /' +kubectl expose deployment lbtest --port=80 --target-port=8080 --type=LoadBalancer 2>&1 | sed 's/^/ /' +LBIP="" +for i in $(seq 1 24); do + LBIP=$(kubectl get svc lbtest -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) + printf '%s [%02d] external-ip=%s\n' "$(date +%T)" "$i" "${LBIP:-}" + [ -n "$LBIP" ] && break + sleep 25 +done +[ -n "$LBIP" ] || { echo "FAIL: no EXTERNAL-IP within ~10 min"; kubectl describe svc lbtest 2>&1 | tail -12; exit 12; } +CURL_OK=0 +for i in $(seq 1 12); do + R=$(curl -s --max-time 8 "http://$LBIP/hostname" 2>&1 || true) + printf '%s curl[%02d]: %s\n' "$(date +%T)" "$i" "${R:-}" + case "$R" in lbtest-*) CURL_OK=1; break;; esac + sleep 10 +done +[ "$CURL_OK" = 1 ] || { echo "FAIL: LB not serving pod hostname"; exit 12; } +LBOS=$( (admin_env; openstack loadbalancer list -f value -c name -c provisioning_status -c operating_status &1) || true ) +echo "--- octavia view (admin) ---"; printf '%s\n' "$LBOS" | sed 's/^/ /' +echo " P2: PASS (OCCM -> Octavia -> FIP $LBIP -> pod)" + +echo +echo "=== P3: isolation -- foil tenant must see NOTHING of ${CLIENT} ===" +VIOL=0 +AT=$( (foil_env; openstack token issue -f value -c project_id &1) || true ) +case "$AT" in *[!0-9a-f]*|'') echo "PRECOND: foil auth failed -- raw: $AT"; exit 14;; esac +echo " foil authenticated (project $AT)" +C1=$( (foil_env; openstack coe cluster list -f value -c name &1) || true ) +echo " foil coe cluster list: ${C1:-}" +printf '%s\n' "$C1" | grep -q "${CLIENT}-cluster" && { echo " *** VIOLATION: foil sees ${CLIENT}-cluster ***"; VIOL=1; } +C2=$( (foil_env; openstack coe cluster show "$BUUID" -f value -c name &1) || true ) +echo " foil show ${CLIENT} cluster uuid: $C2" +printf '%s\n' "$C2" | grep -qiE 'not.*found|404|denied|403' || { echo " *** VIOLATION: foil can read ${CLIENT} cluster ***"; VIOL=1; } +C3=$( (foil_env; openstack network show "${CLIENT}-net" -f value -c id &1) || true ) +echo " foil show ${CLIENT}-net: $C3" +printf '%s\n' "$C3" | grep -qiE 'no network|not.*found|404|unable' || { echo " *** VIOLATION: foil sees ${CLIENT}-net ***"; VIOL=1; } +C4=$( (foil_env; openstack server list -f value -c Name &1) || true ) +echo " foil server list: ${C4:-}" +printf '%s\n' "$C4" | grep -qi "${CLIENT}-cluster" && { echo " *** VIOLATION: foil sees ${CLIENT} nodes ***"; VIOL=1; } +C5=$( (foil_env; openstack loadbalancer list -f value -c name &1) || true ) +echo " foil loadbalancer list: ${C5:-}" +printf '%s\n' "$C5" | grep -qE 'kube|lbtest|k8s' && { echo " *** VIOLATION: foil sees ${CLIENT} LB ***"; VIOL=1; } +[ "$VIOL" = 0 ] || { echo "P3: ISOLATION VIOLATION -- CRITICAL"; exit 13; } +echo " P3: PASS (all ${CLIENT} resources invisible to foil)" +echo +echo "RESULT: t3-01 TENANT ACCEPTANCE PASS (P1 kube, P2 LB, P3 isolation; health=$H)" +exit 0