Newer
Older
openstack-caracal-ipv4 / tests / tenant-acceptance / run-tests.sh
#!/usr/bin/env bash
# tests/tenant-acceptance/run-tests.sh -- offline harness for scripts/tenant-acceptance.sh.
# Fake openstack/kubectl/curl replay per-case fixtures keyed by FIXDIR; identity is
# derived from the env the script's tenant_env/admin_env/foil_env helpers set up.
# The trust-list fake emits a realistic deprecation-style notice ON STDERR every call,
# so the happy path itself proves the DOCFIX-094 stdout-only JSON capture at :43
# (pre-fix, the merged notice killed the parse and the trustee checks fell to WARN).
# Fake sleep compresses the P2 wait loops. Requires no cloud access; mutates NOTHING
# outside mktemp dirs.
# Cases: happy PASS (incl. trustee resolution under stderr noise + surfaced NOTE);
# health-lag WARN-only; no-args=14; missing-cred=14; bad-uuid=14; foil-auth-fail=14;
# <2 Ready nodes=11; no EXTERNAL-IP=12; LB not serving=12; foil list leak=13;
# foil show leak=13.
# Exit: 0 all pass | 1 any case failed.  ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0
BIN="$TMP/bin"; mkdir -p "$BIN"

# ---- stable fake ids (32-hex keystone shape; 36-char magnum cluster uuid) ----
export CUID="1a000000000000000000000000000001"    # tenant -cluster user id (in cred file)
export PID="b0000000000000000000000000000001"     # tenant project id
export UDOM="d0000000000000000000000000000001"    # tenant user domain id
export CUUID="11111111-2222-3333-4444-555555555555"  # cluster uuid
export TRID="f0000000000000000000000000000001"    # per-cluster trustee user id
export MAGDOM="e0000000000000000000000000000001"  # magnum domain id

cat > "$BIN/openstack" <<'FB'
#!/usr/bin/env bash
# Identity comes from the env the script's *_env helpers set up.
if [ -n "${OS_APPLICATION_CREDENTIAL_ID:-}" ]; then WHO=foil
elif [ "${OS_USERNAME:-}" = "testco-cluster" ]; then WHO=tenant
else WHO=admin; fi
case "$WHO:$*" in
  "tenant:coe cluster show testco-cluster -f value -c health_status") cat "$FIXDIR/health.txt" ;;
  "tenant:coe cluster show testco-cluster -f value -c uuid")          cat "$FIXDIR/uuid.txt" ;;
  "tenant:coe cluster config testco-cluster --dir "*" --force")
      D=""; PREV=""; for a in "$@"; do [ "$PREV" = "--dir" ] && D="$a"; PREV="$a"; done
      printf 'apiVersion: v1\nkind: Config\n' > "$D/config"
      echo "Saved kubeconfig at $D/config" ;;
  "admin:trust list -f json")
      # Truthful mock: the real client emits SDK/deprecation notices on stderr.
      # This is what exercises the DOCFIX-094 stdout-only capture at acceptance:43.
      echo "WARNING: this fake emits a deprecation-style notice on stderr" >&2
      cat "$FIXDIR/trusts.json" ;;
  "admin:user show $TRID -f value -c name")      echo "${CUUID}_${PID}" ;;
  "admin:user show $TRID -f value -c domain_id") echo "$MAGDOM" ;;
  "admin:domain show magnum -f value -c id")     echo "$MAGDOM" ;;
  "admin:loadbalancer list -f value -c name -c provisioning_status -c operating_status")
      echo "kube_service_${CUUID}_default_lbtest ACTIVE ONLINE" ;;
  "foil:token issue -f value -c project_id")     cat "$FIXDIR/foil-token.txt" ;;
  "foil:coe cluster list -f value -c name")      cat "$FIXDIR/foil-clusters.txt" ;;
  "foil:coe cluster show $CUUID -f value -c name")
      cat "$FIXDIR/foil-show.txt"; exit "$(cat "$FIXDIR/foil-show-rc.txt")" ;;
  "foil:network show testco-net -f value -c id")
      echo "Error while executing command: No Network found for 'testco-net'" >&2; exit 1 ;;
  "foil:server list -f value -c Name")           : ;;
  "foil:loadbalancer list -f value -c name")     : ;;
  *) echo "fake-openstack: unmatched [$WHO]: $*" >&2; exit 1 ;;
esac
FB
chmod +x "$BIN/openstack"

cat > "$BIN/kubectl" <<'FB'
#!/usr/bin/env bash
case "$*" in
  "get nodes -o wide")             cat "$FIXDIR/nodes.txt" ;;
  "get pods -A --no-headers")      cat "$FIXDIR/pods.txt" ;;
  "create deployment lbtest "*)    echo "deployment.apps/lbtest created" ;;
  "expose deployment lbtest "*)    echo "service/lbtest exposed" ;;
  "get svc lbtest -o jsonpath="*)  cat "$FIXDIR/lbip.txt" ;;
  "describe svc lbtest")           echo "Events: fake describe output" ;;
  *) echo "fake-kubectl: unmatched: $*" >&2; exit 1 ;;
esac
FB
chmod +x "$BIN/kubectl"

cat > "$BIN/curl" <<'FB'
#!/usr/bin/env bash
case "$*" in
  *"/hostname") cat "$FIXDIR/curl.txt" ;;
  *) echo "fake-curl: unmatched: $*" >&2; exit 1 ;;
esac
FB
chmod +x "$BIN/curl"

printf '#!/usr/bin/env bash\nexit 0\n' > "$BIN/sleep"; chmod +x "$BIN/sleep"

mk_home(){ # mk_home <dir> [cred: good|none]
  local h="$1" cred="${2:-good}"
  mkdir -p "$h/vault-init"
  printf 'export OS_AUTH_URL=https://fake:5000/v3\nexport OS_USERNAME=admin\n' > "$h/admin-openrc"
  touch "$h/vault-init/vault-ca-root.pem"
  printf 'id="ac00000000000000000000000000000001"\nsecret="fake-foil-secret"\n' > "$h/foil-appcred.txt"
  chmod 600 "$h/foil-appcred.txt"
  if [ "$cred" = "good" ]; then
    local d="$h/tenant-testco"; mkdir -p "$d"; chmod 700 "$d"
    { echo "auth_url=https://fake:5000/v3"
      echo "user_id=$CUID"
      echo "user_domain_id=$UDOM"
      echo "project_id=$PID"
      echo "password=x"; } > "$d/testco-cluster-cred.txt"
    chmod 600 "$d/testco-cluster-cred.txt"
  fi
}

mk_fix(){ # mk_fix <dir>  (happy defaults; cases override individual files)
  local d="$1"; mkdir -p "$d"
  echo "HEALTHY"  > "$d/health.txt"
  echo "$CUUID"   > "$d/uuid.txt"
  printf '[{"ID":"tr01","Trustor User ID":"%s","Trustee User ID":"%s"}]\n' "$CUID" "$TRID" > "$d/trusts.json"
  { echo "NAME STATUS ROLES AGE VERSION"
    echo "testco-cluster-node-0 Ready master 10m v1.28.9"
    echo "testco-cluster-node-1 Ready worker 10m v1.28.9"; } > "$d/nodes.txt"
  { echo "kube-system coredns-5d8b 1/1 Running 0 9m"
    echo "kube-system openstack-cloud-controller-manager-x 1/1 Running 0 9m"; } > "$d/pods.txt"
  echo "10.99.0.7" > "$d/lbip.txt"
  echo "lbtest-6d9f7c-abcde" > "$d/curl.txt"
  echo "cc000000000000000000000000000001" > "$d/foil-token.txt"
  : > "$d/foil-clusters.txt"
  printf 'Cluster %s could not be found (HTTP 404)\n' "$CUUID" > "$d/foil-show.txt"
  echo 1 > "$d/foil-show-rc.txt"
}

run_case(){ # run_case <home> <fixdir> [args...] -> OUT/RC globals
  OUT=$(env HOME="$1" FIXDIR="$2" PATH="$BIN:$PATH" \
        bash "$REPO/scripts/tenant-acceptance.sh" "${@:3}" 2>&1); RC=$?
}
check(){ # check <want_rc> <regex> <label>
  if [ "$RC" = "$1" ] && grep -qE "$2" <<<"$OUT"; then echo "  PASS  $3"; PASS=$((PASS+1))
  else echo "  FAIL  $3 (rc=$RC want=$1)"; echo "$OUT" | tail -6 | sed 's/^/        /'; FAIL=$((FAIL+1)); fi
}
check_not(){ # check_not <regex> <label> -- OUT must NOT match
  if grep -qE "$1" <<<"$OUT"; then echo "  FAIL  $2 (forbidden match)"; echo "$OUT" | grep -E "$1" | head -2 | sed 's/^/        /'; FAIL=$((FAIL+1))
  else echo "  PASS  $2"; PASS=$((PASS+1)); fi
}

# T1 happy path end-to-end -> rc 0. The trust-list fake ALWAYS emits stderr noise,
# so T1b/T1c/T1d are the DOCFIX-094 regression proof: pre-fix (2>&1 merge) the JSON
# parse died, TRUSTEE fell to '', and 'trustee name: PASS' never printed.
mk_home "$TMP/h1"; mk_fix "$TMP/f1"
run_case "$TMP/h1" "$TMP/f1" testco "$TMP/h1/foil-appcred.txt"
check 0 'RESULT: t3-01 TENANT ACCEPTANCE PASS' "T1a happy path passes end-to-end"
check 0 'trustee name: PASS \(per-cluster' "T1b trustee resolved despite stderr noise (DOCFIX-094)"
check 0 'NOTE: trust list stderr: WARNING' "T1c trust-list stderr surfaced, not discarded"
check 0 'trustee domain: PASS \(magnum domain\)' "T1d trustee domain matches magnum"
check_not 'VIOLATION|trustee: WARN could not resolve' "T1e no violation, no trustee-resolve WARN"

# T2 health not yet HEALTHY -> WARN only, never a gate failure (rc stays 0)
mk_home "$TMP/h2"; mk_fix "$TMP/f2"; echo "CREATE_IN_PROGRESS" > "$TMP/f2/health.txt"
run_case "$TMP/h2" "$TMP/f2" testco "$TMP/h2/foil-appcred.txt"
check 0 'health: WARN \(not HEALTHY yet' "T2 health lag is WARN-only, acceptance still passes"

# T3 no args -> precondition 14
mk_home "$TMP/h3"; mk_fix "$TMP/f3"
run_case "$TMP/h3" "$TMP/f3"
check 14 'usage: tenant-acceptance.sh' "T3 missing client arg is a precondition (14)"

# T4 missing tenant cred file -> precondition 14
mk_home "$TMP/h4" none; mk_fix "$TMP/f4"
run_case "$TMP/h4" "$TMP/f4" testco "$TMP/h4/foil-appcred.txt"
check 14 'PRECOND: no .*testco-cluster-cred.txt' "T4 missing cred file is a precondition (14)"

# T5 uuid capture polluted (error text instead of a uuid) -> precondition 14
mk_home "$TMP/h5"; mk_fix "$TMP/f5"
echo "Cluster testco-cluster could not be found (HTTP 404)" > "$TMP/f5/uuid.txt"
run_case "$TMP/h5" "$TMP/f5" testco "$TMP/h5/foil-appcred.txt"
check 14 'PRECOND: bad beta uuid' "T5 non-uuid cluster-show output fails closed (14)"

# T6 <2 Ready nodes -> kube fail 11
mk_home "$TMP/h6"; mk_fix "$TMP/f6"
{ echo "NAME STATUS ROLES AGE VERSION"
  echo "testco-cluster-node-0 Ready master 10m v1.28.9"
  echo "testco-cluster-node-1 NotReady worker 10m v1.28.9"; } > "$TMP/f6/nodes.txt"
run_case "$TMP/h6" "$TMP/f6" testco "$TMP/h6/foil-appcred.txt"
check 11 'FAIL: expected >=2 Ready nodes, got 1' "T6 <2 Ready nodes fails P1 (11)"

# T7 EXTERNAL-IP never assigned -> LB fail 12 (fake sleep compresses the 24x25s loop)
mk_home "$TMP/h7"; mk_fix "$TMP/f7"; : > "$TMP/f7/lbip.txt"
run_case "$TMP/h7" "$TMP/f7" testco "$TMP/h7/foil-appcred.txt"
check 12 'FAIL: no EXTERNAL-IP within' "T7 pending EXTERNAL-IP fails P2 (12)"

# T8 LB up but not serving the pod hostname -> LB fail 12
mk_home "$TMP/h8"; mk_fix "$TMP/f8"; echo "curl: (7) Failed to connect" > "$TMP/f8/curl.txt"
run_case "$TMP/h8" "$TMP/f8" testco "$TMP/h8/foil-appcred.txt"
check 12 'FAIL: LB not serving pod hostname' "T8 dead LB endpoint fails P2 (12)"

# T9 foil auth failure -> precondition 14 (isolation probe needs a working foil)
mk_home "$TMP/h9"; mk_fix "$TMP/f9"
echo "The request you have made requires authentication. (HTTP 401)" > "$TMP/f9/foil-token.txt"
run_case "$TMP/h9" "$TMP/f9" testco "$TMP/h9/foil-appcred.txt"
check 14 'PRECOND: foil auth failed' "T9 foil auth failure is a precondition (14)"

# T10 foil sees the tenant cluster in a LIST -> isolation violation 13
mk_home "$TMP/h10"; mk_fix "$TMP/f10"; echo "testco-cluster" > "$TMP/f10/foil-clusters.txt"
run_case "$TMP/h10" "$TMP/f10" testco "$TMP/h10/foil-appcred.txt"
check 13 'VIOLATION: foil sees testco-cluster' "T10a foil list visibility trips the gate"
check 13 'P3: ISOLATION VIOLATION -- CRITICAL' "T10b isolation violation exits 13"

# T11 foil can READ the tenant cluster by uuid -> isolation violation 13
mk_home "$TMP/h11"; mk_fix "$TMP/f11"
echo "testco-cluster" > "$TMP/f11/foil-show.txt"; echo 0 > "$TMP/f11/foil-show-rc.txt"
run_case "$TMP/h11" "$TMP/f11" testco "$TMP/h11/foil-appcred.txt"
check 13 'VIOLATION: foil can read testco cluster' "T11 foil direct read trips the gate (13)"

echo; echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1