Newer
Older
openstack-caracal-ipv4 / tests / tenant-onboard / run-tests.sh
#!/usr/bin/env bash
# tests/tenant-onboard/run-tests.sh -- focused offline harness for guard behaviors:
#   - stage4 duplicate-CIDR guard (2026-07-03 sweep; was fail-open under SIGPIPE)
#   - stage3 H2 re-run guards (2026-07-06): app-cred/keypair exists + local secret
#     intact -> SKIP rc0; exists WITHOUT usable local secret -> STOP rc1 with
#     operator guidance (no implicit credential rotation on a re-run)
#   - stage7 H3 canary (2026-07-06): happy path CANARY PASS + teardown rc0;
#     ssh-dead path CANARY FAIL rc1 AND teardown still runs (trap EXIT)
# Fake openstack/ssh replay fixtures; fake sleep compresses wait loops; fake HOMEs
# provide stub admin-openrc + CA. Not a full-stage harness -- assertions target
# the guard/canary behaviors only.
# Exit: 0 all pass | 1 any case failed.  ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0
BIN="$TMP/bin"; H="$TMP/home"; mkdir -p "$BIN" "$H"
printf 'export OS_AUTH_URL=https://fake:5000/v3\nexport OS_CACERT=$HOME/vault-init/vault-ca-root.pem\n' > "$H/admin-openrc"
mkdir -p "$H/vault-init"
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -subj /CN=fake-ca \
  -keyout "$H/vault-init/ca.key" -out "$H/vault-init/vault-ca-root.pem" -days 2 >/dev/null 2>&1

DOMID="d0000000000000000000000000000001"
PID="b0000000000000000000000000000001"
ACID="e0000000000000000000000000000001"
CLUID="c0000000000000000000000000000002"
SVCID="c0000000000000000000000000000003"

cat > "$BIN/openstack" <<FB
#!/usr/bin/env bash
case "\$*" in
  *"subnet list"*)                                          cat "\$FIXDIR/subnets.txt" ;;
  "token issue -f value -c domain_id")                      echo "$DOMID" ;;
  "project create --domain $DOMID"*)                        : ;;
  "project show testco-prod --domain $DOMID -f value -c id") echo "$PID" ;;
  "user create --domain $DOMID"*)                           : ;;
  "user show testco-cluster --domain $DOMID -f value -c id") echo "$CLUID" ;;
  "user show testco-svc --domain $DOMID -f value -c id")    echo "$SVCID" ;;
  "role add --project $PID"*)                               : ;;
  "role assignment list --project $PID --user $SVCID --names -f value -c Role")
      RARC=\$(cat "\$FIXDIR/roleassign-rc" 2>/dev/null || echo 0)
      if [ "\$RARC" != 0 ]; then echo "An unexpected error prevented the server from fulfilling your request. (HTTP 500)" >&2; exit "\$RARC"; fi
      cat "\$FIXDIR/roleassign.txt" ;;
  "subnet show capi-mgmt-subnet -f value -c cidr")          cat "\$FIXDIR/capi-cidr.txt" ;;
  "domain show testco -f value -c id")                      echo "$DOMID" ;;
  "token issue -f value -c project_id")                     echo "$PID" ;;
  "application credential show testco-svc-cred -f value -c id") cat "\$FIXDIR/appcred-id.txt" ;;
  "keypair show testco-key -f value -c name")               cat "\$FIXDIR/keypair-name.txt" ;;
  "image list --public -f json")                            cat "\$FIXDIR/images.json" ;;
  "security group create testco-canary-sg"*)                : ;;
  "security group rule create testco-canary-sg"*)           : ;;
  "server create testco-canary"*)                           : ;;
  "server show testco-canary -f value -c status")           cat "\$FIXDIR/status.txt" ;;
  "floating ip create provider-ext -f value -c floating_ip_address") echo "10.99.0.5" ;;
  "server add floating ip testco-canary 10.99.0.5")         : ;;
  "server delete --wait testco-canary")                     echo "deleted" >> "\$FIXDIR/teardown.log" ;;
  "floating ip delete 10.99.0.5")                           echo "fip-del" >> "\$FIXDIR/teardown.log" ;;
  "security group delete testco-canary-sg")                 echo "sg-del"  >> "\$FIXDIR/teardown.log" ;;
  *) echo "fake-openstack: unmatched: \$*" >&2; exit 1 ;;
esac
FB
chmod +x "$BIN/openstack"
cat > "$BIN/ssh" <<'FB'
#!/usr/bin/env bash
exit "$(cat "$FIXDIR/sshrc" 2>/dev/null || echo 1)"
FB
chmod +x "$BIN/ssh"
cat > "$BIN/sleep" <<'FB'
#!/usr/bin/env bash
exit 0
FB
chmod +x "$BIN/sleep"

mk_case_home(){ # mk_case_home <dir> [seed: full|noac|nokey|none]
  local h="$1" seed="${2:-full}"
  mkdir -p "$h"; cp "$H/admin-openrc" "$h/"; cp -r "$H/vault-init" "$h/"
  local d="$h/tenant-testco"; mkdir -p "$d"; chmod 700 "$d"
  printf 'username=testco-svc\nproject_id=%s\npassword=x\n' "$PID" > "$d/testco-svc-cred.txt"
  printf 'username=testco-cluster\nproject_id=%s\npassword=x\n' "$PID" > "$d/testco-cluster-cred.txt"
  printf 'domain=testco\ndomain_id=%s\nusername=testco-domain-admin\npassword=x\n' "$DOMID" > "$d/testco-domain-admin-cred.txt"
  if [ "$seed" = "full" ] || [ "$seed" = "nokey" ]; then
    printf 'id="%s"\nsecret="fake-secret-value"\n' "$ACID" > "$d/testco-svc-appcred.txt"
  fi
  if [ "$seed" = "full" ] || [ "$seed" = "noac" ]; then
    printf -- '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n' > "$d/testco-key.pem"
  fi
  chmod 600 "$d"/* 2>/dev/null || true
}

run_stage(){ # run_stage <home> <fixdir> <stage> [cidr] -> OUT/RC
  OUT=$( cd "$TMP" && env HOME="$1" FIXDIR="$2" PATH="$BIN:$PATH" \
         TENANT_CIDR="${4:-10.20.24.0/24}" \
         bash "$REPO/scripts/tenant-onboard.sh" testco "$3" 2>&1 ); RC=$?
}
check(){ # check <want_rc|-> <want:regex> <must_not:regex|-> <label>
  local wrc="$1" want="$2" not="$3" label="$4"
  local ok=1
  [ "$wrc" = "-" ] || [ "$RC" = "$wrc" ] || ok=0
  grep -qE "$want" <<<"$OUT" || ok=0
  [ "$not" = "-" ] || { grep -qE "$not" <<<"$OUT" && ok=0; }
  if [ "$ok" = 1 ]; then echo "  PASS  $label"; PASS=$((PASS+1))
  else echo "  FAIL  $label (rc=$RC want=$wrc)"; echo "$OUT" | tail -5 | sed 's/^/        /'; FAIL=$((FAIL+1)); fi
}

# ---- stage4 CIDR guard (D-074: reserved-ranges-only, overlap math) ----
mkdir -p "$TMP/fx-collide" "$TMP/fx-clear" "$TMP/fx-capifail"
printf '10.20.7.0/24\n10.20.24.0/24\n10.20.9.0/24\n' > "$TMP/fx-collide/subnets.txt"
printf '10.20.7.0/24\n10.20.9.0/24\n'                > "$TMP/fx-clear/subnets.txt"
printf '10.20.0.0/24\n' > "$TMP/fx-collide/capi-cidr.txt"
printf '10.20.0.0/24\n' > "$TMP/fx-clear/capi-cidr.txt"
printf 'No Subnet found for capi-mgmt-subnet\n' > "$TMP/fx-capifail/capi-cidr.txt"

# T1 (D-074): tenant-vs-tenant EXACT overlap is now ALLOWED -- another tenant
# already holds 10.20.24.0/24 in the fixture, and the guard must NOT die.
run_stage "$H" "$TMP/fx-collide" stage4
check - '== stage4:' 'rejected|in use' "T1 tenant-vs-tenant exact overlap ALLOWED (D-074)"
# T2: disjoint RFC1918 range passes (no false die)
run_stage "$H" "$TMP/fx-clear" stage4 "10.100.0.0/24"
check - '== stage4:' 'rejected|in use' "T2 disjoint CIDR passes the guard"
# T3: PARTIAL overlap against a reserved plane range -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "10.12.5.0/24"
check - 'rejected: RESERVED-OVERLAP 10.12.4.0/22' '-' "T3 partial overlap vs reserved plane rejected"
[ "$RC" -ne 0 ] || { echo "  FAIL  T3b guard must exit non-zero"; FAIL=$((FAIL+1)); }
# T4: EXACT match on a reserved plane range -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "10.12.4.0/22"
check - 'rejected: RESERVED-OVERLAP 10.12.4.0/22' '-' "T4 exact reserved match rejected"
# T5: partial overlap against the live-resolved capi-mgmt net -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "10.20.0.128/25"
check - 'rejected: RESERVED-OVERLAP 10.20.0.0/24' '-' "T5 capi-mgmt overlap rejected (dynamic lookup)"
# T6: non-RFC1918 (TEST-NET, which python is_private would wrongly accept) -> REJECTED
run_stage "$H" "$TMP/fx-clear" stage4 "192.0.2.0/24"
check - 'rejected: NOT-RFC1918' '-' "T6 non-RFC1918 rejected"
# T7: capi-mgmt lookup failure -> guard fails CLOSED
run_stage "$H" "$TMP/fx-capifail" stage4 "10.100.0.0/24"
check - 'capi-mgmt-subnet CIDR lookup failed' '-' "T7 capi lookup miss fails closed"
[ "$RC" -ne 0 ] || { echo "  FAIL  T7b fail-closed must exit non-zero"; FAIL=$((FAIL+1)); }

# ---- stage3 H2 re-run guards ----
# T3 both exist remotely + both local files intact -> double SKIP, rc0
mk_case_home "$TMP/h3" full; mkdir -p "$TMP/fx3"
echo "$ACID"      > "$TMP/fx3/appcred-id.txt"
echo "testco-key" > "$TMP/fx3/keypair-name.txt"
run_stage "$TMP/h3" "$TMP/fx3" stage3
check 0 'app cred testco-svc-cred exists \+ local secret intact -- SKIP' '-' "T3a app-cred re-run SKIPs"
check 0 'keypair testco-key exists \+ local private key intact -- SKIP' '-' "T3b keypair re-run SKIPs"

# T4 app cred exists remotely, local secret file MISSING -> STOP rc1 with guidance
mk_case_home "$TMP/h4" noac; mkdir -p "$TMP/fx4"
echo "$ACID"      > "$TMP/fx4/appcred-id.txt"
echo "testco-key" > "$TMP/fx4/keypair-name.txt"
run_stage "$TMP/h4" "$TMP/fx4" stage3
check 1 'OPERATOR DECISION required \(rotates a live credential\)' '-' "T4 lost-secret re-run STOPs with guidance"

# T5 keypair exists remotely, local key MISSING -> STOP rc1 with guidance
mk_case_home "$TMP/h5" nokey; mkdir -p "$TMP/fx5"
printf 'x\n'      > "$TMP/fx5/appcred-id.txt"   # app cred absent -> would create; but skip via intact file
echo "$ACID"      > "$TMP/fx5/appcred-id.txt"
echo "testco-key" > "$TMP/fx5/keypair-name.txt"
run_stage "$TMP/h5" "$TMP/fx5" stage3
check 1 'OPERATOR DECISION required \(breaks existing node access\)' '-' "T5 lost-private-key re-run STOPs with guidance"

# ---- stage2 anti-escalation guard (capture-then-test; was a raced live pipe) ----
# T8a happy: no admin row -> anti-escalation OK, stage2 completes rc0
mk_case_home "$TMP/h8a" full; mkdir -p "$TMP/fx8a"
printf 'member\nload-balancer_member\n' > "$TMP/fx8a/roleassign.txt"
run_stage "$TMP/h8a" "$TMP/fx8a" stage2
check 0 'anti-escalation OK \(admin grant denied\)' '\*\*\* ESCALATION' "T8a stage2 happy: anti-escalation OK"
# T8b admin row present -> ESCALATION-STOP rc1 (pre-fix: SIGPIPE on the match
# could kill the pipe and print OK -- the guard failed OPEN)
mk_case_home "$TMP/h8b" full; mkdir -p "$TMP/fx8b"
printf 'admin\nmember\nload-balancer_member\n' > "$TMP/fx8b/roleassign.txt"
run_stage "$TMP/h8b" "$TMP/fx8b" stage2
check 1 '\*\*\* ESCALATION: manager granted admin -- STOP \*\*\*' 'anti-escalation OK' "T8b admin row triggers ESCALATION-STOP"
# T8c role-assignment list FAILS -> guard fails CLOSED (never prints OK)
mk_case_home "$TMP/h8c" full; mkdir -p "$TMP/fx8c"
printf 'member\n' > "$TMP/fx8c/roleassign.txt"; echo 1 > "$TMP/fx8c/roleassign-rc"
run_stage "$TMP/h8c" "$TMP/fx8c" stage2
check 1 'anti-escalation check UNVERIFIABLE .*HTTP 500' 'anti-escalation OK|\*\*\* ESCALATION: manager granted' "T8c failed list fails CLOSED, no OK"

# ---- stage7 H3 canary ----
mk_fix7(){ local d="$1"; mkdir -p "$d"
  printf '[{"Name":"fedora-coreos-kube-1.28","ID":"11111111-2222-3333-4444-555555555555"}]\n' > "$d/images.json"
  echo "ACTIVE" > "$d/status.txt"; : > "$d/teardown.log"
}
# T6 happy path: ACTIVE + ssh ok -> CANARY PASS, teardown ran, rc0
mk_case_home "$TMP/h6" full; mk_fix7 "$TMP/fx6"; echo 0 > "$TMP/fx6/sshrc"
run_stage "$TMP/h6" "$TMP/fx6" stage7
check 0 'CANARY PASS' '-' "T6a canary happy path passes"
check 0 'teardown done' '-' "T6b teardown ran on success"
grep -q 'deleted' "$TMP/fx6/teardown.log" && grep -q 'sg-del' "$TMP/fx6/teardown.log" \
  && { echo "  PASS  T6c teardown actually deleted server+SG"; PASS=$((PASS+1)); } \
  || { echo "  FAIL  T6c teardown log incomplete"; FAIL=$((FAIL+1)); }

# T7 ssh dead: CANARY FAIL rc1 AND teardown still ran (trap EXIT)
mk_case_home "$TMP/h7" full; mk_fix7 "$TMP/fx7"; echo 1 > "$TMP/fx7/sshrc"
run_stage "$TMP/h7" "$TMP/fx7" stage7
check 1 'CANARY FAIL: no SSH' '-' "T7a dead SSH fails the canary"
check 1 'teardown done' '-' "T7b teardown still ran on failure"
grep -q 'deleted' "$TMP/fx7/teardown.log" \
  && { echo "  PASS  T7c server deleted despite failure"; PASS=$((PASS+1)); } \
  || { echo "  FAIL  T7c no server delete on failure path"; FAIL=$((FAIL+1)); }

echo; echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1