Newer
Older
openstack-caracal-ipv4 / tests / clientdocs-scripts / run-tests.sh
#!/usr/bin/env bash
# tests/clientdocs-scripts/run-tests.sh -- offline harness for the client
# starter kit under clientdocs/scripts/ (smoke-test.sh, acceptance-run.sh,
# ci-cleanup-sweep.sh, clouds.yaml.template).
#
# A fake openstack CLI on PATH replays fixture behavior and logs every call
# to $FAKELOG; resource existence lives as files under a per-case $STATE dir
# so creates/deletes/lists stay consistent (the acceptance teardown leftover
# check reads post-teardown reality). Several fakes deliberately emit notices
# ON STDERR, so the happy paths prove the scripts' stdout-only JSON captures.
# A fake sleep compresses the 10s poll loops. No cloud access; mutates
# NOTHING outside mktemp dirs.
#
# Cases: smoke pass / auth-fail triage / missing env; acceptance happy path
# (expected-refusal A6 passes, teardown clean + reverse-ordered), broken
# refusal boundary (A6 fails, leaked user removed), midway server failure
# (dependent rows SKIP, EXIT-trap teardown still runs); sweep dry-run vs
# --apply, age filter, detached-only volume rule, floating-ip description
# matching, prefix isolation; clouds.yaml.template placeholder inventory
# matches the CI integration guide.
# Exit: 0 all pass | 1 any case failed.  ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
KIT="$REPO/clientdocs/scripts"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0; CN=0
BIN="$TMP/bin"; mkdir -p "$BIN"

# ---------------- fake openstack (stateful; logs every call) ----------------
cat > "$BIN/openstack" <<'FB'
#!/usr/bin/env bash
set -u
echo "$*" >> "$FAKELOG"
err(){ echo "$1" >&2; exit 1; }
meta(){ cat "$STATE/$1" 2>/dev/null || true; }
names_json(){ # names_json <kind> <idkey> <namekey> -- list JSON from state files
  local k="$1" ik="$2" nk="$3" first=1 f n
  printf '['
  for f in "$STATE/$k".*; do
    [ -e "$f" ] || continue
    n="${f##*/}"; n="${n#"$k".}"
    [ "$first" = 1 ] || printf ','
    first=0
    printf '{"%s":"%s","%s":"%s"}' "$ik" "$n" "$nk" "$n"
  done
  printf ']\n'
}
case "$*" in
  "token issue -f json")
      echo "SDK notice: fake always writes this line on stderr" >&2
      [ -e "$FIXDIR/fail-auth" ] && err "The request you have made requires authentication. (HTTP 401)"
      printf '{"expires":"2027-01-01T00:00:00+0000","id":"gAAAAfaketoken","project_id":"b0000000000000000000000000000001","user_id":"u0000000000000000000000000000001"}\n' ;;
  "catalog list -f json")
      echo "WARNING: fake deprecation-style notice on stderr" >&2
      printf '[{"Name":"keystone","Type":"identity"},{"Name":"nova","Type":"compute"},{"Name":"glance","Type":"image"},{"Name":"neutron","Type":"network"},{"Name":"cinderv3","Type":"volumev3"},{"Name":"octavia","Type":"load-balancer"},{"Name":"barbican","Type":"key-manager"},{"Name":"placement","Type":"placement"}]\n' ;;
  "limits show --absolute -f json")
      printf '[{"Name":"maxTotalInstances","Value":10},{"Name":"totalInstancesUsed","Value":1},{"Name":"maxTotalCores","Value":20},{"Name":"totalCoresUsed","Value":2},{"Name":"maxTotalRAMSize","Value":51200},{"Name":"totalRAMUsed","Value":2048}]\n' ;;
  "user create ci-accept-probe -f json")
      if [ -e "$FIXDIR/refusal-broken" ]; then
        : > "$STATE/user.ci-accept-probe"
        printf '{"id":"u-probe","name":"ci-accept-probe"}\n'
      else
        err "You are not authorized to perform the requested action: identity:create_user. (HTTP 403)"
      fi ;;
  "user delete ci-accept-probe") rm -f "$STATE/user.ci-accept-probe" ;;
  "project show b0000000000000000000000000000001 -f json")
      printf '{"id":"b0000000000000000000000000000001","name":"testco-prod"}\n' ;;
  "network show testco-net -f json")
      printf '{"id":"net-handover0001","name":"testco-net","created_at":"2026-01-01T00:00:00Z"}\n' ;;
  "router show testco-router -f json")
      printf '{"id":"rtr-handover0001","name":"testco-router","external_gateway_info":{"network_id":"extnet0000000001"}}\n' ;;
  "network show provider-ext -f json")
      printf '{"id":"extnet0000000001","name":"provider-ext"}\n' ;;
  "network create ci-accept-net -f json")
      : > "$STATE/network.ci-accept-net"
      printf '{"id":"net-ci-accept0001","name":"ci-accept-net"}\n' ;;
  "subnet create ci-accept-subnet --network ci-accept-net --subnet-range "*" -f json")
      : > "$STATE/subnet.ci-accept-subnet"
      printf '{"id":"sub-ci-accept0001","name":"ci-accept-subnet"}\n' ;;
  "security group create ci-accept-sg -f json")
      : > "$STATE/secgroup.ci-accept-sg"
      printf '{"id":"sg-ci-accept0001","name":"ci-accept-sg"}\n' ;;
  "security group rule create ci-accept-sg "*)
      printf '{"id":"sgr-ci-accept0001"}\n' ;;
  "floating ip create provider-ext --description ci-accept-fip -f json")
      printf 'now|ci-accept-fip|-' > "$STATE/fip.fip0001"
      printf '{"id":"fip0001","floating_ip_address":"203.0.113.10","description":"ci-accept-fip"}\n' ;;
  "image list -f json")
      printf '[{"ID":"img1","Name":"fedora-40","Status":"active"},{"ID":"img2","Name":"ubuntu-24.04","Status":"active"}]\n' ;;
  "flavor list -f json")
      printf '[{"ID":"f1","Name":"m1.small","RAM":2048,"VCPUs":1},{"ID":"f0","Name":"m1.tiny","RAM":512,"VCPUs":1}]\n' ;;
  "keypair create ci-accept-key")
      : > "$STATE/keypair.ci-accept-key"
      printf -- '-----BEGIN FAKE PRIVATE KEY-----\nnot-a-real-key\n-----END FAKE PRIVATE KEY-----\n' ;;
  "keypair delete ci-accept-key") rm -f "$STATE/keypair.ci-accept-key" ;;
  "server create ci-accept-vm "*)
      [ -e "$FIXDIR/fail-server" ] && err "Quota exceeded for instances: Requested 1, but already used 10 of 10 instances (HTTP 403)"
      : > "$STATE/server.ci-accept-vm"
      printf '{"id":"srv-ci-accept0001","name":"ci-accept-vm","status":"ACTIVE"}\n' ;;
  "server show ci-accept-vm -f json")
      [ -e "$STATE/server.ci-accept-vm" ] || err "No server with a name or ID of 'ci-accept-vm' exists."
      printf '{"id":"srv-ci-accept0001","name":"ci-accept-vm","status":"ACTIVE","created":"2026-07-06T00:00:00Z","addresses":{"ci-accept-net":["192.168.199.11"]}}\n' ;;
  "volume create --size 1 ci-accept-vol -f json")
      echo creating > "$STATE/volstatus"
      printf 'now|creating' > "$STATE/volume.ci-accept-vol"
      printf '{"id":"vol-ci-accept0001","name":"ci-accept-vol","status":"creating"}\n' ;;
  "volume show ci-accept-vol -f json")
      [ -e "$STATE/volume.ci-accept-vol" ] || err "No volume with a name or ID of 'ci-accept-vol' exists."
      ST=$(cat "$STATE/volstatus")
      [ "$ST" = "creating" ] && echo available > "$STATE/volstatus"  # progresses next call
      printf '{"id":"vol-ci-accept0001","name":"ci-accept-vol","status":"%s"}\n' "$ST" ;;
  "server add volume ci-accept-vm ci-accept-vol")    echo in-use > "$STATE/volstatus" ;;
  "server remove volume ci-accept-vm ci-accept-vol") echo available > "$STATE/volstatus" ;;
  "loadbalancer create --name ci-accept-lb --vip-subnet-id ci-accept-subnet --wait -f json")
      echo "octavia client notice on stderr" >&2
      : > "$STATE/lb.ci-accept-lb"
      printf '{"id":"lb-ci-accept0001","name":"ci-accept-lb","provisioning_status":"ACTIVE","operating_status":"ONLINE"}\n' ;;
  "loadbalancer show ci-accept-lb -f json")
      [ -e "$STATE/lb.ci-accept-lb" ] || err "Unable to locate load balancer ci-accept-lb"
      printf '{"id":"lb-ci-accept0001","name":"ci-accept-lb","provisioning_status":"ACTIVE","created_at":"2026-07-06T00:00:00Z"}\n' ;;
  "loadbalancer listener create --name ci-accept-listener "*) printf '{"id":"lsn0001","name":"ci-accept-listener"}\n' ;;
  "loadbalancer pool create --name ci-accept-pool "*)         printf '{"id":"pool0001","name":"ci-accept-pool"}\n' ;;
  "loadbalancer member create --name ci-accept-member "*)     printf '{"id":"mem0001","name":"ci-accept-member"}\n' ;;
  "secret store --name ci-accept-secret --payload "*" -f json")
      PREV=""; PAY=""
      for a in "$@"; do [ "$PREV" = "--payload" ] && PAY="$a"; PREV="$a"; done
      printf '%s\n' "$PAY" > "$STATE/secretpayload"
      : > "$STATE/secret.ci-accept-secret"
      printf '{"Secret href":"https://secrets.example/v1/secrets/href0001"}\n' ;;
  "secret get https://secrets.example/v1/secrets/href0001 --payload -f value -c Payload")
      cat "$STATE/secretpayload" ;;
  "secret delete https://secrets.example/v1/secrets/href0001")
      rm -f "$STATE/secret.ci-accept-secret" ;;
  # ---- deletes (shared by acceptance teardown and the sweep) ----
  "loadbalancer delete "*" --cascade --wait") rm -f "$STATE/lb.$3" ;;
  "floating ip delete "*)                     rm -f "$STATE/fip.$4" ;;
  "server delete "*" --wait")                 rm -f "$STATE/server.$3" ;;
  "volume delete "*)                          rm -f "$STATE/volume.$3" ;;
  "security group delete "*)                  rm -f "$STATE/secgroup.$4" ;;
  "subnet delete "*)                          rm -f "$STATE/subnet.$3" ;;
  "network delete "*)                         rm -f "$STATE/network.$3" ;;
  # ---- lists (built from state so post-teardown reality is truthful) ----
  "server list -f json")         names_json server ID Name ;;
  "volume list -f json")         names_json volume ID Name ;;
  "network list -f json")        names_json network ID Name ;;
  "security group list -f json") names_json secgroup ID Name ;;
  "loadbalancer list -f json")   names_json lb id name ;;   # octavia lists use lowercase keys
  "floating ip list -f json")    names_json fip ID Name ;;
  # ---- generic shows (sweep age/status checks; state file carries metadata) ----
  "server show "*" -f json")
      [ -e "$STATE/server.$3" ] || err "No server with a name or ID of '$3' exists."
      printf '{"id":"%s","name":"%s","status":"ACTIVE","created":"%s"}\n' "$3" "$3" "$(meta "server.$3")" ;;
  "loadbalancer show "*" -f json")
      printf '{"id":"%s","name":"%s","provisioning_status":"ACTIVE","created_at":"%s"}\n' "$3" "$3" "$(meta "lb.$3")" ;;
  "volume show "*" -f json")
      IFS='|' read -r TS VST < "$STATE/volume.$3" || true
      printf '{"id":"%s","name":"%s","status":"%s","created_at":"%s"}\n' "$3" "$3" "$VST" "$TS" ;;
  "floating ip show "*" -f json")
      IFS='|' read -r TS DESC PORT < "$STATE/fip.$4" || true
      [ "${PORT:-}" = "-" ] && PORT=""
      if [ -n "${PORT:-}" ]; then PJ="\"$PORT\""; else PJ=null; fi
      printf '{"id":"%s","floating_ip_address":"203.0.113.5","description":"%s","port_id":%s,"created_at":"%s"}\n' "$4" "${DESC:-}" "$PJ" "$TS" ;;
  "security group show "*" -f json")
      printf '{"id":"%s","name":"%s","created_at":"%s"}\n' "$4" "$4" "$(meta "secgroup.$4")" ;;
  "network show "*" -f json")
      printf '{"id":"%s","name":"%s","created_at":"%s"}\n' "$3" "$3" "$(meta "network.$3")" ;;
  *) echo "fake-openstack: unmatched: $*" >&2; exit 1 ;;
esac
exit 0
FB
chmod +x "$BIN/openstack"
printf '#!/usr/bin/env bash\nexit 0\n' > "$BIN/sleep"; chmod +x "$BIN/sleep"

# ---------------- harness helpers ----------------
newcase() { CN=$((CN+1)); CASEDIR="$TMP/case$CN"; mkdir -p "$CASEDIR/fix" "$CASEDIR/state"
            FAKELOG="$CASEDIR/fakelog"; : > "$FAKELOG"; }
run_kit() { # run_kit <script> [args...] -> OUT/RC
  OUT=$(env OS_CLOUD=testco FIXDIR="$CASEDIR/fix" STATE="$CASEDIR/state" FAKELOG="$FAKELOG" \
        PATH="$BIN:$PATH" bash "$KIT/$1" "${@:2}" 2>&1); RC=$?
}
check() { # check <want_rc> <regex> <label>
  if [ "$RC" = "$1" ] && grep -qE "$2" <<<"$OUT"; then echo "  PASS  $3"; PASS=$((PASS+1))
  else echo "  FAIL  $3 (rc=$RC want=$1)"; echo "$OUT" | tail -8 | sed 's/^/        /'; FAIL=$((FAIL+1)); fi
}
check_not() { # check_not <regex> <label> -- OUT must NOT match
  if grep -qE "$1" <<<"$OUT"; then echo "  FAIL  $2 (forbidden match)"
    grep -E "$1" <<<"$OUT" | head -2 | sed 's/^/        /'; FAIL=$((FAIL+1))
  else echo "  PASS  $2"; PASS=$((PASS+1)); fi
}
log_has() { # log_has <regex> <label>
  if grep -qE "$1" "$FAKELOG"; then echo "  PASS  $2"; PASS=$((PASS+1))
  else echo "  FAIL  $2 (no '$1' in fake log)"; FAIL=$((FAIL+1)); fi
}
log_not() { # log_not <regex> <label>
  if grep -qE "$1" "$FAKELOG"; then echo "  FAIL  $2 (forbidden call in fake log)"
    grep -E "$1" "$FAKELOG" | head -2 | sed 's/^/        /'; FAIL=$((FAIL+1))
  else echo "  PASS  $2"; PASS=$((PASS+1)); fi
}

# ---------------- smoke-test.sh ----------------
newcase; run_kit smoke-test.sh
check 0 'RESULT: SMOKE TEST PASS' "S1 smoke happy path passes (stderr noise tolerated)"
check 0 'instances 1/10, cores 2/20' "S2 smoke reports the quota envelope"

newcase; : > "$CASEDIR/fix/fail-auth"; run_kit smoke-test.sh
check 1 'revoked or has expired' "S3 smoke auth failure fails with triage hints"
check 1 'EXACT command' "S4 smoke failure points at the verbatim-error rule"

newcase
OUT=$(env OS_CLOUD= OS_AUTH_URL= FIXDIR="$CASEDIR/fix" STATE="$CASEDIR/state" FAKELOG="$FAKELOG" \
      PATH="$BIN:$PATH" bash "$KIT/smoke-test.sh" 2>&1); RC=$?
check 2 'no OpenStack environment' "S5 smoke without an environment exits 2"

# ---------------- acceptance-run.sh ----------------
newcase; run_kit acceptance-run.sh
check 0 'RESULT: ACCEPTANCE \(automated rows\) PASS' "A1 acceptance happy path passes end-to-end"
check 0 'A6 +PASS +identity create refused as expected' "A2 expected-refusal row fails-to-PASS"
check 0 'C1 +PASS +handover network and router intact' "A3 handover check discovered from project name"
check 0 'teardown: CLEAN' "A4 teardown reports clean"
check 0 'rows: 15 PASS, 0 FAIL, 0 SKIP' "A5 all 15 automated rows pass"
check_not 'TEARDOWN FAIL|leftover ci-accept' "A6 no teardown failure or leftover"
L_LB=$(grep -n '^loadbalancer delete ci-accept-lb' "$FAKELOG" | cut -d: -f1 | head -1)
L_SRV=$(grep -n '^server delete ci-accept-vm' "$FAKELOG" | cut -d: -f1 | head -1)
L_NET=$(grep -n '^network delete ci-accept-net' "$FAKELOG" | cut -d: -f1 | head -1)
if [ -n "$L_LB" ] && [ -n "$L_SRV" ] && [ -n "$L_NET" ] && [ "$L_LB" -lt "$L_SRV" ] && [ "$L_SRV" -lt "$L_NET" ]; then
  echo "  PASS  A7 teardown runs in reverse order (lb < server < network)"; PASS=$((PASS+1))
else
  echo "  FAIL  A7 teardown order wrong (lb=$L_LB server=$L_SRV net=$L_NET)"; FAIL=$((FAIL+1))
fi
log_has '^secret delete ' "A8 stored secret is deleted (G1 includes its own delete)"

newcase; : > "$CASEDIR/fix/refusal-broken"; run_kit acceptance-run.sh
check 1 'A6 +FAIL +user create unexpectedly SUCCEEDED' "A9 broken boundary makes A6 FAIL (fail-to-pass inverted correctly)"
log_has '^user delete ci-accept-probe' "A10 leaked probe user is removed at teardown"

newcase; : > "$CASEDIR/fix/fail-server"; run_kit acceptance-run.sh
check 1 'D2 +FAIL +server create failed: Quota exceeded' "A11 midway server failure marks D2 FAIL, run continues"
check 1 'E2 +SKIP +blocked' "A12 dependent row E2 SKIPs instead of crashing"
check 1 'F2 +SKIP +blocked' "A13 dependent row F2 SKIPs instead of crashing"
check 1 'teardown: CLEAN' "A14 teardown still clean after midway failure"
log_has '^network delete ci-accept-net' "A15 EXIT trap tears the network down despite the failure"
log_has '^keypair delete ci-accept-key' "A16 EXIT trap removes the keypair created before the failure"
log_not '^server delete ci-accept-vm' "A17 no delete attempted for the server that never existed"

newcase; : > "$CASEDIR/fix/fail-auth"; run_kit acceptance-run.sh
check 1 'blocked: authentication \(A1\) failed' "A18 auth failure blocks the remaining rows"
log_not ' delete ' "A19 nothing is deleted when nothing was created"

# ---------------- ci-cleanup-sweep.sh ----------------
seed_sweep() { # aged fixture population; ci-* old and new, plus non-matching names
  local S="$CASEDIR/state" OLD NEW
  OLD=$(date -u -d "-36 hours" +%Y-%m-%dT%H:%M:%SZ)
  NEW=$(date -u -d "-1 hours" +%Y-%m-%dT%H:%M:%SZ)
  printf '%s' "$OLD" > "$S/server.ci-old-vm"
  printf '%s' "$OLD" > "$S/server.prod-db"
  printf '%s' "$NEW" > "$S/server.ci-new-vm"
  printf '%s' "$OLD" > "$S/lb.ci-old-lb"
  printf '%s|available' "$OLD" > "$S/volume.ci-old-vol"
  printf '%s|in-use'    "$OLD" > "$S/volume.ci-busy-vol"
  printf '%s|ci-accept-fip|-' "$OLD"        > "$S/fip.fipA"   # matching desc, detached, old
  printf '%s||port-in-use-01' "$OLD"        > "$S/fip.fipB"   # no description: never touched
  printf '%s|ci-accept-fip|port-in-use-02' "$OLD" > "$S/fip.fipC"  # matching but attached
  printf '%s' "$OLD" > "$S/secgroup.ci-old-sg"
  printf '%s' "$OLD" > "$S/secgroup.default"
  printf '%s' "$OLD" > "$S/network.ci-old-net"
  printf '%s' "$OLD" > "$S/network.testco-net"
}

newcase; seed_sweep; run_kit ci-cleanup-sweep.sh
check 0 'WOULD DELETE server ci-old-vm' "W1 dry-run reports the aged ci- server"
check 0 'keep server ci-new-vm \(younger' "W2 dry-run keeps the young ci- server (age filter)"
check 0 'keep volume ci-busy-vol \(status .in-use' "W3 dry-run keeps the attached volume"
check_not 'DELETE.*(prod-db|testco-net|default|fipB)' "W4 dry-run never proposes non-matching names"
log_not ' delete ' "W5 dry-run issues no delete calls at all"

newcase; seed_sweep; run_kit ci-cleanup-sweep.sh --apply
check 0 'DELETED server ci-old-vm' "W6 --apply deletes the aged ci- server"
check 0 'DELETED load balancer ci-old-lb \(cascade\)' "W7 --apply cascade-deletes the aged ci- load balancer"
check 0 'DELETED floating ip .* \(released\)' "W8 --apply releases the matching detached floating ip"
check 0 'keep floating ip .* \(still attached\)' "W9 --apply keeps the attached floating ip"
check 0 'RESULT: SWEEP OK' "W10 --apply run ends OK"
log_has '^server delete ci-old-vm --wait' "W11 server delete issued with --wait"
log_has '^floating ip delete fipA' "W12 floating ip delete targets only fipA"
log_not '(delete prod-db|delete testco-net|delete default|delete ci-new-vm|delete ci-busy-vol|delete fipB|delete fipC)' \
        "W13 prefix/age/attachment isolation: nothing else deleted"

newcase; seed_sweep; run_kit ci-cleanup-sweep.sh --prefix nomatch- --apply
check 0 'SWEEP SUMMARY: 0 resource\(s\) deleted' "W14 non-matching prefix deletes nothing"
log_not ' delete ' "W15 non-matching prefix issues no delete calls"

newcase; run_kit ci-cleanup-sweep.sh --hours notanumber
check 2 'hours must be a number' "W16 bad --hours value is a usage error (2)"
newcase; run_kit ci-cleanup-sweep.sh --bogus
check 2 'unknown argument' "W17 unknown flag is a usage error (2)"

# ---------------- clouds.yaml.template placeholder inventory ----------------
TPL="$KIT/clouds.yaml.template"
GUIDE="$REPO/clientdocs/ci-integration-guide.md"
GOT=$(grep -o '{{[A-Z_]*}}' "$TPL" | sort -u | tr '\n' ' ')
WANT="{{AUTH_URL}} {{CA_BUNDLE_FILE}} {{REGION}} {{TENANT_SHORT_NAME}} "
if [ "$GOT" = "$WANT" ]; then echo "  PASS  P1 template placeholder set is exactly the guide's four"; PASS=$((PASS+1))
else echo "  FAIL  P1 placeholder set mismatch (got: $GOT)"; FAIL=$((FAIL+1)); fi
P_OK=1
for ph in '{{TENANT_SHORT_NAME}}' '{{AUTH_URL}}' '{{REGION}}' '{{CA_BUNDLE_FILE}}'; do
  grep -qF "$ph" "$GUIDE" || { echo "  FAIL  P2 $ph missing from the CI guide"; FAIL=$((FAIL+1)); P_OK=0; }
done
[ "$P_OK" = 1 ] && { echo "  PASS  P2 every template placeholder appears in the CI guide"; PASS=$((PASS+1)); }
if grep -qF '<id from the delivered credential file>' "$TPL" \
   && grep -qF '<injected from your CI secret store>' "$TPL"; then
  echo "  PASS  P3 credential id/secret stay client-filled (never operator-prefilled)"; PASS=$((PASS+1))
else
  echo "  FAIL  P3 credential id/secret markers missing from the template"; FAIL=$((FAIL+1))
fi

# ---------------- static sanity: the kit scripts parse ----------------
for s in smoke-test.sh acceptance-run.sh ci-cleanup-sweep.sh; do
  if bash -n "$KIT/$s" 2>/dev/null; then echo "  PASS  N1 $s parses (bash -n)"; PASS=$((PASS+1))
  else echo "  FAIL  N1 $s does not parse"; FAIL=$((FAIL+1)); fi
done

echo; echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1