#!/usr/bin/env bash
# tests/cloud-assert/run-tests.sh -- offline fakebin harness for cloud-assert.sh
# (DOCFIX-075). Fake juju/openstack/jq-free: real jq stays real; fake juju and
# openstack replay fixture outputs keyed by argument patterns, selected per test
# via FIXDIR. Asserts the behavioral checks fire on the exact incident
# signatures they were written for (D-045/D-046/D-051/D-062/vault-sealed).
# Mutates nothing outside $TMP. Exit: 0 all pass | 1 any failed. ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0
BIN="$TMP/bin"; mkdir -p "$BIN"
cat > "$BIN/juju" <<'FB'
#!/usr/bin/env bash
# fake juju: replay fixtures from $FIXDIR keyed by argv patterns
# `juju config -m <model> <app> <key>` is matched POSITIONALLY (argv 4 and 5), not by
# a glob on the joined string: the A10 arity check needs per-app/per-key answers, and
# a missing fixture must reproduce real juju's "key not found" error + rc1 so the
# REFUSAL branch is exercised rather than being handed an empty success.
if [ "${1:-}" = "config" ]; then
f="$FIXDIR/config/${4:-}.${5:-}"
[ -f "$f" ] && { cat "$f"; exit 0; }
echo "ERROR key \"${5:-}\" not found" >&2; exit 1
fi
A="$*"
case "$A" in
*"status -m"*"keystone --format=json"*) cat "$FIXDIR/ks-status.json" ;;
*"status -m"*"--format=json"*) cat "$FIXDIR/status.json" ;;
*"vault/0"*"vault status"*) cat "$FIXDIR/vault.txt" ;;
*"cluster/status OVN_Northbound"*) cat "$FIXDIR/ovn-nb.txt" ;;
*"cluster/status OVN_Southbound"*) cat "$FIXDIR/ovn-sb.txt" ;;
*"connection-status"*) cat "$FIXDIR/chassis.txt" ;;
*"ps -ww -C magnum-conductor"*) cat "$FIXDIR/conductor.txt" ;;
*) echo "fake-juju: unmatched: $A" >&2; exit 1 ;;
esac
FB
cat > "$BIN/openstack" <<'FB'
#!/usr/bin/env bash
A="$*"
# truthful-mock knobs (DOCFIX-094): FIXDIR/noise -> realistic deprecation notice on
# STDERR for the three list calls; FIXDIR/svc-err -> compute service list ERRORS
# (stderr + rc1, no stdout), the shape that was silently fail-open pre-fix.
case "$A" in
*"hypervisor list"*|*"compute service list"*|*"loadbalancer list"*)
[ -f "$FIXDIR/noise" ] && echo "WARNING: this CLI call is deprecated and will be removed" >&2 ;;
esac
case "$A" in
*"hypervisor list"*) cat "$FIXDIR/hypervisors.json" ;;
*"compute service list"*)
if [ -f "$FIXDIR/svc-err" ]; then echo "Unauthorized (HTTP 401)" >&2; exit 1; fi
cat "$FIXDIR/services.json" ;;
*"loadbalancer list"*) cat "$FIXDIR/lbs.json" ;;
*"domain show magnum"*) cat "$FIXDIR/domain.txt" ;;
*"user show magnum_domain_admin"*) cat "$FIXDIR/user.txt" ;;
*"coe service list"*) cat "$FIXDIR/coe.txt" ;;
*) echo "fake-openstack: unmatched: $A" >&2; exit 1 ;;
esac
FB
chmod +x "$BIN/juju" "$BIN/openstack"
mkgood() { # healthy-cloud fixture set
local d="$1"; mkdir -p "$d"
cat > "$d/status.json" <<'J'
{"applications":{
"mysql-innodb-cluster":{"units":{
"mysql-innodb-cluster/0":{"workload-status":{"current":"active","message":"Unit is ready: Mode: R/W, Cluster is ONLINE"}},
"mysql-innodb-cluster/1":{"workload-status":{"current":"active","message":"Unit is ready: Mode: R/O, Cluster is ONLINE"}},
"mysql-innodb-cluster/2":{"workload-status":{"current":"active","message":"Unit is ready: Mode: R/O, Cluster is ONLINE"}}}},
"ovn-central":{"units":{"ovn-central/0":{"workload-status":{"current":"active"}},
"ovn-central/1":{"workload-status":{"current":"active"}},"ovn-central/2":{"workload-status":{"current":"active"}}}},
"nova-compute":{"units":{"nova-compute/0":{"workload-status":{"current":"active"}}}},
"octavia":{"units":{"octavia/0":{"workload-status":{"current":"active"}}}},
"glance-simplestreams-sync":{"units":{"glance-simplestreams-sync/0":{"workload-status":{"current":"unknown"}}}}
}}
J
printf 'Initialized true\nSealed false\n' > "$d/vault.txt"
printf 'Cluster ID: abcd (uuid)\nServers: 3\n' > "$d/ovn-nb.txt"
cp "$d/ovn-nb.txt" "$d/ovn-sb.txt"
printf 'connected\n' > "$d/chassis.txt"
printf '/usr/bin/python3 /usr/bin/magnum-conductor --config-file /etc/magnum/magnum.conf --config-dir /etc/magnum/magnum.conf.d\n' > "$d/conductor.txt"
printf '[{"Hypervisor Hostname":"openstack0","State":"up"}]\n' > "$d/hypervisors.json"
printf '[{"Binary":"nova-compute","Host":"openstack0","Status":"enabled","State":"up"}]\n' > "$d/services.json"
printf '[]\n' > "$d/lbs.json"
printf 'True\n' > "$d/domain.txt"
printf 'magnum_domain_admin\n' > "$d/user.txt"
printf 'magnum-conductor\n' > "$d/coe.txt"
# truthful mock: real juju 3.6 status JSON shape (line-format omits messages,
# which is exactly the DOCFIX-087 false-negative this fixture used to hide)
cat > "$d/ks-status.json" <<'J'
{"applications":{"keystone":{
"application-status":{"current":"active","message":"Application Ready"},
"units":{"keystone/0":{"workload-status":{"current":"active","message":"PO: Unit is ready"}}}}}}
J
}
run() { # run <want_rc> <regex> <label> <fixdir> [env...]
local want="$1" rx="$2" label="$3" fd="$4"; shift 4
local out rc
out="$(cd "$TMP" && env "$@" FIXDIR="$fd" PATH="$BIN:$PATH" OS_AUTH_URL="https://fake:5000/v3" \
bash "$REPO/scripts/cloud-assert.sh" 2>&1)"; rc=$?
if [[ "$rc" == "$want" ]] && grep -qE "$rx" <<<"$out"; then
echo " PASS $label"; PASS=$((PASS+1))
else
echo " FAIL $label (rc=$rc want=$want)"; grep -E '\[(FAIL|WARN)\]|CLOUD-ASSERT' <<<"$out" | head -6 | sed 's/^/ /'; FAIL=$((FAIL+1))
fi
}
F="$TMP/good"; mkgood "$F"; run 0 'CLOUD-ASSERT: PASS' "T1 healthy cloud -> PASS" "$F"
F="$TMP/sealed"; mkgood "$F"; printf 'Initialized true\nSealed true\n' > "$F/vault.txt"
run 1 'vault SEALED' "T2 sealed vault FAILS (with unseal pointer)" "$F"
F="$TMP/rw2"; mkgood "$F"; python3 - "$F/status.json" <<'PY'
import json,sys; p=sys.argv[1]; d=json.load(open(p))
u=d["applications"]["mysql-innodb-cluster"]["units"]
u["mysql-innodb-cluster/1"]["workload-status"]["message"]="Unit is ready: Mode: R/W, Cluster is ONLINE"
json.dump(d,open(p,"w"))
PY
run 1 'R/W count=2' "T3 mysql split-brain (2x R/W) FAILS" "$F"
F="$TMP/chassis"; mkgood "$F"; printf 'not connected\n' > "$F/chassis.txt"
run 1 'chassis NOT connected' "T4 dead chassis FAILS (D-045 class)" "$F"
F="$TMP/pobroken"; mkgood "$F"; cat > "$F/ks-status.json" <<'J'
{"applications":{"keystone":{
"application-status":{"current":"active","message":"Application Ready"},
"units":{"keystone/0":{"workload-status":{"current":"active","message":"PO (broken): policy zip attached but unparsed"}}}}}}
J
run 1 'PO \(broken\)' "T5 PO(broken) FAILS (DOCFIX-071)" "$F"
F="$TMP/nodomain"; mkgood "$F"; printf 'No domain with a name or ID\n' > "$F/domain.txt"
run 1 "domain-setup" "T6 missing trustee domain FAILS (D-046)" "$F"
F="$TMP/graft"; mkgood "$F"; printf '/usr/bin/python3 /usr/bin/magnum-conductor --config-file /etc/magnum/magnum.conf\n' > "$F/conductor.txt"
run 1 'lack --config-dir' "T7 dead conductor graft FAILS (D-037)" "$F"
F="$TMP/lberr"; mkgood "$F"; printf '[{"name":"lb1","provisioning_status":"ERROR","operating_status":"OFFLINE"}]\n' > "$F/lbs.json"
run 1 'LBs unhealthy' "T8 LB in ERROR FAILS (with failover pointer)" "$F"
# T10 (DOCFIX-087 regression): healthy keystone but NO PO: in any message -> FAIL
F="$TMP/nopo"; mkgood "$F"; cat > "$F/ks-status.json" <<'J'
{"applications":{"keystone":{
"application-status":{"current":"active","message":"Application Ready"},
"units":{"keystone/0":{"workload-status":{"current":"active","message":"Unit is ready"}}}}}}
J
run 1 'NO policy override' "T10 missing PO: marker FAILS (DOCFIX-087)" "$F"
# T11 (DOCFIX-094): stderr deprecation noise on healthy cloud must NOT break the
# structured captures (pre-fix: merged noise killed the A5 parse -> false FAIL)
F="$TMP/noise"; mkgood "$F"; : > "$F/noise"
run 0 'CLOUD-ASSERT: PASS' "T11 stderr noise does not break A5/A6 parsing" "$F"
# T12 (DOCFIX-094): compute service list ERRORING must FAIL (pre-fix: silently OK)
F="$TMP/svcerr"; mkgood "$F"; : > "$F/svc-err"
run 1 'compute service list unreadable' "T12 erroring service-list FAILS (was fail-open)" "$F"
# T13-T15 (D-068 item 3): A9 vault-kv consumer auth section
addvault() { # add a vault app to a fixture's status.json
python3 - "$1/status.json" <<'PY'
import json,sys; p=sys.argv[1]; d=json.load(open(p))
d["applications"]["vault"]={"units":{"vault/0":{"workload-status":{"current":"active"}}}}
json.dump(d,open(p,"w"))
PY
}
# T13: no vault app in model -> A9 skips with ok (the pre-Stage-5 VR1 shape)
F="$TMP/novault"; mkgood "$F"
run 0 'A9 skipped \(vault arrives Stage 5\)' "T13 no vault app -> A9 skip, still PASS" "$F"
# T14: vault present, probe driver passes -> PASS line
F="$TMP/vkpass"; mkgood "$F"; addvault "$F"
printf '#!/usr/bin/env bash\necho PASS: all consumers\nexit 0\n' > "$TMP/vkh-pass.sh"
run 0 'every consumer AppRole login 200' "T14 vault-kv health pass -> ok" "$F" CLOUD_ASSERT_VKH="$TMP/vkh-pass.sh"
# T15: vault present, probe driver fails -> section FAILs the sweep
F="$TMP/vkfail"; mkgood "$F"; addvault "$F"
printf '#!/usr/bin/env bash\necho "FAIL: barbican-vault login_http=403"\nexit 1\n' > "$TMP/vkh-fail.sh"
run 1 'vault-kv health FAILED' "T15 vault-kv health fail -> FAIL (D-068 item 3)" "$F" CLOUD_ASSERT_VKH="$TMP/vkh-fail.sh"
# ---- T16-T21 (2026-07-29): A10 HA arity + the non-vacuous A3 uniformity ----
# Before these, HA was asserted by NOTHING after deploy: this script arity-checked one
# application (mysql, A2) and A3 reported "uniform across units" over a SINGLE unit.
addha() { # addha <fixdir> <principal-units> <cluster_count-or-empty>
python3 - "$1/status.json" "$2" <<'PY'
import json,sys
p,n=sys.argv[1],int(sys.argv[2]); d=json.load(open(p))
d["applications"]["keystone"]={"charm-name":"keystone","units":{
"keystone/%d"%i:{"workload-status":{"current":"active"}} for i in range(n)}}
# MEASURED shape from asbuilt/20260706-224851/juju-status.json: the hacluster
# subordinate app has charm-name "hacluster", subordinate-to [principal], units {}.
d["applications"]["keystone-hacluster"]={"charm-name":"hacluster",
"subordinate-to":["keystone"]}
json.dump(d,open(p,"w"))
PY
mkdir -p "$1/config"
[ -n "${3:-}" ] && printf '%s\n' "$3" > "$1/config/keystone-hacluster.cluster_count"
return 0
}
# T16 GREEN: 3 live units, cluster_count 3 -> the check can pass, not only fail
F="$TMP/ha3"; mkgood "$F"; addha "$F" 3 3
run 0 'keystone: 3 live unit\(s\) == keystone-hacluster cluster_count=3' \
"T16 3 units vs cluster_count=3 -> ok" "$F"
# T17 RED: the missing unit A2's mysql-only arity check could never see
F="$TMP/ha2"; mkgood "$F"; addha "$F" 2 3
run 1 'keystone has 2 live unit\(s\) but keystone-hacluster declares cluster_count=3' \
"T17 2 live units vs cluster_count=3 FAILS" "$F"
# T18 RED the other way: a quorum left over from a scale-down
F="$TMP/hadown"; mkgood "$F"; addha "$F" 1 3
run 1 'HA arity broken' "T18 scale-down leaving cluster_count=3 FAILS" "$F"
# T19 REFUSAL: cluster_count unreadable -> never "verified by default"
F="$TMP/hanocc"; mkgood "$F"; addha "$F" 3 ""
run 1 'REFUSING to report keystone' \
"T19 unreadable cluster_count REFUSES (not a pass)" "$F"
# T20 rabbitmq clusters natively, so A10's hacluster loop cannot see it: multi-unit
# without min-cluster-size is the D-009 AMENDMENT race
F="$TMP/rabbit"; mkgood "$F"; python3 - "$F/status.json" <<'PY'
import json,sys; p=sys.argv[1]; d=json.load(open(p))
d["applications"]["rabbitmq-server"]={"charm-name":"rabbitmq-server","units":{
"rabbitmq-server/%d"%i:{"workload-status":{"current":"active"}} for i in range(3)}}
json.dump(d,open(p,"w"))
PY
run 1 'min-cluster-size is unset/unreadable' \
"T20 3-unit rabbitmq with no min-cluster-size FAILS (D-009 amendment)" "$F"
mkdir -p "$TMP/rabbit-ok"; mkgood "$TMP/rabbit-ok"
cp "$F/status.json" "$TMP/rabbit-ok/status.json"; mkdir -p "$TMP/rabbit-ok/config"
printf '3\n' > "$TMP/rabbit-ok/config/rabbitmq-server.min-cluster-size"
run 0 'rabbitmq-server: 3 live unit\(s\) == min-cluster-size=3' \
"T20b 3-unit rabbitmq with min-cluster-size=3 -> ok" "$TMP/rabbit-ok"
# T21b the skip must distinguish "none present" from "discovery missed them": an app
# NAMED *-hacluster while charm-name discovery returns nothing means the status
# shape changed, and skipping would pass over every principal.
F="$TMP/hashape"; mkgood "$F"; python3 - "$F/status.json" <<'PY'
import json,sys; p=sys.argv[1]; d=json.load(open(p))
d["applications"]["keystone-hacluster"]={"charm-name":"SOMETHING-ELSE","subordinate-to":["keystone"]}
json.dump(d,open(p,"w"))
PY
run 1 'REFUSING to skip A10' \
"T21b named -hacluster app but no charm-name match REFUSES the skip" "$F"
# T21 A3 must not certify uniformity over one unit -- it used to print
# "Cluster ID uniform across units" for a single ovn-central unit.
F="$TMP/ovn1"; mkgood "$F"; python3 - "$F/status.json" <<'PY'
import json,sys; p=sys.argv[1]; d=json.load(open(p))
u=d["applications"]["ovn-central"]["units"]
for k in list(u)[1:]: del u[k]
json.dump(d,open(p,"w"))
PY
run 1 'uniformity NOT assertable: only 1 unit' \
"T21 single-unit ovn-central is NOT 'uniform' (vacuous pass removed)" "$F"
# T9: no admin scope -> A5-A7 HELD, exit 2 (never a silent pass)
F="$TMP/held"; mkgood "$F"
out="$(cd "$TMP" && env -u OS_AUTH_URL FIXDIR="$F" PATH="$BIN:$PATH" bash "$REPO/scripts/cloud-assert.sh" 2>&1)"; rc=$?
if [[ "$rc" == "2" ]] && grep -q 'HELD: no admin scope' <<<"$out"; then echo " PASS T9 missing admin scope -> HELD (rc 2)"; PASS=$((PASS+1))
else echo " FAIL T9 (rc=$rc)"; FAIL=$((FAIL+1)); fi
echo; echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1
