#!/usr/bin/env bash
# scripts/cloud-assert.sh [--capture] [MODEL]
#
# Behavioral cloud verifier (DOCFIX-075). Runs EVERY "the service's own verdict"
# gate learned from the D-045/D-046/D-051/D-042 incident family -- the checks
# juju status is BLIND to -- as one idempotent, read-only sweep. Run it:
# post-deploy (phase-08 acceptance), post-restart (ops-restart-procedure Stage 7),
# pre-change baseline, and post-incident. Absorbs the jumphost-local
# post-maintenance-health-check.sh (which was never version-controlled).
#
# Sections (each independent; worst exit wins):
#   A0 juju reachable; no units in error/blocked (carried gss 'unknown' tolerated)
#   A1 vault: Initialized=true, Sealed=false (sealed-after-reboot is BY DESIGN --
#      the FIX is manual unseal per ops-restart-procedure, not a redeploy)
#   A2 mysql: 3 units 'Cluster is ONLINE', exactly one R/W (D-062 world)
#   A3 OVN central: Cluster ID uniform across the 3 units, both NB + SB DBs
#   A4 OVN chassis: ovn-controller 'connected' on every nova-compute + octavia
#   A5 compute plane: hypervisors up, compute services enabled+up   [needs OS_*]
#   A6 octavia: every LB provisioning=ACTIVE + operating=ONLINE (skip if none) [OS_*]
#   A7 identity/magnum behavior: keystone shows 'PO:' (D-051/DOCFIX-071);
#      trustee domain 'magnum' + magnum_domain_admin exist (D-046);
#      'openstack coe service list' returns the conductor row (no 403)  [OS_*]
#   A8 conductor graft: magnum-conductor LIVE process args carry --config-dir
#      (D-037: verify the launched cmdline, never the config text)
#   A9 vault-kv consumer AppRole auth (D-068 item 3, ruled 2026-07-21):
#      delegates to vault-kv-health.sh (one real 60s-TTL login per consumer);
#      SKIPS (ok) when the model has no vault application -- pre-Stage-5 VR1
#   A10 HA arity: every hacluster principal's LIVE unit count equals the quorum its
#      own subordinate declares (cluster_count), and rabbitmq-server's equals
#      min-cluster-size. Added 2026-07-29 -- before it, this script arity-checked
#      exactly ONE application (mysql, A2), so a control plane that came up with two
#      of three units of everything else read PASS.
#
# --capture: additionally writes a deploy BOM (bill of materials) to
#   asbuilt/<UTC timestamp>/ : exported bundle, juju status yaml, driver pip
#   version, image list. Commit the directory -- it is the drift baseline for
#   Roosevelt multi-DC comparison. Capture is the ONLY write this script does,
#   and only under asbuilt/.
#
# Requirements: jumphost with juju; sections A5-A7 need an admin scope
# (source ~/admin-openrc first) -- absent scope is a HOLD (exit 2), not a skip,
# so a half-run is never mistaken for a pass.
# Exit: 0 all pass | 1 any FAIL | 2 warnings/holds only.  ASCII + LF.

set -uo pipefail
shopt -s inherit_errexit 2>/dev/null || true

MODEL="openstack"; CAPTURE=0
for a in "$@"; do
  case "$a" in
    --capture) CAPTURE=1 ;;
    -*) echo "FAIL: unknown flag $a"; exit 1 ;;
    *) MODEL="$a" ;;
  esac
done
J() { juju "$@" </dev/null 2>&1; }   # capture-then-test everywhere (SIGPIPE rule)
# Structured (-f json) captures take stdout ONLY -- merged stderr lands at char 0
# and kills the parse (DOCFIX-094; the A5-services variant was silently fail-OPEN).
OERR="$(mktemp)"; trap 'rm -f "$OERR"' EXIT
O() { openstack "$@" </dev/null 2>"$OERR"; }
oerr() { head -1 "$OERR" 2>/dev/null || true; }
WORST=0
fail() { echo "  [FAIL] $*"; WORST=1; }
warn() { echo "  [WARN] $*"; [ "$WORST" -ne 1 ] && WORST=2; }
ok()   { echo "  [ok]   $*"; }

command -v juju >/dev/null 2>&1 || { echo "FAIL: juju not found"; exit 1; }
command -v jq   >/dev/null 2>&1 || { echo "FAIL: jq required"; exit 1; }

echo "================ A0: juju model + unit states ================"
ST=$(J status -m "$MODEL" --format=json || true)
if ! jq -e .applications >/dev/null 2>&1 <<<"$ST"; then
  fail "cannot read juju status for model '$MODEL'"; echo "$ST" | head -3
  echo; echo "CLOUD-ASSERT: FAIL (model unreachable)"; exit 1
fi
BAD=$(jq -r '.applications | to_entries[] as $app | ($app.value.units // {}) | to_entries[]
      | select(.value["workload-status"].current | IN("error","blocked"))
      | "\(.key) \(.value["workload-status"].current): \(.value["workload-status"].message // "")"' <<<"$ST" || true)
UNK=$(jq -r '.applications | to_entries[] as $app | ($app.value.units // {}) | to_entries[]
      | select(.value["workload-status"].current == "unknown") | .key' <<<"$ST" || true)
if [ -n "$BAD" ]; then while IFS= read -r l; do fail "unit $l"; done <<<"$BAD"
else ok "no units in error/blocked"; fi
if [ -n "$UNK" ]; then
  if grep -qv 'glance-simplestreams-sync' <<<"$UNK"; then warn "units 'unknown' beyond parked gss: $(tr '\n' ' ' <<<"$UNK")"
  else ok "only parked gss reports 'unknown' (documented carry)"; fi
fi

echo "================ A1: vault unsealed ================"
V=$(J ssh -m "$MODEL" vault/0 -- 'VAULT_ADDR=http://127.0.0.1:8200 vault status 2>&1' || true)
grep -q 'Initialized *true'  <<<"$V" || fail "vault not initialized"
if grep -q 'Sealed *false' <<<"$V"; then ok "vault Initialized=true Sealed=false"
else fail "vault SEALED -- manual 3-of-5 unseal (ops-restart-procedure Stage 3); sealed-after-restart is by design"; fi

echo "================ A2: mysql innodb cluster ================"
M=$(jq -r '.applications["mysql-innodb-cluster"].units // {} | to_entries[]
     | "\(.key) \(.value["workload-status"].message // "")"' <<<"$ST" || true)
N=$(grep -c 'Cluster is ONLINE' <<<"$M" || true)
RW=$(grep -c 'Mode: R/W' <<<"$M" || true)
[ "${N:-0}" -eq 3 ] || fail "mysql ONLINE units=$N (want 3): $(tr '\n' ' | ' <<<"$M")"
[ "${RW:-0}" -eq 1 ] || fail "mysql R/W count=$RW (want exactly 1)"
[ "${N:-0}" -eq 3 ] && [ "${RW:-0}" -eq 1 ] && ok "mysql 3x ONLINE, exactly one R/W"

echo "================ A3: OVN central cluster unity ================"
UNITS=$(jq -r '.applications["ovn-central"].units // {} | keys[]' <<<"$ST" || true)
if [ -z "$UNITS" ]; then warn "ovn-central units not found -- skipped"
else
  for spec in 'OVN_Northbound:ovnnb_db.ctl' 'OVN_Southbound:ovnsb_db.ctl'; do
    db=${spec%%:*}; sock=${spec##*:}; IDS=""
    while IFS= read -r u; do
      O=$(J ssh -m "$MODEL" "$u" -- "sudo ovs-appctl -t /var/run/ovn/${sock} cluster/status ${db} 2>&1" || true)
      CID=$(awk '/^Cluster ID/{print $3; exit}' <<<"$O")
      [ -n "$CID" ] && IDS="$IDS $CID" || fail "$u: no Cluster ID for $db"
    done <<<"$UNITS"
    # A uniformity assertion over ONE contributing unit is vacuous -- it passed a
    # single-unit ovn-central and reported "uniform across units" (found 2026-07-29).
    # bundle.yaml pins ovn-central num_units=3, so fewer than 3 IDs means units are
    # MISSING or unreadable, and "could not look" is never "nothing there".
    NC=$(tr ' ' '\n' <<<"$IDS" | sed '/^$/d' | wc -l)
    U=$(tr ' ' '\n' <<<"$IDS" | sed '/^$/d' | sort -u | wc -l)
    if [ "$NC" -lt 3 ]; then
      fail "$db uniformity NOT assertable: only $NC unit(s) returned a Cluster ID (bundle.yaml pins ovn-central num_units=3) -- one unit is trivially 'uniform'"
    elif [ "$U" -eq 1 ]; then ok "$db Cluster ID uniform across $NC units"
    else fail "$db Cluster IDs differ:$IDS"; fi
  done
fi

echo "================ A4: OVN chassis connectivity ================"
CH=$(jq -r '.applications | to_entries[] | select(.key=="nova-compute" or .key=="octavia")
     | .value.units // {} | keys[]' <<<"$ST" || true)
if [ -z "$CH" ]; then warn "no chassis principals found -- skipped"
else while IFS= read -r u; do
  C=$(J ssh -m "$MODEL" "$u" -- 'sudo ovn-appctl -t ovn-controller connection-status 2>&1' | tr -d '\r' || true)
  grep -q '^connected' <<<"$C" && ok "$u chassis connected" \
    || fail "$u chassis NOT connected ($C) -- post-vault TLS sweep (appendix-A / restart Stage 4)"
done <<<"$CH"; fi

if [ -z "${OS_AUTH_URL:-}" ]; then
  warn "A5-A7 HELD: no admin scope in env (source ~/admin-openrc, then re-run)"
else
  echo "================ A5: compute plane ================"
  H=$(O hypervisor list -f json || true)
  if jq -e . >/dev/null 2>&1 <<<"$H"; then
    DOWN=$(jq -r '.[] | select((.State // ."state") != "up") | (."Hypervisor Hostname" // .hypervisor_hostname)' <<<"$H" || true)
    [ -z "$DOWN" ] && ok "all hypervisors up" || fail "hypervisors down: $(tr '\n' ' ' <<<"$DOWN") (restart nova-compute; see appendix-A)"
  else fail "hypervisor list unreadable: $(oerr)"; fi
  S=$(O compute service list -f json || true)
  if jq -e . >/dev/null 2>&1 <<<"$S"; then
    BADS=$(jq -r '.[] | select(.Status=="enabled" and .State!="up") | "\(.Binary)@\(.Host)"' <<<"$S" || true)
    [ -z "$BADS" ] && ok "all enabled compute services up" || fail "compute services down: $(tr '\n' ' ' <<<"$BADS")"
  else fail "compute service list unreadable: $(oerr) (was silently fail-open pre-DOCFIX-094)"; fi

  echo "================ A6: octavia load balancers ================"
  L=$(O loadbalancer list -f json || true)
  if jq -e . >/dev/null 2>&1 <<<"$L"; then
    CNT=$(jq 'length' <<<"$L")
    if [ "$CNT" -eq 0 ]; then ok "no LBs present (nothing to assert)"
    else
      BADL=$(jq -r '.[] | select(.provisioning_status!="ACTIVE" or .operating_status!="ONLINE")
             | "\(.name) prov=\(.provisioning_status) op=\(.operating_status)"' <<<"$L" || true)
      [ -z "$BADL" ] && ok "$CNT LB(s) ACTIVE/ONLINE" \
        || fail "LBs unhealthy: $(tr '\n' ' | ' <<<"$BADL") (failover pattern: ops-restart-procedure Stage 6)"
    fi
  else warn "octavia not answering (absent?): $(oerr)"; fi

  echo "================ A7: identity + magnum behavior ================"
  # DOCFIX-087: read the PO: marker from status JSON messages. juju 3.6
  # --format=line omits workload MESSAGES entirely, so the old line-format
  # grep false-FAILed on a healthy override (found live 2026-07-05).
  KS=$(J status -m "$MODEL" keystone --format=json || true)
  KMSG=$(jq -r '[(.applications.keystone // {})
        | (((."application-status".message) // "")),
          ((((.units // {}) | to_entries[]) | .value."workload-status".message) // "")]
        | join(" | ")' 2>/dev/null <<<"$KS" || true)
  if grep -q 'PO (broken)' <<<"$KMSG"; then fail "keystone PO (broken) -- policy zip attached but unparsed"
  elif grep -q 'PO:' <<<"$KMSG"; then ok "keystone policy override loaded (PO:) -- run G3 (appendix-C C.4) for the behavioral gate"
  else fail "keystone shows NO policy override (DOCFIX-071: redeploy shipped without domain-manager RBAC?)"; fi
  D=$(openstack domain show magnum -f value -c enabled </dev/null 2>&1 || true)
  grep -q '^True$' <<<"$D" && ok "trustee domain 'magnum' exists (D-046)" \
    || fail "trustee domain 'magnum' missing -- run 'juju run magnum/leader domain-setup' (D-046)"
  U=$(openstack user show magnum_domain_admin --domain magnum -f value -c name </dev/null 2>&1 || true)
  grep -q '^magnum_domain_admin$' <<<"$U" && ok "magnum_domain_admin exists" || fail "magnum_domain_admin missing (D-046)"
  C=$(openstack coe service list -f value -c binary </dev/null 2>&1 || true)
  grep -q 'magnum-conductor' <<<"$C" && ok "coe service list returns conductor (no 403)" \
    || fail "coe service list has no conductor row: $(head -1 <<<"$C")"
fi

echo "================ A8: conductor graft (launched args, D-037) ================"
P=$(J ssh -m "$MODEL" magnum/0 -- 'ps -ww -C magnum-conductor -o args= 2>&1' || true)
if grep -q -- '--config-dir /etc/magnum/magnum.conf.d' <<<"$P"; then ok "conductor runs with --config-dir (graft live)"
elif grep -qi 'no such\|not found\|cannot' <<<"$P"; then warn "magnum/0 unreachable for A8: $(head -1 <<<"$P")"
else fail "conductor LIVE args lack --config-dir (graft not in effect; config-file presence proves nothing): $(head -1 <<<"$P")"; fi

echo "================ A9: vault-kv consumer AppRole auth (D-068 item 3) ================"
# Ruled 2026-07-21 ("Adopt + probe on VR1 now"): the proactive login check is
# part of every sweep -- expiry surfaces here, never via a tenant's failed
# cluster-create. Delegates to vault-kv-health.sh (dynamic consumer discovery;
# exit 0 pass / 1 fail / 2 coverage-gap). CLOUD_ASSERT_VKH overrides the
# script path -- test seam for the offline harness, unset in real runs.
SD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VKH="${CLOUD_ASSERT_VKH:-$SD/vault-kv-health.sh}"
if ! jq -e '.applications.vault' >/dev/null 2>&1 <<<"$ST"; then
  ok "no vault application in model '$MODEL' -- A9 skipped (vault arrives Stage 5)"
elif [ ! -s "$VKH" ]; then
  fail "vault present but probe driver missing: $VKH"
else
  VKO=$(MODEL="$MODEL" bash "$VKH" 2>&1) ; VKRC=$?
  case "$VKRC" in
    0) ok "vault-kv health: every consumer AppRole login 200" ;;
    2) warn "vault-kv health coverage gap/warnings: $(grep -m2 '^WARN' <<<"$VKO" | tr '\n' ' ')" ;;
    *) fail "vault-kv health FAILED: $(grep -m3 '^FAIL' <<<"$VKO" | tr '\n' ' ')" ;;
  esac
fi

echo "================ A10: HA arity -- declared quorum vs live units ================"
# THE LIFECYCLE GAP THIS CLOSES (measured 2026-07-29): provider-bundle-check asserts
# cluster_count in the BUNDLE, but nothing compared it to a RUNNING model, and this
# script asserted a unit count exactly once (mysql, A2). So HA was asserted by NOTHING
# after deploy: a model up with 2 of 3 units of every API charm read PASS.
# DISCOVERY is from status JSON alone -- MEASURED shape, asbuilt/20260706-224851/
# juju-status.json: an hacluster subordinate application carries charm-name "hacluster"
# and subordinate-to ["<principal>"]. No app list is hardcoded, so the 14 apps
# overlays/dc-ha-scaleup.yaml scales are covered without naming any of them.
# The DECLARED quorum comes from the charm's OWN config via `juju config -m <model>
# <app> <key>` -- the form already in scripts/phase-05-octavia-verify.sh:54-57 and
# runbooks/dc-dc-phase4-juju-bundle-per-dc.md:855 (`juju config -m "$DC_MODEL"
# <app>-hacluster cluster_count`). An unreadable or non-numeric value REFUSES.
HAP=$(jq -r '.applications // {} | to_entries[]
      | select(.value["charm-name"] == "hacluster")
      | .key as $sub | (.value["subordinate-to"] // [])[] | "\($sub) \(.)"' <<<"$ST" || true)
if [ -z "$HAP" ]; then
  # A skip must be able to tell "none present" from "discovery missed them" -- otherwise
  # this section becomes the very silent pass it was added to remove. Cross-check by
  # NAME: this bundle's hacluster subordinates are all <principal>-hacluster, and no
  # other application carries that suffix (mysql-innodb-cluster / ovn-central / ceph-mon
  # cluster natively and have none).
  NAMED=$(jq -r '.applications // {} | keys[] | select(endswith("-hacluster"))' <<<"$ST" || true)
  if [ -n "$NAMED" ]; then
    fail "charm-name discovery found NO hacluster subordinates, yet the model carries $(tr '\n' ' ' <<<"$NAMED")-- REFUSING to skip A10; the status-JSON shape changed and this section would pass over every principal"
  else
    ok "no hacluster subordinates in model '$MODEL' -- A10 hacluster arity skipped (VR1 HA arrives Stage 5)"
  fi
else
  while read -r sub prin; do
    [ -n "$sub" ] && [ -n "$prin" ] || continue
    LIVE=$(jq -r --arg a "$prin" '.applications[$a].units // {} | length' <<<"$ST" || true)
    CC=$(J config -m "$MODEL" "$sub" cluster_count | head -1 | tr -d ' \r' || true)
    case "$CC" in
      ''|*[!0-9]*)
        fail "cannot read $sub cluster_count (got '$CC') -- REFUSING to report $prin's HA arity as verified" ;;
      *)
        if [ "${LIVE:-x}" = "$CC" ]; then ok "$prin: $LIVE live unit(s) == $sub cluster_count=$CC"
        else fail "$prin has ${LIVE:-?} live unit(s) but $sub declares cluster_count=$CC -- HA arity broken (a missing unit, or a quorum left over from a scale change)"; fi ;;
    esac
  done <<<"$HAP"
fi
# rabbitmq-server clusters NATIVELY (quorum, no hacluster), so the loop above cannot
# see it -- yet it is one of the 14 apps dc-ha-scaleup scales. Its declared quorum is
# min-cluster-size, MANDATORY at multi-unit per the D-009 AMENDMENT (2026-07-02):
# without it the charm accepts client relations BEFORE the cluster forms.
if jq -e '.applications["rabbitmq-server"]' >/dev/null 2>&1 <<<"$ST"; then
  RN=$(jq -r '.applications["rabbitmq-server"].units // {} | length' <<<"$ST" || true)
  MCS=$(J config -m "$MODEL" rabbitmq-server min-cluster-size | head -1 | tr -d ' \r' || true)
  case "$MCS" in
    ''|*[!0-9]*)
      if [ "${RN:-0}" -gt 1 ]; then
        fail "rabbitmq-server has $RN units but min-cluster-size is unset/unreadable ('$MCS') -- the charm accepts client relations BEFORE the cluster forms (D-009 AMENDMENT 2026-07-02)"
      else
        ok "rabbitmq-server min-cluster-size unset at ${RN:-0} unit(s) -- the single-unit VR0 shape (D-009); rabbitmq arity deliberately NOT asserted here"
      fi ;;
    *)
      if [ "${RN:-x}" = "$MCS" ]; then ok "rabbitmq-server: $RN live unit(s) == min-cluster-size=$MCS"
      else fail "rabbitmq-server has ${RN:-?} live unit(s) but min-cluster-size=$MCS -- the native quorum will not form (D-009 AMENDMENT)"; fi ;;
  esac
fi

if [ "$CAPTURE" -eq 1 ]; then
  echo "================ CAPTURE: as-built BOM ================"
  DIR="asbuilt/$(date -u +%Y%m%d-%H%M%S)"; mkdir -p "$DIR"
  J export-bundle -m "$MODEL" > "$DIR/bundle-exported.yaml" || warn "export-bundle failed"
  printf '%s\n' "$ST" > "$DIR/juju-status.json"
  J ssh -m "$MODEL" magnum/0 -- 'pip show magnum-capi-helm 2>/dev/null | egrep "Version|Location"' > "$DIR/driver-version.txt" || true
  if [ -n "${OS_AUTH_URL:-}" ]; then
    openstack image list --long -f json </dev/null > "$DIR/images.json" 2>/dev/null || true
  fi
  ok "BOM written to $DIR -- commit it (Roosevelt drift baseline)"
fi

case "$WORST" in
  0) echo; echo "CLOUD-ASSERT: PASS" ;;
  2) echo; echo "CLOUD-ASSERT: WARN/HELD -- review before trusting" ;;
  *) echo; echo "CLOUD-ASSERT: FAIL" ;;
esac
exit "$WORST"