#!/usr/bin/env bash
# scripts/phase-05-octavia-verify.sh [MODEL]
#
# Read-only verify for phase-05 (Octavia enablement, D-021). Auto-detects where the
# phase is and reports accordingly; mutates NOTHING, safe to re-run:
#
#   CONFIG GATE (always): the LP#1937003 / Ceph-RBD preconditions --
#     octavia-diskimage-retrofit use-internal-endpoints=true, image-format=raw,
#     amp-image-tag set AND == octavia amp-image-tag. A mismatch HOLDs regardless of state.
#
#   then, by octavia/0 workload status:
#     blocked  -> PRE: charm-octavia resources must be EMPTY -> PROCEED (run 5.1 configure-resources)
#     active   -> POST: lb-mgmt net/subnet/sec-grp present + o-hm0 (UP/UNKNOWN) w/ fc00::/ ULA;
#                 then amphora image tagged $OTAG ACTIVE?
#                   no  -> CONTROL-PLANE-DONE (5.1 done; run 5.2 amphora pipeline)
#                   yes -> PASS (phase-05 EXIT GATE met)
#     other    -> HOLD (in flight / unexpected; re-check, do not re-fire 5.1)
#
# Usage:  source ~/admin-openrc && scripts/phase-05-octavia-verify.sh [MODEL]
# Exit:   0 PROCEED|CONTROL-PLANE-DONE|PASS | 1 HOLD/FAIL | 2 precondition
#
# Resolves dynamically; tags read from charm config (not hardcoded). Read-only. ASCII + LF.

set -euo pipefail
shopt -s inherit_errexit 2>/dev/null || true
IFS=$'\n\t'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/lib-net.sh
. "$SCRIPT_DIR/lib-net.sh"

MODEL="${1:-openstack}"
RETRO="octavia-diskimage-retrofit"

FATAL=0
fail() { echo "FAIL: $*" >&2; FATAL=$((FATAL + 1)); }
pass() { echo "PASS: $*"; }

# --- preconditions ------------------------------------------------------------------
need_jq || exit 2
command -v openstack >/dev/null 2>&1 || { echo "FAIL: openstack client not found" >&2; exit 2; }
command -v juju      >/dev/null 2>&1 || { echo "FAIL: juju client not found" >&2; exit 2; }
[ -n "${OS_AUTH_URL:-}" ] || { echo "FAIL: OS_AUTH_URL unset -- 'source ~/admin-openrc' first" >&2; exit 2; }

J="$(juju status octavia -m "$MODEL" --format json 2>/dev/null || true)"
printf '%s' "$J" | jq -e '.applications.octavia.units."octavia/0"' >/dev/null 2>&1 \
  || { echo "FAIL: octavia/0 not found in model '$MODEL'" >&2; exit 2; }

echo "=== phase-05 octavia-enablement verify (read-only) ==="
echo

# --- CONFIG GATE (LP#1937003 + Ceph-RBD; always) -----------------------------------
echo "--- config gate (use-internal-endpoints / image-format / amp-image-tag match) ---"
UIE="$(juju config "$RETRO" use-internal-endpoints 2>/dev/null || true)"
IMGFMT="$(juju config "$RETRO" image-format 2>/dev/null || true)"
RTAG="$(juju config "$RETRO" amp-image-tag 2>/dev/null || true)"
OTAG="$(juju config octavia amp-image-tag 2>/dev/null || true)"
echo "  retrofit use-internal-endpoints=$UIE  image-format=$IMGFMT  amp-image-tag=$RTAG ; octavia amp-image-tag=$OTAG"
[ "$UIE" = "true" ] || fail "retrofit use-internal-endpoints=$UIE (need true; retrofit is metal-only -> internal glance)"
[ "$IMGFMT" = "raw" ] || fail "retrofit image-format=$IMGFMT (need raw; Ceph RBD fast-clone)"
{ [ -n "$RTAG" ] && [ "$RTAG" = "$OTAG" ]; } || fail "amp-image-tag mismatch retrofit='$RTAG' octavia='$OTAG' (LP#1937003)"
[ "$FATAL" -eq 0 ] && pass "config gate clear"
echo

# --- octavia workload state ---------------------------------------------------------
OSTATE="$(printf '%s' "$J" | jq -r '.applications.octavia.units."octavia/0"."workload-status".current // "unknown"')"
OMSG="$(printf '%s' "$J" | jq -r '.applications.octavia.units."octavia/0"."workload-status".message // ""')"
echo "--- octavia/0 workload: $OSTATE ($OMSG) ---"

# helper: count non-empty Name rows of a charm-octavia-tagged list
count_tagged() { "$@" --tags charm-octavia -f value -c Name 2>/dev/null | grep -c . || true; }

if [ "$OSTATE" = "blocked" ]; then
  # PRE: resources must be empty (idempotency baseline)
  n_net="$(count_tagged openstack network list)"
  n_sub="$(count_tagged openstack subnet list)"
  n_sg="$(count_tagged openstack security group list)"
  echo "  charm-octavia resources: networks=$n_net subnets=$n_sub sec-groups=$n_sg (expect 0/0/0 pre-run)"
  if [ "$n_net" -ne 0 ] || [ "$n_sub" -ne 0 ] || [ "$n_sg" -ne 0 ]; then
    fail "octavia blocked but charm-octavia resources already exist -- inconsistent; investigate before re-firing"
  fi
  echo
  if [ "$FATAL" -ne 0 ]; then echo "Summary: HOLD/FAIL -- $FATAL gate(s) failed."; exit 1; fi
  echo "Summary: PROCEED -- config gate clear, octavia awaiting configure-resources (resources empty)."
  echo "  Next: juju run octavia/leader configure-resources -m $MODEL --wait=20m  (Step 5.1)"
  exit 0

elif [ "$OSTATE" = "active" ]; then
  # POST: control-plane resources + o-hm0
  n_net="$(count_tagged openstack network list)"
  n_sub="$(count_tagged openstack subnet list)"
  n_sg="$(count_tagged openstack security group list)"
  echo "  charm-octavia resources: networks=$n_net subnets=$n_sub sec-groups=$n_sg (expect >=1 each)"
  { [ "$n_net" -ge 1 ] && [ "$n_sub" -ge 1 ] && [ "$n_sg" -ge 1 ]; } \
    || fail "octavia active but charm-octavia lb-mgmt resources missing (net/sub/sg = $n_net/$n_sub/$n_sg)"
  OHM0="$(juju exec --unit octavia/0 -m "$MODEL" -- 'ip -br addr show o-hm0' </dev/null 2>/dev/null || true)"
  # o-hm0 is an OVS internal port: kernel operstate reads UNKNOWN (no carrier reporting),
  # NOT UP -- so accept UP or UNKNOWN, reject only DOWN/absent. The fc00::/ ULA is the
  # actual success signal (the do-doc GATE: an fc00:: IPv6-ULA on a br-int port).
  OHM0_STATE="$(printf '%s' "$OHM0" | awk 'NR==1{print $2}')"
  if [ -n "$OHM0_STATE" ] && [ "$OHM0_STATE" != "DOWN" ] && printf '%s' "$OHM0" | grep -q 'fc00:'; then
    pass "o-hm0 present (state=$OHM0_STATE; UNKNOWN/UP both normal for an OVS internal port) with fc00::/ IPv6-ULA"
  else
    fail "o-hm0 unhealthy: state='${OHM0_STATE:-<absent>}' need state!=DOWN and an fc00::/ ULA (got: ${OHM0:-<none>})"
  fi

  # --- R8a (RULED 2026-07-27, OPS under GA-R3): o-hm0 MTU must MATCH lb-mgmt-net's ---
  # LP #2018998 (charm-octavia, High): a jumbo lb-mgmt-net beside a 1500 o-hm0 SILENTLY
  # drops health heartbeats >1500B, which surfaces as SPURIOUS load-balancer failovers --
  # a symptom that reads as a Ceph, network or amphora fault long before anyone suspects
  # MTU. Fix Released for our lineage, but a recurrence was reported 2025-12-31 against
  # octavia 14.0.0 / 2024.1 stable, which is EXACTLY the channel bundle.yaml pins. R3's
  # jumbo-underlay ruling is this bug's precondition, so the risk is live here.
  #
  # This VERIFIES rather than sets: the ruling refused the blind `mtu_request` workaround
  # because it fights the charm's own fix where that fix works, and would MASK a
  # regression. It also refuses to trust the charm's claim -- the assertion is on the two
  # MEASURED values, not on the charm having been asked to set them.
  #
  # The network is resolved BY TAG, like the counts above, never by the name 'lb-mgmt-net'.
  # Every capture below ends `|| true`. This file runs under `set -euo pipefail` WITH
  # `inherit_errexit`, so a bare `$(... )` that exits non-zero -- an unresolvable network,
  # a `head` on empty input -- ABORTS THE WHOLE SCRIPT mid-check rather than falling
  # through to the refusal branch. Measured: the first draft did exactly that, and the
  # pre-existing tests/phase-05 harness caught it.
  OHM0_MTU="$(juju exec --unit octavia/0 -m "$MODEL" -- 'ip -o link show o-hm0' </dev/null 2>/dev/null \
              | sed -n 's/.*[[:space:]]mtu[[:space:]]\([0-9]\{1,\}\).*/\1/p' | head -1 || true)"
  LBNET="$(openstack network list --tags charm-octavia -f value -c Name 2>/dev/null | head -1 || true)"
  LBNET_MTU=""
  if [ -n "${LBNET:-}" ]; then
    LBNET_MTU="$(openstack network show "$LBNET" -f value -c mtu 2>/dev/null | head -1 || true)"
  fi
  if [ -z "${OHM0_MTU:-}" ] || [ -z "${LBNET_MTU:-}" ]; then
    # "Could not look" is never "nothing there" -- refuse rather than default to success.
    fail "o-hm0/lb-mgmt-net MTU UNVERIFIABLE (o-hm0='${OHM0_MTU:-<unread>}' net='${LBNET:-<unresolved>}' mtu='${LBNET_MTU:-<unread>}') -- LP#2018998 cannot be ruled out"
  elif [ "$OHM0_MTU" = "$LBNET_MTU" ]; then
    pass "o-hm0 MTU $OHM0_MTU MATCHES lb-mgmt-net ($LBNET) MTU $LBNET_MTU (LP#2018998 clear)"
  else
    fail "LP#2018998 MTU MISMATCH: o-hm0=$OHM0_MTU but lb-mgmt-net ($LBNET)=$LBNET_MTU -- health heartbeats larger than $OHM0_MTU are dropped SILENTLY and surface as spurious LB failovers"
  fi
  # 5.2: amphora image
  AMPH_ACTIVE="$(openstack image list --tag "$OTAG" -f value -c Status 2>/dev/null | grep -xc active || true)"
  echo "  images tagged $OTAG with Status=active: $AMPH_ACTIVE"
  echo
  if [ "$FATAL" -ne 0 ]; then echo "Summary: HOLD/FAIL -- $FATAL gate(s) failed."; exit 1; fi
  if [ "$AMPH_ACTIVE" -ge 1 ]; then
    echo "Summary: PASS -- phase-05 EXIT GATE met (octavia active; lb-mgmt resources + o-hm0 up; ACTIVE amphora tagged $OTAG)."
    exit 0
  fi
  echo "Summary: CONTROL-PLANE-DONE -- Step 5.1 complete; no ACTIVE amphora image yet."
  echo "  Next: run Step 5.2 (amphora image pipeline), then re-run this for PASS."
  exit 0

else
  echo
  echo "Summary: HOLD -- octavia/0 workload '$OSTATE' (not blocked/active). In flight or unexpected;"
  echo "  re-check; do NOT re-fire configure-resources on a transient (appendix-A)."
  [ "$FATAL" -ne 0 ] && exit 1
  exit 1
fi