#!/usr/bin/env bash
# scripts/checks/d011-04-octavia-lb.sh -- D-011 item 4: Octavia LB pattern
# (round-robin -> member failover -> recovery -> AMPHORA failover), per Bobcat v3 work.
#
# Stands up its OWN 2-replica backend + LoadBalancer Service in a tenant cluster (beta),
# drives the pattern, and self-cleans (trap). Two tiers:
#   ALWAYS (additive, self-cleaning): round-robin distribution + member failover + recovery.
#   --disruptive ONLY: amphora failover (destroys+rebuilds the amphora). Guarded by an N+1
#     scheduler-headroom pre-check -- Octavia STANDALONE failover transiently needs room for
#     one MORE amphora; a cloud at its ceiling cannot heal its own LB, so at-ceiling => HOLD
#     (never FAIL, and never TRIGGER a failover that would strand the LB in ERROR).
#
# Exit: 0 PASS | 1 FAIL (a pattern assertion broke) | 2 HOLD (octavia not ready, LB setup
#       undetermined, or amphora headroom not confidently available) | 4 SKIPPED (only if the
#       WHOLE check is not applicable). Amphora tier not requested => PASS on the additive tiers
#       with an explicit "amphora SKIPPED (needs --include-disruptive)" note.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/lib-validate.sh
. "$HERE/../lib-validate.sh"
ID=d011-04-octavia-lb; vr_begin "$ID"
CLIENT="${VR_TENANT:-beta}"
AGNHOST="${VR_AGNHOST:-registry.k8s.io/e2e-test-images/agnhost:2.47}"
NPROBE="${VR_RR_PROBES:-12}"
FAILS=0; HELD=0; AMPHORA_NOTE="amphora tier not run"

vr_need kubectl openstack curl jq awk || { emit "$ID" "$VR_HOLD" "missing tool"; exit "$VR_HOLD"; }

# --- precondition: octavia reachable as admin ---
if ! vr_admin_env; then emit "$ID" "$VR_HOLD" "no admin scope (octavia ops need it)"; exit "$VR_HOLD"; fi
if ! vr_json _LBL openstack loadbalancer list -f json; then
  vr_err_tail; emit "$ID" "$VR_HOLD" "octavia not reachable (loadbalancer list failed)"; exit "$VR_HOLD"
fi

# --- tenant kubeconfig ---
CF=""
for c in "$HOME/tenant-${CLIENT}/${CLIENT}-cluster-cred.txt"; do [ -s "$c" ] && CF="$c"; done
KCFG="$HOME/tenant-${CLIENT}/kube/config"
[ -s "$KCFG" ] || { emit "$ID" "$VR_HOLD" "no tenant kubeconfig at $KCFG (run tenant-acceptance/d011-03 first)"; exit "$VR_HOLD"; }
export KUBECONFIG="$KCFG"

SVC="d011lb$RANDOM"
cleanup(){
  kubectl delete svc "$SVC" --ignore-not-found --now >/dev/null 2>&1 || true
  kubectl delete deploy "$SVC" --ignore-not-found --now >/dev/null 2>&1 || true
}
trap cleanup EXIT

# --- stand up 2-replica backend + LoadBalancer ---
echo "  creating 2-replica backend + LB service ($SVC)"
run kubectl create deployment "$SVC" --image="$AGNHOST" --replicas=2 -- /agnhost netexec --http-port=8080 || { emit "$ID" "$VR_HOLD" "deployment create failed"; exit "$VR_HOLD"; }
run kubectl expose deployment "$SVC" --port=80 --target-port=8080 --type=LoadBalancer || { emit "$ID" "$VR_HOLD" "service expose failed"; exit "$VR_HOLD"; }
kubectl wait --for=condition=available --timeout=120s deploy/"$SVC" >/dev/null 2>&1 || true

# wait for FIP
FIP=""
for _ in $(seq 1 40); do
  FIP="$(kubectl get svc "$SVC" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)"
  [ -n "$FIP" ] && break; sleep 15
done
[ -n "$FIP" ] || { emit "$ID" "$VR_FAIL" "LB never got an EXTERNAL-IP (provisioning broken)"; exit "$VR_FAIL"; }
echo "  LB EXTERNAL-IP=$FIP"

rr_distinct(){ # echo count of distinct backend hostnames over NPROBE curls
  local i h; declare -A seen=()
  for i in $(seq 1 "$NPROBE"); do
    h="$(curl -s --max-time 5 "http://$FIP/hostname" 2>/dev/null || true)"
    [ -n "$h" ] && seen["$h"]=1
  done
  echo "${#seen[@]}"
}
codes_ok(){ # echo count of 200s over N curls
  local i c n=0; for i in $(seq 1 "$1"); do
    c="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://$FIP/hostname" 2>/dev/null || true)"
    [ "$c" = 200 ] && n=$((n+1)); done; echo "$n"
}

# --- TIER 1: round-robin ---
D="$(rr_distinct)"
if [ "$D" -ge 2 ]; then echo "  round-robin: $D distinct backends (PASS)"; else echo "  round-robin: only $D distinct backend(s) (FAIL)"; FAILS=$((FAILS+1)); fi

# --- TIER 2: member failover + recovery ---
POD1="$(kubectl get pods -l app="$SVC" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [ -n "$POD1" ]; then
  echo "  member failover: deleting $POD1"
  run kubectl delete pod "$POD1" --now || true
  OK="$(codes_ok 6)"
  if [ "$OK" -ge 5 ]; then echo "  continuity during member loss: $OK/6 200s (PASS)"; else echo "  continuity broken: $OK/6 200s (FAIL)"; FAILS=$((FAILS+1)); fi
  kubectl wait --for=condition=available --timeout=120s deploy/"$SVC" >/dev/null 2>&1 || true
  D2="$(rr_distinct)"
  if [ "$D2" -ge 2 ]; then echo "  recovery: back to $D2 distinct backends (PASS)"; else echo "  recovery: only $D2 backend(s) after heal (FAIL)"; FAILS=$((FAILS+1)); fi
else echo "  member failover: could not identify a backend pod (FAIL)"; FAILS=$((FAILS+1)); fi

# --- locate the Octavia LB for this service (admin), by name substring ---
LBID="$(openstack loadbalancer list -f json </dev/null 2>/dev/null | jq -r --arg s "$SVC" '.[] | select(.name|test($s)) | .id' | head -1 || true)"

# --- TIER 3: amphora failover (disruptive only, headroom-guarded) ---
amphora_headroom(){ # echo OK|NO|UNKNOWN for +1 amphora of the LB's amphora flavor
  local lb="$1" aj cid sj fl fv fr hj free_ok=0
  aj="$(openstack loadbalancer amphora list --loadbalancer "$lb" -f json </dev/null 2>/dev/null || true)"
  cid="$(jq -r '.[0].compute_id // empty' <<<"$aj" 2>/dev/null || true)"
  [ -n "$cid" ] || { echo UNKNOWN; return; }
  sj="$(openstack server show "$cid" -f json </dev/null 2>/dev/null || true)"
  fl="$(jq -r '.flavor // empty' <<<"$sj" 2>/dev/null | grep -oE '[A-Za-z0-9._-]+' | head -1 || true)"
  [ -n "$fl" ] || { echo UNKNOWN; return; }
  fv="$(openstack flavor show "$fl" -f json </dev/null 2>/dev/null | jq -r '.vcpus // empty' 2>/dev/null || true)"
  fr="$(openstack flavor show "$fl" -f json </dev/null 2>/dev/null | jq -r '.ram // empty' 2>/dev/null || true)"
  { [ -n "$fv" ] && [ -n "$fr" ]; } || { echo UNKNOWN; return; }
  hj="$(openstack hypervisor list --long -f json </dev/null 2>/dev/null || true)"
  [ -n "$hj" ] || { echo UNKNOWN; return; }
  # any hypervisor with free vcpu and ram for +1 amphora flavor?
  free_ok="$(jq -r --argjson v "$fv" --argjson r "$fr" '
    [ .[] | select(((.vcpus // 0)-(.vcpus_used // 0)) >= $v and ((.memory_mb // 0)-(.memory_mb_used // 0)) >= $r) ] | length' <<<"$hj" 2>/dev/null || echo 0)"
  case "$free_ok" in ''|0) echo NO;; *) echo OK;; esac
}

if vr_disruptive_ok; then
  if [ -z "$LBID" ]; then echo "  amphora failover: could not locate Octavia LB for $SVC (HOLD)"; HELD=1; AMPHORA_NOTE="amphora LB not located"
  else
    HR="$(amphora_headroom "$LBID")"
    echo "  amphora headroom (N+1 for STANDALONE failover): $HR"
    if [ "$HR" != OK ]; then
      echo "  amphora failover HELD: headroom $HR -- will not risk failover at/near ceiling"
      HELD=1; AMPHORA_NOTE="amphora failover held (headroom=$HR)"
    else
      echo "  triggering amphora failover on $LBID"
      run openstack loadbalancer failover "$LBID" || { FAILS=$((FAILS+1)); }
      PS=""
      for _ in $(seq 1 30); do
        PS="$(openstack loadbalancer show "$LBID" -f value -c provisioning_status </dev/null 2>/dev/null || true)"
        [ "$PS" = ACTIVE ] && break; [ "$PS" = ERROR ] && break; sleep 10
      done
      OK="$(codes_ok 6)"
      if [ "$PS" = ACTIVE ] && [ "$OK" -ge 5 ]; then echo "  amphora failover recovered: ACTIVE + $OK/6 200s (PASS)"; AMPHORA_NOTE="amphora failover PASS"
      else echo "  amphora failover did NOT recover (status=$PS, $OK/6 200s) (FAIL)"; FAILS=$((FAILS+1)); AMPHORA_NOTE="amphora failover FAIL"; fi
    fi
  fi
else
  echo "  amphora failover SKIPPED (needs --include-disruptive)"; AMPHORA_NOTE="amphora SKIPPED (not disruptive)"
fi

# --- verdict ---
if [ "$FAILS" -gt 0 ]; then emit "$ID" "$VR_FAIL" "$FAILS LB-pattern assertion(s) failed; $AMPHORA_NOTE"; exit "$VR_FAIL"; fi
if [ "$HELD" -gt 0 ]; then emit "$ID" "$VR_HOLD" "LB round-robin+member-failover PASS; $AMPHORA_NOTE"; exit "$VR_HOLD"; fi
emit "$ID" "$VR_PASS" "octavia LB pattern PASS (round-robin+member-failover+recovery); $AMPHORA_NOTE"; exit "$VR_PASS"