#!/usr/bin/env bash
# scripts/dc-node-v6-verify.sh <plan|node|bridges> <site> [spec] -- gate G19:
# do the DC nodes actually BRING UP their IPv6 statics, and does a plane carry
# IPv6 between two nodes? site = vr1-dc0 | vr1-dc1. Governing: D-139 (GUA carve),
# D-101 (v6 unless v4 is necessary), D-134 (node octets).
#
#   plan <site> [--profile P]  on a host holding the maas CLI. DERIVES the
#     expectation from live MAAS and prints one SPEC per machine. NO prefix
#     literal exists in this file: D-139 retires the VR1 ULA /48 for GUA, so a
#     baked table would be right today and wrong on carve day.
#   node <site> <spec>         ON the node: bash -s -- node <site> <spec>.
#     Read-only, no root; touches only ip and ping.
#   bridges <site>             on the RACK. DIAGNOSTIC, never a verdict: prints
#     each plane bridge's multicast_snooping/multicast_querier so a red `node`
#     can be triaged in one step. Exits 0 (reported) or 3 (could not read).
#
# WHAT A GREEN DOES NOT MEAN: this asserts the NIC against MAAS, not against
# D-139's ruled GUA table. MAAS-vs-ruling is dc-node-v6-carve.py check and
# tests/d139-gua-carve. `plan` prints the derived prefixes, so a reader can see
# which carve is being asserted.
#
# SPEC := entry[,entry...];  entry := iface=subnet_cidr=addr=peer=peerhost
#   peer '-' means no second machine on that plane -- that plane REFUSES.
#
# EXIT: 0 PASS | 1 FAIL | 2 bad args | 3 REFUSE (could not evaluate -- NEVER a
# pass). ASCII + LF.
set -uo pipefail

MODE="${1:-}"; SITE="${2:-}"
case "$MODE" in plan|node|bridges) ;; *)
  echo "usage: dc-node-v6-verify.sh <plan|node|bridges> <site> [spec]" >&2; exit 2 ;; esac
case "$SITE" in vr1-dc0|vr1-dc1) ;; *)
  echo "FAIL: unknown site '$SITE' (expected vr1-dc0 | vr1-dc1)" >&2; exit 2 ;; esac

# TEST SEAM. EMPTY in every production invocation, so the sysfs paths below are
# byte-identical to the absolute ones. Precedent: DC_SNAP_PROXY_SYSROOT.
SYSROOT="${DC_NODE_V6_SYSROOT:-}"

FAILED=0; REFUSED=0
ok(){      echo "  OK      $1"; }
no(){      echo "  FAIL    $1"; FAILED=1; }
refuse(){  echo "  REFUSE  $1"; REFUSED=1; }
verdict(){ # verdict <label>
  if [ "$FAILED" -ne 0 ]; then echo "dc-node-v6-verify $1 ($SITE): FAIL"; exit 1; fi
  if [ "$REFUSED" -ne 0 ]; then
    echo "dc-node-v6-verify $1 ($SITE): REFUSE -- could not evaluate; this is NOT a pass"; exit 3
  fi
  echo "dc-node-v6-verify $1 ($SITE): PASS"; exit 0
}

# ---------------------------------------------------------------------------
# plan -- MAAS is the AUTHORITY FOR THE EXPECTATION, never for the verdict.
# ---------------------------------------------------------------------------
do_plan() {
  local profile="${MAAS_PROFILE:-admin}" src="built-in default" a
  [ -n "${MAAS_PROFILE:-}" ] && src="MAAS_PROFILE env"
  shift 2 || true
  while [ $# -gt 0 ]; do
    case "$1" in
      --profile) profile="${2:-}"; src="--profile"; shift 2 || exit 2 ;;
      *) echo "FAIL: unknown plan option '$1'" >&2; exit 2 ;;
    esac
  done
  [ -n "$profile" ] || { echo "FAIL: --profile needs a value" >&2; exit 2; }
  echo "dc-node-v6-verify plan ($SITE) -- maas profile '$profile' (from $src)"
  command -v maas >/dev/null 2>&1 || {
    refuse "no 'maas' CLI on this host -- a MISSING TOOL is not an empty MAAS; run from the D-128 Plane-2 host"
    verdict plan; }
  command -v python3 >/dev/null 2>&1 || { refuse "no python3 -- cannot parse the MAAS answer"; verdict plan; }
  local js
  js="$(maas "$profile" machines read 2>&1)" || {
    refuse "'maas $profile machines read' failed -- ${js:0:200}"; verdict plan; }
  printf '%s' "$js" | python3 -c "$PLAN_PY" "openstack-$SITE" "$SITE"
  exit $?
}

# Emits the OK/FAIL/REFUSE lines, the SPEC block and the final verdict itself,
# so plan has exactly one exit contract. Grouping and peer choice are why this
# is python and not jq.
PLAN_PY='
import json, sys
tag, site = sys.argv[1], sys.argv[2]
failed = refused = 0
def ok(m): print("  OK      " + m)
def no(m):
    global failed; failed = 1; print("  FAIL    " + m)
def rf(m):
    global refused; refused = 1; print("  REFUSE  " + m)
try:
    machines = json.load(sys.stdin)
except Exception as e:
    rf("unparseable JSON from machines read: %s" % e); machines = None
if machines is not None:
    sel = [m for m in machines if tag in (m.get("tag_names") or [])]
    if not sel:
        rf("no machine carries the tag %s -- refusing a heuristic fallback; the "
           "wrong region or an uncarved DC both look like this" % tag)
    else:
        ok("tag %s selects %d machine(s)" % (tag, len(sel)))
        links = {}
        for m in sel:
            h = m["hostname"]
            for i in m.get("interface_set", []) or []:
                for l in i.get("links", []) or []:
                    ip = l.get("ip_address"); sub = l.get("subnet") or {}
                    cidr = sub.get("cidr") or ""
                    if ip and ":" in str(ip) and ":" in cidr:
                        links.setdefault((h, cidr), []).append((i["name"], str(ip)))
        dup = sorted(k for k, v in links.items() if len(v) > 1)
        if dup:
            for h, c in dup[:6]:
                no("%s holds %d v6 links on plane %s (%s) -- the expected address "
                   "is AMBIGUOUS; resolve in MAAS before gating"
                   % (h, len(links[(h, c)]), c, links[(h, c)]))
        else:
            ok("no machine holds two v6 links on one plane")
        sets = {}
        for m in sel:
            sets[m["hostname"]] = frozenset(c for (h, c) in links if h == m["hostname"])
        empty = sorted(h for h, s in sets.items() if not s)
        for h in empty[:6]:
            no("%s carries NO v6 link in MAAS -- nothing to assert on its NICs" % h)
        nonempty = {h: s for h, s in sets.items() if s}
        if nonempty:
            modal = max(set(nonempty.values()), key=lambda s: list(nonempty.values()).count(s))
            odd = sorted(h for h, s in nonempty.items() if s != modal)
            if odd:
                for h in odd[:6]:
                    no("%s v6 plane set DIFFERS from the other machines: missing %s / extra %s"
                       % (h, sorted(modal - nonempty[h]) or "-", sorted(nonempty[h] - modal) or "-"))
            else:
                ok("v6 plane set is IDENTICAL across %d machine(s): %d plane(s)" % (len(nonempty), len(modal)))
            for c in sorted(modal):
                print("            %s" % c)
            for c in sorted(modal):
                holders = sorted(h for h in nonempty if c in nonempty[h])
                if len(holders) < 2:
                    rf("plane %s has only %d machine(s) with a v6 static -- assertion (2), "
                       "does the plane CARRY v6, cannot be evaluated for it" % (c, len(holders)))
        print("== SPEC (arg 3 to `node`; one line per machine) ==")
        for h in sorted(nonempty):
            ents = []
            for c in sorted(nonempty[h]):
                iface, addr = links[(h, c)][0]
                peers = sorted((links[(g, c)][0][1], g) for g in nonempty if c in nonempty[g] and g != h)
                p, ph = peers[0] if peers else ("-", "-")
                ents.append("%s=%s=%s=%s=%s" % (iface, c, addr, p, ph))
            print("%s  %s" % (h, ",".join(ents)))
print("dc-node-v6-verify plan (%s): %s" % (site, "FAIL" if failed else
      ("REFUSE -- could not evaluate; this is NOT a pass" if refused else "PASS")))
sys.exit(1 if failed else (3 if refused else 0))
'

# ---------------------------------------------------------------------------
# node -- the assertion MAAS cannot make: a held link is not a carried address.
# ---------------------------------------------------------------------------
NEIGH_UP="REACHABLE STALE DELAY PROBE PERMANENT NOARP"
NEIGH_DOWN="FAILED INCOMPLETE"

do_node() {
  local spec="${3:-}" e iface cidr addr peer peerhost plen want link addrs line state prc
  [ -n "$spec" ] || { echo "FAIL: node needs a SPEC (get one from: dc-node-v6-verify.sh plan $SITE)" >&2; exit 2; }
  echo "dc-node-v6-verify node ($SITE) -- on $(uname -n 2>/dev/null || echo '?')"
  command -v ip >/dev/null 2>&1 || { refuse "no 'ip' on this node -- cannot evaluate any address"; verdict node; }
  local have_ping=1; command -v ping >/dev/null 2>&1 || have_ping=0

  local IFS_SAVE="$IFS"; local -a ENTRIES
  IFS=',' read -ra ENTRIES <<<"$spec"; IFS="$IFS_SAVE"
  for e in "${ENTRIES[@]}"; do
    IFS='=' read -r iface cidr addr peer peerhost <<<"$e"
    if [ -z "${peerhost:-}" ] || [ -z "$addr" ] || [ -z "$cidr" ] || [ -z "$iface" ]; then
      echo "FAIL: malformed SPEC entry '$e' (want iface=cidr=addr=peer=peerhost)" >&2; exit 2
    fi
    plen="${cidr##*/}"; want="$addr/$plen"

    # (1a) the link itself. An address on a carrier-less NIC carries nothing.
    link="$(ip -o link show dev "$iface" 2>/dev/null)"
    if [ -z "$link" ]; then
      no "$iface: interface ABSENT on this node, but MAAS holds $want on it"; continue
    elif ! grep -q 'LOWER_UP' <<<"$link"; then
      no "$iface ($cidr): no LOWER_UP -- no carrier, so $want cannot be carried"; continue
    fi

    # (1b) THE address, and ONLY it. An extra global is the stale-prefix
    # signature D-139's ULA->GUA move will produce if the old link is not removed.
    addrs="$(ip -6 -o addr show dev "$iface" scope global 2>/dev/null)"
    line="$(awk -v w="$want" '$4==w' <<<"$addrs")"
    if [ -z "$line" ]; then
      no "$iface ($cidr): MAAS holds $want but the NIC does not carry it -- a held link is not a carried address"
    elif grep -qw 'dadfailed' <<<"$line"; then
      no "$iface: $want is DADFAILED -- duplicate address on $cidr"
    elif grep -qw 'tentative' <<<"$line"; then
      no "$iface: $want is still TENTATIVE -- DAD has not completed, the address is not usable"
    else
      local extra
      extra="$(awk -v w="$want" '$4!=w && $4!=""{print $4}' <<<"$addrs")"
      if [ -n "$extra" ]; then
        no "$iface ($cidr): carries $want AND unexpected global(s) $(tr '\n' ' ' <<<"$extra")-- MAAS holds exactly one. A stale prefix left behind by a re-carve is this shape; so is an HA VIP, which is NOT a defect -- see the D-139 VIP row note in docs/audit/g19-ipv6-plane-gate-build-20260801.txt"
      else
        ok "$iface ($cidr): $want up, global, DAD complete, sole global on the NIC"
      fi
    fi

    # (2) the PLANE, not the node. ping first: it is what populates the cache
    # that the next assertion reads.
    if [ "$peer" = "-" ]; then
      refuse "$iface ($cidr): no second machine holds a v6 static -- the data path cannot be evaluated from here"
      continue
    fi
    if [ "$have_ping" -eq 0 ]; then
      refuse "$iface ($cidr): no 'ping' on this node -- reachability to $peer is UNKNOWN, not absent"; continue
    fi
    ping -6 -c 3 -W 2 -n -I "$iface" "$peer" >/dev/null 2>&1; prc=$?
    state="$(ip -6 neigh show to "$peer" dev "$iface" 2>/dev/null | awk 'NF{print $NF}' | tail -1)"
    [ -n "$state" ] || state="ABSENT"
    if [ "$prc" -gt 1 ]; then
      refuse "$iface ($cidr): ping to $peer ($peerhost) exited $prc -- the probe could not run; reachability UNKNOWN"
    elif grep -qw "$state" <<<"$NEIGH_UP"; then
      if [ "$prc" -eq 0 ]; then ok "$iface ($cidr): $peer ($peerhost) replies, neighbour $state"
      else no "$iface ($cidr): NO reply from $peer ($peerhost) but the neighbour RESOLVED ($state) -- ND worked, so this is NOT the multicast-ND class; look at filtering or the peer's stack"; fi
    elif grep -qw "$state" <<<"$NEIGH_DOWN" || [ "$state" = "ABSENT" ]; then
      if [ "$prc" -eq 0 ]; then no "$iface ($cidr): $peer ($peerhost) replied YET the neighbour is $state -- CONTRADICTION; one of the two readings is lying, do not close a gate on it"
      else no "$iface ($cidr): no reply from $peer ($peerhost) and the neighbour is $state -- Neighbour Discovery did not resolve; run 'dc-node-v6-verify.sh bridges $SITE' on the rack before blaming routing"; fi
    else
      refuse "$iface ($cidr): neighbour state '$state' for $peer is not one this gate recognises -- refusing to interpret it"
    fi
  done
  verdict node
}

# ---------------------------------------------------------------------------
# bridges -- DIAGNOSTIC. Reports; asserts nothing about Linux behaviour. The v6
# planes carry gateway_ip=None (D-139), so peers are on-link and ND -- which is
# multicast -- IS the resolution mechanism. Exit 0 reported | 3 could not read.
# ---------------------------------------------------------------------------
do_bridges() {
  local nets n br s q missing=0 seen=0
  echo "dc-node-v6-verify bridges ($SITE) -- DIAGNOSTIC ONLY, no verdict"
  command -v virsh >/dev/null 2>&1 || {
    echo "  REFUSE  no 'virsh' on this host -- run on the DC rack"; exit 3; }
  nets="$(virsh -c qemu:///system net-list --all --name 2>/dev/null | grep -E "^${SITE}-" )"
  if [ -z "$nets" ]; then
    echo "  REFUSE  no libvirt network named ${SITE}-* -- the plane bridges are UNKNOWN, not absent"; exit 3
  fi
  while read -r n; do
    [ -n "$n" ] || continue
    seen=$((seen+1))
    br="$(virsh -c qemu:///system net-dumpxml "$n" 2>/dev/null | sed -n "s/.*bridge name='\([^']*\)'.*/\1/p")"
    if [ -z "$br" ]; then
      echo "  REFUSE  $n did not resolve to a bridge"; missing=1; continue
    fi
    s="$(cat "${SYSROOT}/sys/class/net/${br}/bridge/multicast_snooping" 2>/dev/null)"
    q="$(cat "${SYSROOT}/sys/class/net/${br}/bridge/multicast_querier" 2>/dev/null)"
    if [ -z "$s" ] || [ -z "$q" ]; then
      echo "  REFUSE  $n ($br): multicast state unreadable under ${SYSROOT}/sys/class/net/${br}/bridge/"; missing=1; continue
    fi
    printf '  READ    %-28s %-10s multicast_snooping=%s multicast_querier=%s%s\n' \
      "$n" "$br" "$s" "$q" "$([ "$s" = 1 ] && [ "$q" = 0 ] && echo '  <-- SUSPECT')"
  done <<<"$nets"
  echo "  NOTE    snooping=1 with querier=0 is REPORTED as suspect, not diagnosed: IPv6"
  echo "          Neighbour Discovery is multicast, and no Linux behaviour is asserted here."
  [ "$missing" -eq 0 ] || { echo "dc-node-v6-verify bridges ($SITE): REFUSE -- $seen network(s) seen, at least one unreadable"; exit 3; }
  echo "dc-node-v6-verify bridges ($SITE): reported $seen bridge(s)"; exit 0
}

case "$MODE" in
  plan)    do_plan "$@" ;;
  node)    do_node "$@" ;;
  bridges) do_bridges ;;
esac