Newer
Older
openstack-caracal-dc-dc / tests / octavia-pki / run-tests.sh
#!/usr/bin/env bash
# tests/octavia-pki/run-tests.sh -- harness for scripts/octavia-pki.sh verify
#
# WHAT THIS HARNESS IS FOR. The script asserts things the runbook only hoped for, so the
# assertions themselves must be proven capable of FAILING -- "a checker that cannot fail is
# not a gate". Every case below constructs a specific defect and requires the script to
# catch it. T1 is the non-zero floor: a CORRECT fixture must PASS, because a script that
# fails on everything is as useless as one that passes on everything.
#
# THE CASE THAT MATTERS MOST IS T2. F8 records that the runbook's heredoc paste hazard can
# yield a controller certificate with NO SANs while every command still prints OK, and that
# nothing in this repo asserted the SAN set. T2 is that assertion.
#
# T12 guards the opposite error: openssl prints IPv6 SANs EXPANDED AND UPPERCASE
# (`2602:F3E2:F02:11:0:0:0:57`) while the overlay carries the compressed form. A naive
# string compare would FAIL a CORRECT certificate. T12 proves we do not.
#
# Fixtures are thrown away. Nothing here touches the real ~/octavia-pki or the real repo:
# the script's OCTAVIA_PKI_HOME / OCTAVIA_PKI_REPO / OCTAVIA_PKI_THIS_HOST overrides exist
# for exactly this reason.

set -uo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
CHECK="$REPO/scripts/octavia-pki.sh"

P=0; F=0
ok() { P=$((P+1)); echo "  [ok]   $1"; }
no() { F=$((F+1)); echo "FAIL   $1"; }

TMPROOT="$(mktemp -d)"
trap 'rm -rf "$TMPROOT"' EXIT

V4="10.12.4.57"; V6="2602:f3e2:f02:11::57"
KV4="10.12.4.50"

# Build a throwaway fixture: a fake repo (git-initialised so check-ignore works) plus a
# per-DC PKI workspace. $1=env name  $2=site  $3=CA label  $4=SAN mode
#   SAN mode: full | none | v4only | wrongv4
mkfix() {
  local name="$1" site="$2" label="$3" sanmode="$4" dnsbase="${5:-example}"
  local E="$TMPROOT/$name"; local R="$E/repo"; local W="$E/home/octavia-pki/$site"
  mkdir -p "$R/overlays" "$R/creds-manifests" "$W/issuing-ca" "$W/controller-ca" "$W/controller"

  printf 'headend fixturehost\n' > "$R/creds-manifests/host-identity"
  printf 'overlays/*octavia-pki.yaml\n' > "$R/.gitignore"
  git -C "$R" init -q 2>/dev/null; git -C "$R" add -A >/dev/null 2>&1 || true

  cat > "$R/overlays/${site}-vips.yaml" <<YML
applications:
  keystone:
    options:
      vip: "$KV4 10.12.8.50 10.12.12.50"
  octavia:
    options:
      vip: "$V4 10.12.8.57 10.12.12.57 $V6 fd50:840e:74e2:220::57 fd50:840e:74e2:221::57"
YML

  # two self-signed CAs, distinct trust domains
  ( cd "$W/issuing-ca"
    openssl req -new -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-384 -sha384 -nodes \
      -days 3650 -subj "/CN=${label} Omega Cloud Octavia Issuing CA/O=Neumatrix" \
      -keyout issuing-ca.key.enc -out issuing-ca.cert.pem >/dev/null 2>&1
    printf '0123456789012345678901234567890123456789012\n' > passphrase.txt
    chmod 600 passphrase.txt issuing-ca.key.enc; chmod 644 issuing-ca.cert.pem )
  ( cd "$W/controller-ca"
    openssl req -new -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-384 -sha384 -nodes \
      -days 3650 -subj "/CN=${label} Omega Cloud Octavia Controller CA/O=Neumatrix" \
      -keyout controller-ca.key.enc -out controller-ca.cert.pem >/dev/null 2>&1
    printf '0123456789012345678901234567890123456789012\n' > passphrase.txt
    chmod 600 passphrase.txt controller-ca.key.enc; chmod 644 controller-ca.cert.pem )

  ( cd "$W/controller"
    openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out controller.key >/dev/null 2>&1
    chmod 600 controller.key
    { printf '[req]\ndistinguished_name=dn\nreq_extensions=v3_req\nprompt=no\n\n[dn]\nCN=octavia-controller.example\nO=Neumatrix\n\n[v3_req]\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=clientAuth,serverAuth\n'
      case "$sanmode" in
        none) : ;;   # F8's failure mode: NO subjectAltName at all
        *) printf 'subjectAltName=@alt_names\n\n[alt_names]\nDNS.1=octavia-controller.%s\nDNS.2=octavia.%s\n' "$dnsbase" "$dnsbase" ;;
      esac
      case "$sanmode" in
        full)    printf 'IP.1 = %s\nIP.2 = %s\n' "$V4" "$V6" ;;
        v4only)  printf 'IP.1 = %s\n' "$V4" ;;
        wrongv4) printf 'IP.1 = %s\nIP.2 = %s\n' "10.99.99.99" "$V6" ;;
      esac
    } > controller.cnf
    openssl req -new -sha256 -key controller.key -config controller.cnf -out controller.csr >/dev/null 2>&1
    openssl x509 -req -sha256 -in controller.csr -CA ../controller-ca/controller-ca.cert.pem \
      -CAkey ../controller-ca/controller-ca.key.enc -CAcreateserial -days 730 \
      -extfile controller.cnf -extensions v3_req -out controller.cert.pem >/dev/null 2>&1
    # 600 on the certs AND the serial, matching the generator's E2 fix at source. The serial is
    # created by -CAcreateserial at the inherited umask, so it must be tightened explicitly --
    # exactly the omission that let the live material sit 664 with `verify` still reading 26/0.
    chmod 600 controller.cert.pem ../controller-ca/controller-ca.cert.srl
    cat controller.cert.pem controller.key > controller.bundle.pem; chmod 600 controller.bundle.pem )

  # the deploy overlay: 5 keys, ASCII, 0600, and gitignored by the fixture .gitignore
  { echo "applications:"; echo "  octavia:"; echo "    options:"
    for k in issuing-cacert issuing-ca-private-key issuing-ca-key-passphrase controller-cacert controller-cert; do
      echo "      lb-mgmt-${k}: \"QUJD\""
    done; } > "$R/overlays/${site}-octavia-pki.yaml"
  chmod 600 "$R/overlays/${site}-octavia-pki.yaml"
  echo "$E"
}

run() {  # run <env-dir> <site> [host] -> sets RC, OUT
  local E="$1" site="$2" host="${3:-fixturehost}"
  OUT="$(OCTAVIA_PKI_REPO="$E/repo" OCTAVIA_PKI_HOME="$E/home" OCTAVIA_PKI_THIS_HOST="$host" \
        bash "$CHECK" verify "$site" 2>&1)"; RC=$?
}

echo "=== octavia-pki harness ==="

# T1 -- NON-ZERO FLOOR. A correct fixture must PASS, or nothing below proves anything.
E=$(mkfix good vr1-dc0 "VR1 DC0" full); run "$E" vr1-dc0
[ "$RC" = 0 ] && ok "T1 a correct PKI PASSES (non-zero floor)" \
  || { no "T1 baseline must pass (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T2 -- THE F8 CASE. No subjectAltName at all: the runbook's paste hazard output.
E=$(mkfix nosan vr1-dc0 "VR1 DC0" none); run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "NO subjectAltName" \
  && ok "T2 a cert with NO SANs FAILS -- the defect nothing previously asserted (F8)" \
  || { no "T2 must catch a SAN-less certificate (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T3 -- v6 SAN missing while the overlay declares a v6 provider leg (D-109 note 2026-07-29).
E=$(mkfix v4only vr1-dc0 "VR1 DC0" v4only); run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "MISSING this DC's provider v6" \
  && ok "T3 a v4-only SAN set FAILS when the overlay declares a v6 leg" \
  || { no "T3 must catch a missing v6 SAN (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T4 -- the SAN carries an address that is not this DC's VIP.
E=$(mkfix wrongip vr1-dc0 "VR1 DC0" wrongv4); run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "MISSING this DC's provider v4" \
  && ok "T4 a SAN naming the wrong v4 address FAILS" \
  || { no "T4 must catch a wrong v4 SAN (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T5 -- WRONG DC in the CA subject. This is the sibling of F8: the runbook guards ${DC:?}
# but NOT ${DC_LABEL}, so an unset label yields a DC-less CA with no error anywhere.
E=$(mkfix wronglabel vr1-dc0 "VR1 DC1" full); run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "A4 issuing CA subject does NOT contain" \
  && ok "T5 a CA subject naming the wrong DC FAILS" \
  || { no "T5 must catch a wrong-DC CA subject (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T6 -- E2: a group-writable CA certificate is a SUBSTITUTABLE TRUST ANCHOR. This is the
# finding the generating session dismissed as harmless, so it is pinned here.
E=$(mkfix gwcert vr1-dc0 "VR1 DC0" full)
chmod 664 "$E/home/octavia-pki/vr1-dc0/issuing-ca/issuing-ca.cert.pem"; run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "WRITABLE by group/other" \
  && ok "T6 a group-writable CA cert FAILS (E2)" \
  || { no "T6 must catch a group-writable CA cert (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T7 -- F4: the overlay carries CA key blobs plus a plaintext passphrase. If it is not
# gitignored, a CA private key is committable in a repo SEC-004 records as PUBLIC.
E=$(mkfix notignored vr1-dc0 "VR1 DC0" full)
: > "$E/repo/.gitignore"; run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "NOT gitignored" \
  && ok "T7 an un-gitignored overlay FAILS (F4)" \
  || { no "T7 must catch an un-gitignored overlay (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T8 -- a truncated overlay that still parses. 5 keys is the contract.
E=$(mkfix shortovl vr1-dc0 "VR1 DC0" full)
printf 'applications:\n  octavia:\n    options:\n      lb-mgmt-issuing-cacert: "QUJD"\n' \
  > "$E/repo/overlays/vr1-dc0-octavia-pki.yaml"
chmod 600 "$E/repo/overlays/vr1-dc0-octavia-pki.yaml"; run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "lb-mgmt-\* key(s), expected 5" \
  && ok "T8 an overlay with the wrong key count FAILS" \
  || { no "T8 must catch a short overlay (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T9 -- WRONG HOST must REFUSE, never report absence. The F6 lesson: measuring the wrong
# machine's filesystem and calling the result a verdict is worse than declining to look.
E=$(mkfix wronghost vr1-dc0 "VR1 DC0" full); run "$E" vr1-dc0 someotherhost
[ "$RC" = 3 ] && printf '%s' "$OUT" | grep -q "PKI lives on" \
  && ok "T9 a run on the wrong host REFUSES (exit 3), not FAIL and not PASS" \
  || { no "T9 wrong host must refuse (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T10 -- an ABSENT host binding must REFUSE rather than assume this host is the right one.
E=$(mkfix nobinding vr1-dc0 "VR1 DC0" full)
: > "$E/repo/creds-manifests/host-identity"; run "$E" vr1-dc0
[ "$RC" = 3 ] && printf '%s' "$OUT" | grep -q "no 'headend' binding" \
  && ok "T10 a missing headend binding REFUSES rather than guessing" \
  || { no "T10 missing binding must refuse (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T11 -- an absent workspace is a CONCLUSIVE observation, so FAIL (not REFUSE): we looked.
E=$(mkfix nowork vr1-dc0 "VR1 DC0" full)
rm -rf "$E/home/octavia-pki/vr1-dc0"; run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "has not been generated" \
  && ok "T11 an un-generated DC FAILS (looked, nothing there) rather than refusing" \
  || { no "T11 absent workspace must fail (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T12 -- R7's whole point: no cross-DC amphora root-of-trust. Before F1 the generator's
# paths were shared, so generating dc1 OVERWROTE dc0 and these were byte-identical.
E=$(mkfix indep vr1-dc0 "VR1 DC0" full)
mkdir -p "$E/home/octavia-pki/vr1-dc1"
cp -r "$E/home/octavia-pki/vr1-dc0/." "$E/home/octavia-pki/vr1-dc1/"
run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "IDENTICAL to vr1-dc1" \
  && ok "T12 shared material between DCs FAILS -- the state F1 existed to prevent" \
  || { no "T12 must catch cross-DC shared material (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T13 -- THE FALSE-FAIL GUARD. openssl prints v6 SANs expanded+uppercase while the overlay
# carries the compressed form. T1 already covers a correct cert, so this asserts the
# normalisation explicitly: the v6 assertion must be REACHED and PASS, not skipped.
E=$(mkfix v6norm vr1-dc0 "VR1 DC0" full); run "$E" vr1-dc0
printf '%s' "$OUT" | grep -q "SAN carries this DC's provider v6 VIP" \
  && ok "T13 an expanded/uppercase v6 SAN is normalised and matches (no false FAIL)" \
  || { no "T13 v6 normalisation must match a correct cert"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T16 -- the CA SERIAL file is issuance STATE, not residue. If it is missing, the next
# `-CAcreateserial` starts a fresh sequence and the same CA can issue a DUPLICATE serial.
# It is declared in creds-matrix.tsv, so the gate must assert it too -- the register and the
# gate have to agree on what should exist, or one of them is lying.
E=$(mkfix noserial vr1-dc0 "VR1 DC0" full)
rm -f "$E/home/octavia-pki/vr1-dc0/controller-ca/controller-ca.cert.srl"; run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "controller-ca.cert.srl" \
  && ok "T16 a missing CA serial file FAILS -- duplicate-serial risk on the next reissue" \
  || { no "T16 must catch a missing CA serial (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T17 -- the build INTERMEDIATES must NOT be required. The generator now deletes them
# (E3 fix at source), so a workspace without them is CORRECT and must still PASS. Without
# this case, adding them to A2 later would silently make every clean workspace fail.
E=$(mkfix nointermediates vr1-dc0 "VR1 DC0" full)
rm -f "$E/home/octavia-pki/vr1-dc0/controller/controller.csr" \
      "$E/home/octavia-pki/vr1-dc0/controller/controller.cnf"; run "$E" vr1-dc0
[ "$RC" = 0 ] && ok "T17 a workspace with the intermediates DELETED still PASSES (E3 fix at source)" \
  || { no "T17 deleted intermediates must not fail the gate (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T18 -- the CA SERIAL must be mode-checked too. This case exists because the first version of
# the script MISSED it: `verify` read 26/0 on live material while `creds-matrix.py` E2 reported
# the serial 664 on both DCs. Two gates that can both see a file must not disagree about it.
# The serial holds no secret, so the exposure is INTEGRITY -- rewriting it forces the next
# issuance to reuse a serial.
E=$(mkfix gwserial vr1-dc0 "VR1 DC0" full)
chmod 664 "$E/home/octavia-pki/vr1-dc0/controller-ca/controller-ca.cert.srl"; run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "controller-ca.cert.srl is 664" \
  && ok "T18 a group-writable CA serial FAILS -- duplicate-serial forcing (found by P5, not by this gate)" \
  || { no "T18 must catch a group-writable serial (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# --- A12: the DNS SANs, and the arming that makes F9 unmissable -------------------------
# F9's whole risk is that a future session forgets. So A12 arms itself from the change that
# makes the SANs matter (os-public-hostname appearing), rather than from a note.

# T19 -- UNARMED is the live posture: os-public-hostname set nowhere, so the DNS names resolve
# to nothing and must NOT fail the run. Without this case, arming A12 would have broken every
# current verify -- the same trap T17 guards for the deleted intermediates.
E=$(mkfix dnsinert vr1-dc0 "VR1 DC0" full); run "$E" vr1-dc0
[ "$RC" = 0 ] && printf '%s' "$OUT" | grep -q "A12 DNS SANs are INERT" \
  && ok "T19 with os-public-hostname unset, DNS SANs are INERT and the run still PASSES" \
  || { no "T19 unarmed A12 must not fail the run (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T20 -- ARMED with a wrong-region name is F9 exactly: the live certs say `dc0.vr0` on BOTH
# VR1 DCs, so once hostnames are live the name belongs to another deployment.
E=$(mkfix dnsarmed vr1-dc0 "VR1 DC0" full "omega.dc0.vr0.cloud.neumatrix.local")
printf '      os-public-hostname: keystone.omega.dc0.vr0.cloud.neumatrix.local\n' \
  >> "$E/repo/overlays/vr1-dc0-vips.yaml"
run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "is not in this DC's expected zone 'omega.dc0.vr1.cloud.neumatrix.local'" \
  && ok "T20 ARMED + a wrong-region DNS SAN FAILS (F9, self-arming)" \
  || { no "T20 armed A12 must fail a wrong-region SAN (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T21 -- ARMED with the CORRECT zone now PASSES. ASSERTION REPLACED, NOT DELETED (DOCFIX-205):
# this case previously asserted a REFUSAL (rc=3, "DC-LABEL cannot be validated") on the grounds
# that D-008 + D-106's `dc1.vr1`/`dc2.vr1` left the per-DC label unruled. That premise was
# already dead -- D-117 (2026-07-13) retired `dc1`/`dc2` in favour of `dc0`/`dc1` and says so in
# its own Status line. The label is derivable, so the check asserts it instead of refusing.
E=$(mkfix dnsregionok vr1-dc0 "VR1 DC0" full "omega.dc0.vr1.cloud.neumatrix.local")
printf '      os-public-hostname: keystone.omega.dc0.vr1.cloud.neumatrix.local\n' \
  >> "$E/repo/overlays/vr1-dc0-vips.yaml"
run "$E" vr1-dc0
[ "$RC" = 0 ] && printf '%s' "$OUT" | grep -q "A12 DNS SANs are all in this DC's expected zone" \
  && ok "T21 ARMED + the D-117-correct zone PASSES (assertion replaces the former refusal)" \
  || { no "T21 armed A12 must pass the correct zone (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T21b -- the capability the refusal never had: a WRONG DC LABEL in the RIGHT region. This is
# the exact cross-DC mix-up F1 proved possible on the PKI paths (dc1's material under dc0's
# name). Region-only checking passed it; the derived-zone assertion catches it.
E=$(mkfix dnswrongdc vr1-dc0 "VR1 DC0" full "omega.dc1.vr1.cloud.neumatrix.local")
printf '      os-public-hostname: keystone.omega.dc1.vr1.cloud.neumatrix.local\n' \
  >> "$E/repo/overlays/vr1-dc0-vips.yaml"
run "$E" vr1-dc0
[ "$RC" = 1 ] && printf '%s' "$OUT" | grep -q "is not in this DC's expected zone 'omega.dc0.vr1.cloud.neumatrix.local'" \
  && ok "T21b ARMED + the OTHER DC's label in the right region FAILS (cross-DC mix-up)" \
  || { no "T21b armed A12 must fail a wrong DC label (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T21c -- vr1-dc1 derives its OWN zone, so the check is not a dc0 constant wearing a variable.
E=$(mkfix dnsdc1ok vr1-dc1 "VR1 DC1" full "omega.dc1.vr1.cloud.neumatrix.local")
printf '      os-public-hostname: keystone.omega.dc1.vr1.cloud.neumatrix.local\n' \
  >> "$E/repo/overlays/vr1-dc1-vips.yaml"
run "$E" vr1-dc1
[ "$RC" = 0 ] && printf '%s' "$OUT" | grep -q "A12 DNS SANs are all in this DC's expected zone 'omega.dc1.vr1.cloud.neumatrix.local'" \
  && ok "T21c vr1-dc1 derives dc1.vr1, proving the zone is per-DC and not hardcoded" \
  || { no "T21c vr1-dc1 must derive its own zone (rc=$RC)"; printf '%s\n' "$OUT" | sed 's/^/       /'; }

# T14 -- args. A bare `dcN` has been a live defect class; it must be REJECTED, not guessed.
RC=0; bash "$CHECK" verify dc0 >/dev/null 2>&1 || RC=$?
[ "$RC" = 2 ] && ok "T14 a bare 'dc0' is REJECTED (exit 2), never coerced" || no "T14 bare site must exit 2 (rc=$RC)"
RC=0; bash "$CHECK" generate vr1-dc0 >/dev/null 2>&1 || RC=$?
[ "$RC" = 2 ] && ok "T15 'generate' exits 2 -- unimplemented pending the D-137 fork-1 ruling" || no "T15 generate must exit 2 (rc=$RC)"

echo
if [ "$F" = 0 ]; then echo "octavia-pki: $P/$P PASS"; exit 0; fi
echo "octavia-pki: FAILURES: $F (passed $P)"; exit 1