Newer
Older
openstack-caracal-dc-dc / tests / preflight / run-tests.sh
#!/usr/bin/env bash
# tests/preflight/run-tests.sh -- offline harness for preflight.sh (DOCFIX-073).
# Builds a fixture repo whose gate scripts are stubs with controlled exits, plus
# a fakebin juju for the channel assert. Asserts ordering, aggregation (worst of
# 0/1/2), and fail-closed behavior on a missing gate. Mutates nothing outside $TMP.
# Exit: 0 all pass | 1 any case failed.  ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0

mkfix() { # mkfix <name> <lint_rc> <bundlechk_rc> <preflight_present:0|1> [creds_rc|"omit"]
  local d="$TMP/$1"; mkdir -p "$d/scripts" "$d/fakebin"
  cp "$REPO/scripts/preflight.sh" "$REPO/scripts/channel_assert.py" "$d/scripts/"
  printf 'applications:\n  keystone: {charm: keystone, channel: 2024.1/stable}\n' > "$d/bundle.yaml"
  printf '#!/usr/bin/env bash\necho "  [stub] lint"\nexit %s\n' "$2" > "$d/scripts/repo-lint.sh"
  printf '#!/usr/bin/env python3\nimport sys;print("  [stub] bundlechk");sys.exit(%s)\n' "$3" > "$d/scripts/provider-bundle-check.py"
  if [ "$4" = "1" ]; then
    printf '#!/usr/bin/env bash\necho "  [stub] live preflight"\nexit 0\n' > "$d/scripts/pre-flight-checks.sh"
  fi
  # P5 credential matrix (D-137 ruling 1). "omit" leaves the gate script absent so the
  # fail-closed path can be exercised.
  if [ "${5:-0}" != "omit" ]; then
    printf '#!/usr/bin/env python3\nimport sys;print("  [stub] creds-matrix");sys.exit(%s)\n' \
      "${5:-0}" > "$d/scripts/creds-matrix.py"
  fi
  # fakebin juju: publishes 2024.1/stable in the channel map
  cat > "$d/fakebin/juju" <<'FB'
#!/usr/bin/env bash
if [ "$1" = "info" ]; then
  printf 'name: %s\nchannels:\n  2024.1/stable: 778 2024-05-01\n' "$2"; exit 0
fi
exit 1
FB
  chmod +x "$d/fakebin/juju"
  echo "$d"
}
run() { # run <want_rc> <regex> <label> <dir>
  local out rc
  out="$(PATH="$4/fakebin:$PATH" bash "$4/scripts/preflight.sh" "$4" 2>&1)"; rc=$?
  if [[ "$rc" == "$1" ]] && grep -qE "$2" <<<"$out"; then
    echo "  PASS  $3"; PASS=$((PASS+1))
  else
    echo "  FAIL  $3 (rc=$rc want=$1)"; echo "$out" | tail -6 | sed 's/^/        /'; FAIL=$((FAIL+1))
  fi
}

D=$(mkfix all_pass 0 0 1);   run 0 'PREFLIGHT: PASS' "T1 all gates pass -> PASS" "$D"
D=$(mkfix lint_fail 1 0 1);  run 1 'PREFLIGHT: FAIL' "T2 lint FAIL -> overall FAIL" "$D"
D=$(mkfix bundle_fail 0 1 1); run 1 'PREFLIGHT: FAIL' "T3 bundle FAIL -> overall FAIL" "$D"
D=$(mkfix warn_only 2 0 1);  run 2 'PREFLIGHT: WARN' "T4 lint WARN -> overall WARN" "$D"
D=$(mkfix missing_gate 0 0 0); run 1 'pre-flight-checks.sh missing' "T5 missing gate FAILS CLOSED" "$D"
# T6: channel pinned to a track fakebin does not publish -> FAIL
D=$(mkfix badpin 0 0 1); printf 'applications:\n  vault: {charm: vault, channel: 9.9/stable}\n' > "$D/bundle.yaml"
run 1 'does not publish' "T6 unpublished channel pin FAILS" "$D"
# T7: charmhub unreachable -> WARN (exit 2), not FAIL.
# HERMETIC: simulate failure with a FAILING fake, never by deleting the fake --
# deletion un-shadows the REAL juju on a real host (the jumphost gauntlet run
# fell through to live charmhub and returned 0). Fakes must always shadow.
D=$(mkfix offline 0 0 1)
cat > "$D/fakebin/juju" <<'FB'
#!/usr/bin/env bash
echo "ERROR cannot connect to charmhub.io: connection timed out" >&2; exit 1
FB
chmod +x "$D/fakebin/juju"
run 2 'verify .* manually' "T7 charmhub unreachable -> WARN" "$D"

# T8 -- D-137 ruling 1: the credential gate is BLOCKING. A red register must stop a
# deploy, not warn about it -- advisory was the posture that demonstrably failed.
D=$(mkfix creds_fail 0 0 1 1); run 1 'PREFLIGHT: FAIL' "T8 credential matrix FAIL -> overall FAIL" "$D"
# T9 -- the credential gate must FAIL CLOSED if its checker is missing, exactly as T5
# requires of pre-flight-checks.sh. A gate that vanishes must never read as a pass.
D=$(mkfix creds_gone 0 0 1 omit); run 1 'PREFLIGHT: FAIL' "T9 credential gate missing -> FAILS CLOSED" "$D"
# T10 -- ordering: the credential gate runs with the REAL gates, before the stage-2
# reminders block, so its verdict participates in the deploy decision.
D=$(mkfix creds_order 0 0 1 0)
out="$(PATH="$D/fakebin:$PATH" bash "$D/scripts/preflight.sh" "$D" 2>&1)"
if [[ "$(grep -n 'P5: credential matrix' <<<"$out" | cut -d: -f1)" -lt \
      "$(grep -n 'P6: stage-2 reminders' <<<"$out" | cut -d: -f1)" ]]; then
  echo "  PASS  T10 credential gate runs BEFORE the reminders block"; PASS=$((PASS+1))
else
  echo "  FAIL  T10 credential gate ordering"; FAIL=$((FAIL+1))
fi

# ---- T11-T16 -- R15(3): PREFLIGHT MUST NOT FAIL OPEN (ruled 2026-07-27) ----
# MEASURED DEFECT: note() tested only rc 1 and 2, so a sub-gate exiting 127/126/130
# or any rc>=3 left WORST untouched and preflight printed
#   "PREFLIGHT: PASS -- clear to add-model / deploy"
# Reproduced for all four before the fix. This is the gate that AUTHORISES the deploy.
run 1 'PREFLIGHT: FAIL' "T11 R15(3) sub-gate exit 127 (command not found) -> FAIL, was PASS" "$(mkfix rc127 127 0 1 0)"
run 1 'PREFLIGHT: FAIL' "T12 R15(3) sub-gate exit 126 (not executable) -> FAIL, was PASS" "$(mkfix rc126 0 126 1 0)"
run 1 'PREFLIGHT: FAIL' "T13 R15(3) sub-gate exit 3 (undefined) -> FAIL, was PASS" "$(mkfix rc3 0 0 1 3)"

# rc=2 is PER-GATE and was measured per checker rather than blanket-remapped.
# P2/P5 rc2 means "could not evaluate" -> FAIL. A gate that could not look must never
# read as a gate that found nothing.
run 1 'COULD NOT EVALUATE' "T14 R15(3) P2 rc=2 (could-not-evaluate) -> FAIL" "$(mkfix p2two 0 2 1 0)"
run 1 'COULD NOT EVALUATE' "T15 R15(3) P5 rc=2 (could-not-evaluate) -> FAIL" "$(mkfix p5two 0 0 1 2)"

# P1 rc2 is "warnings only" by its own documented contract. It must STAY a warning --
# a blanket 2->FAIL would have turned the standing L1 legacy-ASCII warn into a deploy
# blocker. This case is the regression lock on NOT over-correcting.
run 2 'PREFLIGHT: WARN' "T16 R15(3) P1 rc=2 (warnings only) stays WARN, not FAIL" "$(mkfix p1two 2 0 1 0)"

echo; echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1