diff --git a/docs/v1-redeploy-changelog.md b/docs/v1-redeploy-changelog.md index 9c74d25..a0b0856 100644 --- a/docs/v1-redeploy-changelog.md +++ b/docs/v1-redeploy-changelog.md @@ -1605,3 +1605,48 @@ for throwaway measurement. Beta left at 2 workers (bonus resize acceptance coverage). Next-free: D-071, DOCFIX-086, BUNDLEFIX-009. + +### 2026-07-03 (addendum 8) -- validate.sh modular foundation: lib-validate + orchestrator + +D-011 validation runner rebuilt as a MODULAR suite (approved design: modular-from-start, +PASS-PENDING-MANUAL verdict tier, amphora failover in item 4, per-check --disruptive gate + +orchestrator --include-disruptive pass-through, machine-readable per-check result lines). +This addendum delivers the FOUNDATION; the six d011-0N checks compose on it next. + +scripts/lib-validate.sh (sourced library; harnessed 35/35): + - STANDARD EXIT CONTRACT for every check + the orchestrator: + 0 PASS | 1 FAIL | 2 HOLD | 3 PASS_PENDING_MANUAL | 4 SKIPPED + - emit() one parseable RESULT line (id/word/code/elapsed/msg); vr_begin() timer. + - vr_json(): structured-output capture that NEVER merges stderr into the JSON + stream (the DOCFIX-085 invariant, now library-enforced so no future check can + reintroduce it); stderr goes to a tempfile surfaced via vr_err_tail. + - run(): capture-then-test mutation helper (DOCFIX-082 regime; no pipeline on the + mutating command -> status is the command's, SIGPIPE-safe). + - vr_admin_env / vr_tenant_env: scrub-then-scope OS_* (missing fields -> HOLD, never + a silent half-scope); vr_is_hex32/vr_is_ipv4/vr_need validators; vr_disruptive_ok gate. + - sourced-library guard (refuses direct exec, exit 2), matching lib-net.sh. + +scripts/validate.sh (orchestrator; harnessed 26/26; replaces the PLACEHOLDER): + - PROFILES (data, not code): full-d011 | post-restart | lb-health | isolation; plus + --checks a,b,c explicit list. --list shows profiles + discovered checks. + - Verdict aggregation worst-wins with a custom precedence FAIL>HOLD>MANUAL>PASS/SKIP; + SKIP never lowers the verdict (a lone skipped disruptive check => OVERALL PASS). + - --stop-on-fail (halt after first FAIL/HOLD) vs default continue-and-report-all. + - --include-disruptive pass-through to checks (VR_DISRUPTIVE); missing check -> HOLD; + a check emitting no RESULT line -> exit code trusted, line synthesized. + - Single structured report: per-check RESULT lines + totals + OVERALL exit. + +HARNESS LESSON (recorded): one orchestrator "failure" during the build was a WRONG TEST +ASSERTION (expected a lone skipped check to yield overall exit 4, contradicting the +SKIP-doesn't-lower-verdict contract just specified). Fixed the test, added a stronger +check-level assertion (read the check's own RESULT line). The mock world / test +expectations must match the specified contract -- same lesson as the 409 and stderr mocks. + +REMAINING (checks compose on the foundation; each harnessed, each standalone-runnable): + d011-01-charms (wrap cloud-assert A0), d011-02-vip-jumphost (author: catalog VIP TLS/HTTP), + d011-03-vip-tenant (author: agnhost on a beta node -> keystone VIP, D-035 pattern), + d011-04-octavia-lb (author: 2-backend RR -> member failover -> AMPHORA failover w/ N+1 + headroom HOLD-guard -> recovery -> self-cleanup; --disruptive-gated), d011-05-magnum-e2e + (wrap tenant-acceptance), d011-06-vault-unseal (attest SEC-003 in security-ledger -> MANUAL). + +Next-free: D-071, DOCFIX-086, BUNDLEFIX-009. diff --git a/scripts/lib-validate.sh b/scripts/lib-validate.sh new file mode 100644 index 0000000..ce8f88e --- /dev/null +++ b/scripts/lib-validate.sh @@ -0,0 +1,109 @@ +# scripts/lib-validate.sh +# +# Shared library for the D-011 validation check suite (scripts/checks/*.sh) and +# the orchestrator (scripts/validate.sh). SOURCED, never executed. ASCII + LF. +# +# WHY THIS EXISTS: every check needs the same env hygiene, structured-output +# capture, and result vocabulary. Authoring them once here (and harnessing them +# once) is the difference between "modular and consistent" and "modular and each +# piece re-implements -- and re-breaks -- the same env/parse bugs". +# +# THE STANDARD EXIT CONTRACT (every check script and the orchestrator conform): +# 0 PASS all assertions held +# 1 FAIL a real failure -- the thing under test is broken +# 2 HOLD could not determine (missing scope/precondition/headroom); +# NOT a pass and NOT a failure -- operator must resolve +# 3 PASS_PENDING_MANUAL a manual/attestation item is outstanding but all else green +# 4 SKIPPED deliberately not run (e.g. disruptive path not requested) +# +# Checks emit ONE machine-readable result line via `emit`, then exit with the +# matching code. The orchestrator parses those lines; humans read them too. +# +# shellcheck shell=bash +# shellcheck disable=SC2034 # constants consumed by sourcing scripts + +if [ "${BASH_SOURCE[0]:-}" = "${0}" ]; then + echo "lib-validate.sh is a sourced library; do not run it directly." >&2 + exit 2 +fi + +# --- exit contract constants (use these names, never bare integers) ---------- +readonly VR_PASS=0 VR_FAIL=1 VR_HOLD=2 VR_MANUAL=3 VR_SKIP=4 +# map code -> word (for reporting); index by code. +VR_WORD=( [0]=PASS [1]=FAIL [2]=HOLD [3]=PASS_PENDING_MANUAL [4]=SKIPPED ) + +# --- result emission --------------------------------------------------------- +# emit +# Prints a single parseable line: "RESULT ". +# _VR_T0 (epoch seconds) is set by vr_begin; elapsed is "-" if unset. +emit() { + local id="$1" code="$2"; shift 2 + local word="${VR_WORD[$code]:-UNKNOWN}" el="-" + if [ -n "${_VR_T0:-}" ]; then el="$(( $(date +%s) - _VR_T0 ))s"; fi + printf 'RESULT %s %s %s %s %s\n' "$id" "$word" "$code" "$el" "$*" +} +# vr_begin : start the timer + announce (checks call this first). +vr_begin() { _VR_ID="$1"; _VR_T0="$(date +%s)"; echo "== check $_VR_ID =="; } + +# --- structured-output capture (DOCFIX-085: NEVER merge stderr into JSON) ----- +# vr_json : run cmd, capture STDOUT into , stderr to $_VR_ERR. +# Returns the command's exit status. Caller checks status AND that parses. +_VR_ERR="" +vr_json() { + local __var="$1"; shift + _VR_ERR="$(mktemp)" + local __out __rc + __out="$("$@" 2>"$_VR_ERR")"; __rc=$? + printf -v "$__var" '%s' "$__out" + return $__rc +} +# vr_err_tail: print (indented) the last stderr captured by vr_json, if any. +vr_err_tail() { [ -s "${_VR_ERR:-/dev/null}" ] && sed 's/^/ stderr: /' "$_VR_ERR"; return 0; } + +# --- run(): capture-then-test mutation helper (DOCFIX-082 regime) ------------ +# run : run with stdin closed, capture combined output, print it +# indented, return the command's status. NO pipeline on the mutating command +# (so status is the command's, not a downstream tee/sed -- SIGPIPE-safe). +run() { + local __out __rc + __out="$("$@" &1)"; __rc=$? + [ -n "$__out" ] && printf '%s\n' "$__out" | sed 's/^/ /' + return $__rc +} + +# --- env hygiene: scrub then scope (subshell callers only) -------------------- +vr_scrub_os() { local v; for v in $(env | awk -F= '/^OS_/{print $1}'); do unset "$v"; done; } +# vr_admin_env: clean admin scope from ~/admin-openrc. Returns 2 if absent. +vr_admin_env() { + vr_scrub_os + [ -r "$HOME/admin-openrc" ] || { echo "HOLD: ~/admin-openrc not readable" >&2; return 2; } + # shellcheck source=/dev/null + . "$HOME/admin-openrc"; return 0 +} +# vr_tenant_env : clean tenant password scope from a cred file +# (username=/user_domain_id=/project_id=/password=/auth_url= lines). Returns 2 if unusable. +vr_tenant_env() { + local cf="$1"; vr_scrub_os + [ -s "$cf" ] || { echo "HOLD: tenant cred $cf missing" >&2; return 2; } + local au un ud pid pw + au="$(awk -F= '/^auth_url=/{print $2}' "$cf")"; un="$(awk -F= '/^username=/{print $2}' "$cf")" + ud="$(awk -F= '/^user_domain_id=/{print $2}' "$cf")"; pid="$(awk -F= '/^project_id=/{print $2}' "$cf")" + pw="$(awk -F= '/^password=/{print $2}' "$cf")" + [ -n "$au" ] && [ -n "$un" ] && [ -n "$pid" ] && [ -n "$pw" ] || { echo "HOLD: $cf missing fields" >&2; return 2; } + export OS_AUTH_URL="$au" OS_IDENTITY_API_VERSION=3 OS_USERNAME="$un" \ + OS_USER_DOMAIN_ID="$ud" OS_PROJECT_ID="$pid" OS_PASSWORD="$pw" \ + OS_CACERT="${OS_CACERT:-$HOME/vault-init/vault-ca-root.pem}" + return 0 +} + +# --- small validators (consistent, tested) ----------------------------------- +vr_is_hex32() { [[ "${1:-}" =~ ^[0-9a-f]{32}$ ]]; } +vr_is_ipv4() { [[ "${1:-}" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; } +# vr_need : HOLD (return 2) if any tool is missing on PATH. +vr_need() { local t; for t in "$@"; do command -v "$t" >/dev/null || { echo "HOLD: missing tool: $t" >&2; return 2; }; done; return 0; } + +# --- disruptive gate --------------------------------------------------------- +# vr_disruptive_ok: true only if the check was invoked with --disruptive (or +# VR_DISRUPTIVE=1 passed by the orchestrator's --include-disruptive). Checks call +# this to guard their destructive path and SKIP (return 4) otherwise. +vr_disruptive_ok() { [ "${VR_DISRUPTIVE:-0}" = "1" ]; } diff --git a/scripts/validate.sh b/scripts/validate.sh index ac9d7b2..a31c7b6 100644 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -1,41 +1,124 @@ #!/usr/bin/env bash -# scripts/validate.sh +# scripts/validate.sh -- D-011 validation orchestrator (modular check runner). # -# STATUS: PLACEHOLDER -- drafted post-deploy. +# Runs a PROFILE (named set) or an explicit list of check scripts from +# scripts/checks/, applies the standard exit contract (see lib-validate.sh), +# and prints ONE structured report + a single overall verdict. Each check is +# ALSO runnable standalone; this composes them for a full acceptance run. # -# Roosevelt-rehearsal validation runner per D-011. Executes the validation -# criteria sequentially and produces a structured report. +# Usage: +# validate.sh [--profile NAME | --checks id,id,...] [options] +# Options: +# --profile NAME run a named profile (default: full-d011) +# --checks a,b,c run an explicit comma-list of check ids (overrides profile) +# --include-disruptive permit checks' destructive paths (amphora failover, etc.) +# --stop-on-fail halt after the first FAIL/HOLD (default: continue, report all) +# --list list profiles + discovered checks, then exit 0 +# -h|--help this help +# +# Profiles (data, not code -- extend freely): +# full-d011 the complete amended D-011 bar (1-6; item 4 needs --include-disruptive) +# post-restart quick confidence after a restart (charms, VIP jumphost, VIP tenant) +# lb-health octavia LB pattern only (non-disruptive unless --include-disruptive) +# isolation tenant e2e incl. cross-tenant isolation (item 5) +# +# Exit (overall verdict, worst-wins with MANUAL as a distinct tier): +# 0 PASS every check PASS (or SKIPPED); nothing outstanding +# 1 FAIL at least one check FAIL +# 2 HOLD no FAIL, but at least one HOLD (undetermined) +# 3 PASS_PENDING_MANUAL no FAIL/HOLD, but a manual item outstanding +# ASCII + LF. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lib-validate.sh +. "$HERE/lib-validate.sh" +CHECKDIR="${VR_CHECKDIR:-$HERE/checks}" -set -euo pipefail -shopt -s inherit_errexit 2>/dev/null || true -IFS=$'\n\t' +PROFILE="full-d011"; EXPLICIT=""; STOP=0; LIST=0 +export VR_DISRUPTIVE="${VR_DISRUPTIVE:-0}" +while [ $# -gt 0 ]; do + case "$1" in + --profile) PROFILE="${2:-}"; shift 2 ;; + --checks) EXPLICIT="${2:-}"; shift 2 ;; + --include-disruptive) VR_DISRUPTIVE=1; shift ;; + --stop-on-fail) STOP=1; shift ;; + --list) LIST=1; shift ;; + -h|--help) sed -n '2,40p' "$HERE/$(basename "${BASH_SOURCE[0]}")" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done -FAIL=0 -PASS=0 -SKIP=0 +# --- profiles: name -> ordered check ids ------------------------------------- +profile_checks() { + case "$1" in + full-d011) echo "d011-01-charms d011-02-vip-jumphost d011-03-vip-tenant d011-04-octavia-lb d011-05-magnum-e2e d011-06-vault-unseal" ;; + post-restart) echo "d011-01-charms d011-02-vip-jumphost d011-03-vip-tenant" ;; + lb-health) echo "d011-04-octavia-lb" ;; + isolation) echo "d011-05-magnum-e2e" ;; + *) return 1 ;; + esac +} -result_fail() { echo "FAIL: $*" >&2; FAIL=$((FAIL+1)); } -result_pass() { echo "PASS: $*"; PASS=$((PASS+1)); } -result_skip() { echo "SKIP: $*"; SKIP=$((SKIP+1)); } +if [ "$LIST" = 1 ]; then + echo "profiles:"; for p in full-d011 post-restart lb-health isolation; do printf ' %-14s %s\n' "$p" "$(profile_checks "$p")"; done + echo "discovered checks in $CHECKDIR:" + shopt -s nullglob; for c in "$CHECKDIR"/*.sh; do echo " $(basename "$c" .sh)"; done + exit 0 +fi -# TODO during drafting (aligned 2026-07-03 to D-011 AS AMENDED -- the prior list -# targeted dropped features): -# Building blocks that ALREADY exist (wrap, do not reimplement): -# - scripts/cloud-assert.sh -- charms/behavioral sweep (D-011 items 1-2, 6-8) -# - scripts/tenant-acceptance.sh -- per-tenant kube + LB + isolation e2e -# - scripts/run-tests-all.sh -- tooling gauntlet (gate scripts trusted) -# Still to author here: -# - public API VIP reachability from jumphost AND from a test tenant VM (Option B) -# - Octavia LB pattern test (create -> two members -> round-robin -> failover -> recovery) -# - Magnum CAPI cluster create end-to-end (fresh tenant, timed) -# - Vault MANUAL unseal-after-restart rehearsal by a SECOND person (D-011.6 as -# amended by phase-08 + D-069 -- auto-unseal is NOT a v1 item) -# DROPPED from the original list (do not implement): -# - Designate hostname resolution (DNS dropped for v1 -- D-019) -# - Snapshot 1/2 existence (D-070 superseded D-012: no snapshot restore path) +# resolve the ordered list +if [ -n "$EXPLICIT" ]; then + IFS=',' read -r -a CHECKS <<< "$EXPLICIT" +else + read -r -a CHECKS <<< "$(profile_checks "$PROFILE")" || { echo "unknown profile: $PROFILE (try --list)" >&2; exit 2; } + [ "${#CHECKS[@]}" -gt 0 ] || { echo "unknown profile: $PROFILE (try --list)" >&2; exit 2; } +fi -echo "Placeholder validate.sh -- not yet implemented." +# --- run loop ---------------------------------------------------------------- +declare -i n_pass=0 n_fail=0 n_hold=0 n_manual=0 n_skip=0 +REPORT=() +worst=$VR_PASS +note_worst() { # keep the worst by a custom precedence: FAIL > HOLD > MANUAL > PASS/SKIP + local c="$1" + case "$c" in + "$VR_FAIL") worst=$VR_FAIL ;; + "$VR_HOLD") [ "$worst" = "$VR_FAIL" ] || worst=$VR_HOLD ;; + "$VR_MANUAL") { [ "$worst" = "$VR_FAIL" ] || [ "$worst" = "$VR_HOLD" ]; } || worst=$VR_MANUAL ;; + esac +} + +for id in "${CHECKS[@]}"; do + script="$CHECKDIR/$id.sh" + if [ ! -f "$script" ]; then + line="RESULT $id HOLD 2 - check script not found: $script"; rc=$VR_HOLD + else + out="$(VR_DISRUPTIVE="$VR_DISRUPTIVE" bash "$script" 2>&1)"; rc=$? + # the check's own RESULT line is authoritative for the message; the EXIT CODE + # is authoritative for the verdict. Prefer the emitted line but trust rc. + line="$(printf '%s\n' "$out" | grep -E '^RESULT ' | tail -1)" + [ -n "$line" ] || line="RESULT $id ${VR_WORD[$rc]:-UNKNOWN} $rc - (no RESULT line emitted)" + # print the check's human output above its result line + printf '%s\n' "$out" | grep -vE '^RESULT ' | sed 's/^/ /' + fi + REPORT+=("$line") + case "$rc" in + "$VR_PASS") n_pass+=1 ;; + "$VR_FAIL") n_fail+=1 ;; + "$VR_HOLD") n_hold+=1 ;; + "$VR_MANUAL") n_manual+=1 ;; + "$VR_SKIP") n_skip+=1 ;; + *) n_hold+=1; line="${line/ $rc / $rc(nonstd) }" ;; + esac + note_worst "$rc" + if [ "$STOP" = 1 ] && { [ "$rc" = "$VR_FAIL" ] || [ "$rc" = "$VR_HOLD" ]; }; then + echo " [stop-on-fail: halting after $id]"; break + fi +done echo -echo "Summary: ${PASS} pass, ${FAIL} fail, ${SKIP} skip" -[[ $FAIL -gt 0 ]] && exit 1 || exit 0 +echo "================ VALIDATION REPORT (profile: ${EXPLICIT:+explicit}${EXPLICIT:-$PROFILE}) ================" +printf '%s\n' "${REPORT[@]}" +echo "----------------------------------------------------------------" +printf 'totals: PASS=%d FAIL=%d HOLD=%d MANUAL=%d SKIP=%d\n' "$n_pass" "$n_fail" "$n_hold" "$n_manual" "$n_skip" +echo "OVERALL: ${VR_WORD[$worst]} (exit $worst)" +exit "$worst" diff --git a/tests/lib-validate/run-tests.sh b/tests/lib-validate/run-tests.sh new file mode 100644 index 0000000..a752041 --- /dev/null +++ b/tests/lib-validate/run-tests.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# tests/lib-validate/run-tests.sh -- unit tests for scripts/lib-validate.sh +# Exercises: exit-contract constants, emit() line format + timing, vr_json +# stdout/stderr separation (the DOCFIX-085 invariant), run() capture-then-test, +# env scrub/scope (incl. missing-field HOLD), validators, disruptive gate. +set -u +SD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; LIB="$SD/../../scripts/lib-validate.sh" +P=0; F=0 +ok(){ echo "PASS: $1"; P=$((P+1)); } +no(){ echo "FAIL: $1"; F=$((F+1)); } +chk(){ [ "$2" = "$3" ] && ok "$1" || no "$1 (got '$2' want '$3')"; } + +# guard: sourcing OK, direct-exec refused +( bash "$LIB" >/dev/null 2>&1 ); chk "direct-exec-refused" "$?" 2 + +# shellcheck source=/dev/null +. "$LIB" + +# exit constants +chk "const PASS" "$VR_PASS" 0 +chk "const FAIL" "$VR_FAIL" 1 +chk "const HOLD" "$VR_HOLD" 2 +chk "const MANUAL" "$VR_MANUAL" 3 +chk "const SKIP" "$VR_SKIP" 4 +chk "word MANUAL" "${VR_WORD[3]}" PASS_PENDING_MANUAL + +# emit format: RESULT +L="$(emit d011-XX 1 the message here)" +set -- $L +chk "emit tag" "$1" RESULT +chk "emit id" "$2" d011-XX +chk "emit word" "$3" FAIL +chk "emit code" "$4" 1 +chk "emit elapsed-unset" "$5" "-" +grep -q 'the message here$' <<<"$L" && ok "emit msg-tail" || no "emit msg-tail" +# with timer running, elapsed is Ns +vr_begin d011-YY >/dev/null; EL="$(emit d011-YY 0 x | awk '{print $5}')" +grep -qE '^[0-9]+s$' <<<"$EL" && ok "emit elapsed-timed" || no "emit elapsed-timed ($EL)" + +# vr_json: stdout captured, stderr NOT merged (the DOCFIX-085 invariant) +mkfake(){ printf '#!/usr/bin/env bash\necho "{\\"k\\":1}"\necho "a deprecation warning" >&2\n' > "$1"; chmod +x "$1"; } +FK="$(mktemp)"; mkfake "$FK" +vr_json GOT "$FK"; RC=$? +chk "vr_json rc" "$RC" 0 +chk "vr_json stdout-only" "$GOT" '{"k":1}' +python3 -c "import json,sys; json.loads(sys.argv[1])" "$GOT" && ok "vr_json parses (no stderr contamination)" || no "vr_json parses" +grep -q 'deprecation' <<<"$(vr_err_tail)" && ok "vr_json stderr-captured-separately" || no "vr_json stderr-captured" +rm -f "$FK" + +# vr_json propagates failure status +FK2="$(mktemp)"; printf '#!/usr/bin/env bash\necho oops >&2\nexit 5\n' > "$FK2"; chmod +x "$FK2" +vr_json X "$FK2"; chk "vr_json propagates-rc" "$?" 5; rm -f "$FK2" + +# run(): capture-then-test returns the COMMAND's status, not a pipeline's +run true; chk "run true-rc" "$?" 0 +run false; chk "run false-rc" "$?" 1 +OUT="$(run echo hello 2>&1)"; grep -q ' hello' <<<"$OUT" && ok "run indents-output" || no "run indents-output" + +# env scrub + tenant scope (missing-field HOLD) +export OS_STALE=1 OS_AUTH_URL=stale +CF="$(mktemp)"; printf 'auth_url=https://x:5000/v3\nusername=beta-cluster\nuser_domain_id=%s\nproject_id=%s\npassword=pw\n' "$(python3 -c 'print("b"*32)')" "$(python3 -c 'print("a"*32)')" > "$CF" +( vr_tenant_env "$CF"; RC=$? + [ "$RC" = 0 ] && [ "$OS_USERNAME" = beta-cluster ] && [ -z "${OS_STALE:-}" ] && [ "$OS_PROJECT_ID" = "$(python3 -c 'print("a"*32)')" ] && echo GOOD ) | grep -q GOOD && ok "vr_tenant_env scopes+scrubs" || no "vr_tenant_env scopes+scrubs" +BAD="$(mktemp)"; printf 'auth_url=https://x\nusername=u\n' > "$BAD" # missing project/password +( vr_tenant_env "$BAD" 2>/dev/null ); chk "vr_tenant_env missing-field HOLD" "$?" 2 +( vr_tenant_env /nonexistent 2>/dev/null ); chk "vr_tenant_env missing-file HOLD" "$?" 2 +rm -f "$CF" "$BAD"; unset OS_STALE OS_AUTH_URL + +# vr_admin_env missing file -> HOLD +( HOME="$(mktemp -d)" vr_admin_env 2>/dev/null ); chk "vr_admin_env missing HOLD" "$?" 2 + +# validators +vr_is_hex32 "$(python3 -c 'print("a"*32)')" && ok "hex32 yes" || no "hex32 yes" +vr_is_hex32 "abc" ; chk "hex32 no" "$?" 1 +vr_is_hex32 "$(python3 -c 'print("g"*32)')"; chk "hex32 nonhex" "$?" 1 +vr_is_ipv4 "10.12.4.154" && ok "ipv4 yes" || no "ipv4 yes" +vr_is_ipv4 "10.12.4" ; chk "ipv4 no" "$?" 1 +vr_need bash awk sed && ok "need present" || no "need present" +( vr_need definitely_not_a_tool_xyz 2>/dev/null ); chk "need missing HOLD" "$?" 2 + +# disruptive gate +( VR_DISRUPTIVE=0; vr_disruptive_ok ); chk "disruptive off" "$?" 1 +( VR_DISRUPTIVE=1; vr_disruptive_ok ); chk "disruptive on" "$?" 0 + +echo; [ "$F" = 0 ] && { echo "ALL PASS ($P checks)"; exit 0; } || { echo "FAILURES: $F"; exit 1; } diff --git a/tests/validate/run-tests.sh b/tests/validate/run-tests.sh new file mode 100644 index 0000000..059a117 --- /dev/null +++ b/tests/validate/run-tests.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# tests/validate/run-tests.sh -- orchestrator tests with MOCK checks. +# Verifies: profile resolution, explicit --checks, verdict aggregation across the +# full contract (PASS/FAIL/HOLD/MANUAL/SKIP), worst-wins precedence +# (FAIL>HOLD>MANUAL>PASS), --stop-on-fail, --include-disruptive pass-through, +# missing-check -> HOLD, and a check that emits NO RESULT line (rc trusted). +set -u +SD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; VAL="$SD/../../scripts/validate.sh" +P=0; F=0; ok(){ echo "PASS: $1"; P=$((P+1)); }; no(){ echo "FAIL: $1"; F=$((F+1)); } +chk(){ [ "$2" = "$3" ] && ok "$1" || no "$1 (got '$2' want '$3')"; } + +# build a mock checkdir where each check's exit code is encoded in its name via env +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT; mkdir -p "$W/checks" +mkcheck(){ # mkcheck [emit?] + local id="$1" ex="$2" em="${3:-yes}" + { echo '#!/usr/bin/env bash' + echo ". \"\$(dirname \"\$0\")/../../../scripts/lib-validate.sh\" 2>/dev/null || true" + echo "echo \"(human output for $id)\"" + [ "$em" = yes ] && echo "echo \"RESULT $id WORD $ex - mock message\"" + echo "exit $ex" + } > "$W/checks/$id.sh"; chmod +x "$W/checks/$id.sh" +} +# disruptive-aware mock: SKIP(4) unless VR_DISRUPTIVE=1 then PASS(0) +mkcheck_disr(){ local id="$1" + { echo '#!/usr/bin/env bash' + echo "if [ \"\${VR_DISRUPTIVE:-0}\" = 1 ]; then echo \"RESULT $id WORD 0 - ran disruptive\"; exit 0;" + echo "else echo \"RESULT $id WORD 4 - skipped (disruptive not requested)\"; exit 4; fi" + } > "$W/checks/$id.sh"; chmod +x "$W/checks/$id.sh" +} +run_val(){ VR_CHECKDIR="$W/checks" bash "$VAL" "$@" 2>&1; } + +# 1. all-pass profile -> exit 0 +mkcheck a 0; mkcheck b 0 +OUT="$(run_val --checks a,b)"; chk "all-pass verdict" "$?" 0 +grep -q 'OVERALL: PASS (exit 0)' <<<"$OUT" && ok "all-pass report" || no "all-pass report" + +# 2. one FAIL dominates -> exit 1 +mkcheck c 1; OUT="$(run_val --checks a,c,b)"; chk "fail-dominates" "$?" 1 +grep -q 'FAIL=1' <<<"$OUT" && ok "fail counted" || no "fail counted" + +# 3. HOLD (no fail) -> exit 2 +mkcheck h 2; OUT="$(run_val --checks a,h)"; chk "hold verdict" "$?" 2 + +# 4. MANUAL (no fail/hold) -> exit 3 +mkcheck m 3; OUT="$(run_val --checks a,m)"; chk "manual verdict" "$?" 3 + +# 5. precedence: FAIL beats HOLD beats MANUAL +OUT="$(run_val --checks m,h,c)"; chk "precedence FAIL>HOLD>MANUAL" "$?" 1 +OUT="$(run_val --checks m,h)"; chk "precedence HOLD>MANUAL" "$?" 2 + +# 6. SKIP doesn't lower verdict (all pass/skip -> 0) +mkcheck s 4; OUT="$(run_val --checks a,s)"; chk "skip stays pass" "$?" 0 +grep -q 'SKIP=1' <<<"$OUT" && ok "skip counted" || no "skip counted" + +# 7. --stop-on-fail halts +mkcheck z 0; OUT="$(run_val --stop-on-fail --checks c,z)" +grep -q 'halting after c' <<<"$OUT" && ok "stop-on-fail halts" || no "stop-on-fail halts" +grep -q 'human output for z' <<<"$OUT" && no "stop-on-fail ran z (should not)" || ok "stop-on-fail skipped z" + +# 8. missing check -> HOLD +OUT="$(run_val --checks a,doesnotexist)"; chk "missing-check verdict" "$?" 2 +grep -q 'not found' <<<"$OUT" && ok "missing-check reported" || no "missing-check reported" + +# 9. check emits NO RESULT line -> rc trusted, synthesized line +mkcheck silent 1 no; OUT="$(run_val --checks silent)"; chk "no-result-line rc-trusted" "$?" 1 +grep -q 'no RESULT line emitted' <<<"$OUT" && ok "no-result-line synthesized" || no "no-result-line synthesized" + +# 10. --include-disruptive pass-through +# (SKIP does not lower the OVERALL verdict -- a lone skipped check => OVERALL PASS(0) +# with SKIP=1 in totals; that is the contract. Assert the check's OWN result, not +# just the overall, by reading its RESULT line.) +mkcheck_disr d +OUT="$(run_val --checks d)"; chk "disruptive-default overall" "$?" 0 +grep -qE '^RESULT d WORD 4 ' <<<"$OUT" && ok "disruptive default emitted SKIP(4)" || no "disruptive default emitted SKIP(4)" +grep -q 'SKIP=1' <<<"$OUT" && ok "disruptive default counted as skip" || no "disruptive default counted as skip" +OUT="$(run_val --include-disruptive --checks d)"; chk "disruptive included overall" "$?" 0 +grep -qE '^RESULT d WORD 0 .*ran disruptive' <<<"$OUT" && ok "disruptive included ran" || no "disruptive included ran" + +# 11. profile resolution +mkcheck d011-01-charms 0; mkcheck d011-02-vip-jumphost 0; mkcheck d011-03-vip-tenant 0 +OUT="$(run_val --profile post-restart)"; chk "profile post-restart" "$?" 0 +grep -q 'd011-03-vip-tenant' <<<"$OUT" && ok "profile expanded" || no "profile expanded" + +# 12. unknown profile -> HOLD(2) +OUT="$(run_val --profile nope 2>&1)"; chk "unknown-profile" "$?" 2 + +# 13. --list exits 0 and shows profiles +OUT="$(run_val --list)"; chk "list rc" "$?" 0 +grep -q 'full-d011' <<<"$OUT" && ok "list shows profile" || no "list shows profile" + +echo; [ "$F" = 0 ] && { echo "ALL PASS ($P checks)"; exit 0; } || { echo "FAILURES: $F"; exit 1; }