diff --git a/docs/handoff-20260705-open-items.md b/docs/handoff-20260705-open-items.md index 145082a..28513c6 100644 --- a/docs/handoff-20260705-open-items.md +++ b/docs/handoff-20260705-open-items.md @@ -85,8 +85,10 @@ - H1 `tenant-assert.sh` post-onboard verifier, with harness (also offboard preflight + periodic drift sweep). DELIVERED addendum 22; live validation pending (foil onboarding). - H2 stage-3 app-cred idempotency re-run guard in tenant-onboard.sh. + DELIVERED addendum 23 (also covers the keypair; harness 12/12). - H3 canary access-proof stage (boot canary + FIP + SSH via tenant keypair + teardown) -- the ruled meaning of "confirm tenant SSH access into their domain" (A1.4 confirmed). + DELIVERED addendum 23 as stage7 (explicit-only); live validation pending (foil). - H4 `keystone-policy-drift.sh` (script backlog item 6) wired as a periodic check. - H5 no-ad-hoc rule embedded in the onboarding runbook. - Horizon manager-role GUI identity probe (contract section 5.3) validated live. diff --git a/docs/v1-redeploy-changelog.md b/docs/v1-redeploy-changelog.md index d6d60e5..1fc6af1 100644 --- a/docs/v1-redeploy-changelog.md +++ b/docs/v1-redeploy-changelog.md @@ -2121,3 +2121,27 @@ REVERT: git rm -r clientdocs scripts/tenant-assert.sh tests/tenant-assert; git checkout HEAD~ -- docs/handoff-20260705-open-items.md docs/session-ledger.md docs/v1-redeploy-changelog.md Next-free after this push (per scan, keep this line unwrapped): D-074, DOCFIX-093, BUNDLEFIX-012. + +### 2026-07-06 (addendum 23, jumphost stream) -- H2 stage-3 re-run guards + H3 canary stage 7 (tenant-onboard.sh) + +Operator-away autonomous queue work (repo-only; operator pre-approved drafting). +- scripts/tenant-onboard.sh stage3 (H2): app-cred and keypair creates are NOT + idempotent and their secrets are one-shot. New guards: exists remotely + local + secret intact -> SKIP rc0 (idempotent re-run); exists remotely WITHOUT usable + local secret -> STOP rc1 with explicit operator guidance (delete-and-recreate + rotates a live credential / breaks node access -- an operator decision, never + an implicit re-run side effect). +- scripts/tenant-onboard.sh stage7 NEW (H3, contract 5.4, A1.4 ruling): optional + canary access-proof as -cluster: SG + server on -net + FIP + SSH with the + -cluster key, then ALWAYS teardown (trap EXIT), pass or fail. Explicit + invocation only -- never part of 'all'. VERIFY-LIVE: CANARY_SSH_USER (default + ubuntu) vs the public image's login user; CANARY_FLAVOR (default gp.mid) vs + quota. Image resolve uses the stderr-safe capture (stderr to file, surfaced on + parse failure) -- NOT the stage0/stage5 "-f json 2>&1" merge pattern (those two + remain as-is pending the DOCFIX-085-class sweep, logged). +- tests/tenant-onboard/run-tests.sh: extended 2 -> 12 cases (guards: skip/stop + branches both credentials; canary: happy, dead-ssh, teardown-always incl. + teardown.log evidence). ALL PASS; gauntlet green. +REVERT: git checkout HEAD~ -- scripts/tenant-onboard.sh tests/tenant-onboard/run-tests.sh docs/handoff-20260705-open-items.md docs/session-ledger.md docs/v1-redeploy-changelog.md + +Next-free after this push (per scan, keep this line unwrapped): D-074, DOCFIX-093, BUNDLEFIX-012. diff --git a/scripts/tenant-onboard.sh b/scripts/tenant-onboard.sh index 1e7f74f..57464c7 100644 --- a/scripts/tenant-onboard.sh +++ b/scripts/tenant-onboard.sh @@ -25,7 +25,7 @@ OUT="$HOME/tenant-${CLIENT}" # 0600 credential handover dir AUTH_URL="https://${KEYSTONE_VIP}:5000/v3" die(){ echo "FATAL: $*" >&2; exit 1; } -[ -n "$CLIENT" ] || die "usage: tenant-onboard.sh [stage0|1|2|3|4|5|6|all]" +[ -n "$CLIENT" ] || die "usage: tenant-onboard.sh [stage0|1|2|3|4|5|6|7|all] (7 = optional canary access-proof, explicit only)" [ -s "$CA" ] || die "OS_CACERT not found: $CA" openssl x509 -in "$CA" -noout -checkend 0 >/dev/null 2>&1 || die "CA expired/unreadable: $CA" umask 077; mkdir -p "$OUT"; chmod 700 "$OUT" @@ -105,7 +105,23 @@ export OS_USERNAME="${CLIENT}-svc" OS_USER_DOMAIN_ID="$DOM" OS_PROJECT_ID="$PID" export OS_PASSWORD="$(awk -F= '/^password=/{print $2}' "$SF")" [ "$(openstack token issue -f value -c project_id &1)" = "$PID" ] || { echo "svc auth FAIL"; exit 1; } - local ACF="$OUT/${CLIENT}-svc-appcred.txt"; umask 077; : > "$ACF"; chmod 600 "$ACF" + local ACF="$OUT/${CLIENT}-svc-appcred.txt" + # H2 re-run guard (addendum 23): app-cred create is NOT idempotent and the secret is + # unrecoverable after create. If it exists AND the local secret file is intact: SKIP. + # If it exists WITHOUT a usable local secret: STOP -- delete-and-recreate rotates a + # live credential, which is an operator decision, never an implicit re-run side effect. + local EXID; EXID=$(openstack application credential show "${CLIENT}-svc-cred" -f value -c id &1 || true) + if [[ "$EXID" =~ ^[0-9a-f]{32}$ ]]; then + if [ -s "$ACF" ] && grep -qE '^secret=' "$ACF"; then + echo " app cred ${CLIENT}-svc-cred exists + local secret intact -- SKIP (idempotent re-run)" + exit 0 + fi + echo " app cred ${CLIENT}-svc-cred EXISTS but the local secret file is missing/incomplete." + echo " The secret is unrecoverable. OPERATOR DECISION required (rotates a live credential):" + echo " as ${CLIENT}-svc: openstack application credential delete ${CLIENT}-svc-cred ; re-run stage3" + exit 1 + fi + umask 077; : > "$ACF"; chmod 600 "$ACF" openstack application credential create "${CLIENT}-svc-cred" --unrestricted --description "$CLIENT non-trust automation" -f shell "$ACF" 2>&1 grep -qE '^id=' "$ACF" || { echo "appcred FAIL"; cat "$ACF"; exit 1; }; chmod 600 "$ACF" echo " app cred -> $ACF (unrestricted; secret len $(awk -F'"' '/^secret=/{print length($2)}' "$ACF"))" @@ -118,7 +134,21 @@ export OS_USERNAME="${CLIENT}-cluster" OS_USER_DOMAIN_ID="$DOM" OS_PROJECT_ID="$PID" export OS_PASSWORD="$(awk -F= '/^password=/{print $2}' "$CF")" [ "$(openstack token issue -f value -c project_id &1)" = "$PID" ] || { echo "cluster-user auth FAIL"; exit 1; } - local KF="$OUT/${CLIENT}-key.pem"; umask 077; openstack keypair create "${CLIENT}-key" "$KF" 2>&1 + local KF="$OUT/${CLIENT}-key.pem" + # H2 re-run guard: keypair create is NOT idempotent and the private key is only + # returned once. Same skip/stop split as the app cred above. + local EXK; EXK=$(openstack keypair show "${CLIENT}-key" -f value -c name &1 || true) + if [ "$EXK" = "${CLIENT}-key" ]; then + if [ -s "$KF" ] && head -1 "$KF" | grep -q 'PRIVATE KEY'; then + echo " keypair ${CLIENT}-key exists + local private key intact -- SKIP (idempotent re-run)" + exit 0 + fi + echo " keypair ${CLIENT}-key EXISTS but the local private key is missing/invalid." + echo " The private key is unrecoverable. OPERATOR DECISION required (breaks existing node access):" + echo " as ${CLIENT}-cluster: openstack keypair delete ${CLIENT}-key ; re-run stage3" + exit 1 + fi + umask 077; openstack keypair create "${CLIENT}-key" "$KF" 2>&1 head -1 "$KF" | grep -q 'PRIVATE KEY' && { chmod 600 "$KF"; echo " keypair -> $KF (owner: ${CLIENT}-cluster)"; } || { echo "keypair FAIL"; cat "$KF"; exit 1; } ) || die "stage3 (cluster keypair) failed" } @@ -181,6 +211,64 @@ ) || die "stage6 failed" } +stage7(){ # OPTIONAL canary access-proof (H3; tenant-onboarding-contract section 5.4). + # Proves the tenant can boot on their network and reach the instance over SSH with + # the -cluster-owned key (the ruled meaning of "confirm tenant SSH access", A1.4). + # MUTATES tenant-scoped resources (1 SG + 1 server + 1 FIP) and ALWAYS tears them + # down, pass or fail. Explicit invocation only -- never part of 'all'. + # VERIFY-LIVE: CANARY_SSH_USER (default 'ubuntu') must match the public image's + # login user; CANARY_FLAVOR (default gp.mid) must fit remaining quota. + admin_env + local DOM; DOM=$(openstack domain show "$CLIENT" -f value -c id &1) + local CF="$OUT/${CLIENT}-cluster-cred.txt"; [ -s "$CF" ] || die "run stage2 first (no cluster cred)" + local PID; PID=$(awk -F= '/^project_id=/{print $2}' "$CF") + local KF="$OUT/${CLIENT}-key.pem" + [ -s "$KF" ] && head -1 "$KF" | grep -q 'PRIVATE KEY' || die "no local private key $KF (run stage3)" + local IERR="$OUT/.canary-img-err" + local IMG; IMG=$(openstack image list --public -f json "$IERR" | jq -r '[.[]|select(.Name|test("kube"))][0].ID // empty' || true) + [[ "$IMG" =~ ^[0-9a-f-]{36}$ ]] || { sed 's/^/ /' "$IERR" 2>/dev/null; rm -f "$IERR"; die "kube image uuid resolve failed"; } + rm -f "$IERR" + local FLV="${CANARY_FLAVOR:-gp.mid}" SSHU="${CANARY_SSH_USER:-ubuntu}" + echo "== stage7: canary access-proof as ${CLIENT}-cluster (flavor=$FLV sshuser=$SSHU) ==" + ( for v in $(env|awk -F= '/^OS_/{print $1}'); do unset "$v"; done + export OS_AUTH_URL="$AUTH_URL" OS_IDENTITY_API_VERSION=3 OS_CACERT="$CA" + export OS_USERNAME="${CLIENT}-cluster" OS_USER_DOMAIN_ID="$DOM" OS_PROJECT_ID="$PID" + export OS_PASSWORD="$(awk -F= '/^password=/{print $2}' "$CF")" + [ "$(openstack token issue -f value -c project_id &1)" = "$PID" ] || { echo "cluster-user auth FAIL"; exit 1; } + SRV="${CLIENT}-canary"; SG="${CLIENT}-canary-sg"; FIP="" + cleanup(){ + openstack server delete --wait "$SRV" /dev/null 2>&1 || true + [ -n "$FIP" ] && { openstack floating ip delete "$FIP" /dev/null 2>&1 || true; } + openstack security group delete "$SG" /dev/null 2>&1 || true + echo " teardown done (server/FIP/SG removed)" + } + trap cleanup EXIT + openstack security group create "$SG" --description "canary ssh (short-lived)" /dev/null 2>&1 || true + openstack security group rule create "$SG" --protocol tcp --dst-port 22 --remote-ip 0.0.0.0/0 /dev/null 2>&1 || true + openstack server create "$SRV" --image "$IMG" --flavor "$FLV" --network "${CLIENT}-net" \ + --key-name "${CLIENT}-key" --security-group "$SG" /dev/null 2>&1 || { echo " CANARY FAIL: server create"; exit 1; } + ST="" + for _ in $(seq 1 30); do + ST=$(openstack server show "$SRV" -f value -c status &1 || true) + [ "$ST" = "ACTIVE" ] && break; [ "$ST" = "ERROR" ] && break; sleep 10 + done + [ "$ST" = "ACTIVE" ] || { echo " CANARY FAIL: server status '$ST' (want ACTIVE)"; exit 1; } + FIP=$(openstack floating ip create provider-ext -f value -c floating_ip_address &1 || true) + [[ "$FIP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo " CANARY FAIL: FIP allocate: $FIP"; FIP=""; exit 1; } + openstack server add floating ip "$SRV" "$FIP" &1 || { echo " CANARY FAIL: FIP attach"; exit 1; } + OKSSH=0 + for _ in $(seq 1 18); do + if ssh -i "$KF" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o BatchMode=yes "${SSHU}@${FIP}" hostname /dev/null 2>&1; then + OKSSH=1; break + fi + sleep 10 + done + [ "$OKSSH" = "1" ] || { echo " CANARY FAIL: no SSH to ${SSHU}@${FIP} within ~3 min of ACTIVE"; exit 1; } + echo " CANARY PASS: booted on ${CLIENT}-net, SSH OK via $FIP with ${CLIENT}-key" + ) || die "stage7 canary FAILED (teardown was attempted; verify no orphans named ${CLIENT}-canary*)" +} + case "$STAGE" in stage0|0) stage0 ;; stage1|1) stage1 ;; @@ -189,7 +277,8 @@ stage4|4) stage4 ;; stage5|5) stage5 ;; stage6|6) stage6 ;; - all) stage0; stage1; stage2; stage3; stage4; stage5; echo "== stages 0-5 done. run stage6 (cluster) explicitly: tenant-onboard.sh $CLIENT 6 ==" ;; + stage7|7) stage7 ;; + all) stage0; stage1; stage2; stage3; stage4; stage5; echo "== stages 0-5 done. run stage6 (cluster) explicitly: tenant-onboard.sh $CLIENT 6 == (stage7 canary is optional, also explicit)" ;; *) die "unknown stage: $STAGE" ;; esac echo "handover creds in: $OUT (0600)" diff --git a/tests/tenant-onboard/run-tests.sh b/tests/tenant-onboard/run-tests.sh index 36b2b8e..59623bd 100644 --- a/tests/tenant-onboard/run-tests.sh +++ b/tests/tenant-onboard/run-tests.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash -# tests/tenant-onboard/run-tests.sh -- focused offline harness for the two -# stage4 guard behaviors hardened in the 2026-07-03 sweep (H1: the raced -# duplicate-CIDR guard previously FAILED OPEN under pipefail SIGPIPE). -# Fake openstack replays fixtures; fake HOME provides a stub admin-openrc. -# Not a full-stage harness: stage4 is cut short right AFTER the guard by an -# empty app-cred path -- the assertions target guard behavior only. +# tests/tenant-onboard/run-tests.sh -- focused offline harness for guard behaviors: +# - stage4 duplicate-CIDR guard (2026-07-03 sweep; was fail-open under SIGPIPE) +# - stage3 H2 re-run guards (2026-07-06): app-cred/keypair exists + local secret +# intact -> SKIP rc0; exists WITHOUT usable local secret -> STOP rc1 with +# operator guidance (no implicit credential rotation on a re-run) +# - stage7 H3 canary (2026-07-06): happy path CANARY PASS + teardown rc0; +# ssh-dead path CANARY FAIL rc1 AND teardown still runs (trap EXIT) +# Fake openstack/ssh replay fixtures; fake sleep compresses wait loops; fake HOMEs +# provide stub admin-openrc + CA. Not a full-stage harness -- assertions target +# the guard/canary behaviors only. # Exit: 0 all pass | 1 any case failed. ASCII + LF. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -16,41 +20,131 @@ mkdir -p "$H/vault-init" openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -subj /CN=fake-ca \ -keyout "$H/vault-init/ca.key" -out "$H/vault-init/vault-ca-root.pem" -days 2 >/dev/null 2>&1 -cat > "$BIN/openstack" <<'FB' + +DOMID="d0000000000000000000000000000001" +PID="b0000000000000000000000000000001" +ACID="e0000000000000000000000000000001" + +cat > "$BIN/openstack" <&2; exit 1 ;; +case "\$*" in + *"subnet list"*) cat "\$FIXDIR/subnets.txt" ;; + "domain show testco -f value -c id") echo "$DOMID" ;; + "token issue -f value -c project_id") echo "$PID" ;; + "application credential show testco-svc-cred -f value -c id") cat "\$FIXDIR/appcred-id.txt" ;; + "keypair show testco-key -f value -c name") cat "\$FIXDIR/keypair-name.txt" ;; + "image list --public -f json") cat "\$FIXDIR/images.json" ;; + "security group create testco-canary-sg"*) : ;; + "security group rule create testco-canary-sg"*) : ;; + "server create testco-canary"*) : ;; + "server show testco-canary -f value -c status") cat "\$FIXDIR/status.txt" ;; + "floating ip create provider-ext -f value -c floating_ip_address") echo "10.99.0.5" ;; + "server add floating ip testco-canary 10.99.0.5") : ;; + "server delete --wait testco-canary") echo "deleted" >> "\$FIXDIR/teardown.log" ;; + "floating ip delete 10.99.0.5") echo "fip-del" >> "\$FIXDIR/teardown.log" ;; + "security group delete testco-canary-sg") echo "sg-del" >> "\$FIXDIR/teardown.log" ;; + *) echo "fake-openstack: unmatched: \$*" >&2; exit 1 ;; esac FB chmod +x "$BIN/openstack" +cat > "$BIN/ssh" <<'FB' +#!/usr/bin/env bash +exit "$(cat "$FIXDIR/sshrc" 2>/dev/null || echo 1)" +FB +chmod +x "$BIN/ssh" +cat > "$BIN/sleep" <<'FB' +#!/usr/bin/env bash +exit 0 +FB +chmod +x "$BIN/sleep" -run_stage4() { # run_stage4 -> captures output+rc of stage4 under fakes - ( cd "$TMP" && env HOME="$H" FIXDIR="$1" PATH="$BIN:$PATH" \ - TENANT_CIDR="10.20.24.0/24" \ - bash "$REPO/scripts/tenant-onboard.sh" testco stage4 2>&1 ) -} -check() { # check