diff --git a/creds-matrix.tsv b/creds-matrix.tsv index 21eb0e7..3eecbba 100644 --- a/creds-matrix.tsv +++ b/creds-matrix.tsv @@ -107,13 +107,13 @@ octavia-controller-ca-key per-DC vr1-dc0 headend controller-ca.key.enc none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:418 SEC-023 n-outside-convention octavia-controller-ca-cert per-DC vr1-dc0 headend controller-ca.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:421 SEC-023 n-outside-convention octavia-controller-key per-DC vr1-dc0 headend controller.key none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:460 SEC-023 n-outside-convention -octavia-controller-bundle per-DC vr1-dc0 headend controller.bundle.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:548 SEC-023 n-outside-convention -octavia-controller-cert per-DC vr1-dc0 headend controller.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:544 SEC-023 n-outside-convention -octavia-controller-ca-serial per-DC vr1-dc0 headend controller-ca.cert.srl none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:542 SEC-023 n-outside-convention +octavia-controller-bundle per-DC vr1-dc0 headend controller.bundle.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:584 SEC-023 n-outside-convention +octavia-controller-cert per-DC vr1-dc0 headend controller.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:580 SEC-023 n-outside-convention +octavia-controller-ca-serial per-DC vr1-dc0 headend controller-ca.cert.srl none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:578 SEC-023 n-outside-convention substrate-tfstate-backup per-DC vr1-dc0 jumphost vr1-dc0-substrate.tfstate.gz none service consolidated stage3 runbook:runbooks/dc-dc-phase2-tofu-dc-substrate.md:783 SEC-023 n-tfstate-backup -octavia-pki-backup per-DC vr1-dc0 jumphost octavia-pki-vr1-dc0.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:658 SEC-023 n-pki-backup -octavia-reissue-backup per-DC vr1-dc0 jumphost vr1-dc0-octavia-pki-.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:817 SEC-023 n-reissue-backup -octavia-pki-overlay per-DC vr1-dc0 headend vr1-dc0-octavia-pki.yaml none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:609 SEC-004 n-overlay-in-clone +octavia-pki-backup per-DC vr1-dc0 jumphost octavia-pki-vr1-dc0.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:694 SEC-023 n-pki-backup +octavia-reissue-backup per-DC vr1-dc0 jumphost vr1-dc0-octavia-pki-.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:853 SEC-023 n-reissue-backup +octavia-pki-overlay per-DC vr1-dc0 headend vr1-dc0-octavia-pki.yaml none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:645 SEC-004 n-overlay-in-clone octavia-issuing-ca-passphrase per-DC vr1-dc1 headend passphrase.txt none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:391 SEC-023 n-outside-convention octavia-issuing-ca-key per-DC vr1-dc1 headend issuing-ca.key.enc none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:395 SEC-023 n-outside-convention octavia-issuing-ca-cert per-DC vr1-dc1 headend issuing-ca.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:398 SEC-023 n-outside-convention @@ -121,13 +121,13 @@ octavia-controller-ca-key per-DC vr1-dc1 headend controller-ca.key.enc none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:418 SEC-023 n-outside-convention octavia-controller-ca-cert per-DC vr1-dc1 headend controller-ca.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:421 SEC-023 n-outside-convention octavia-controller-key per-DC vr1-dc1 headend controller.key none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:460 SEC-023 n-outside-convention -octavia-controller-bundle per-DC vr1-dc1 headend controller.bundle.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:548 SEC-023 n-outside-convention -octavia-controller-cert per-DC vr1-dc1 headend controller.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:544 SEC-023 n-outside-convention -octavia-controller-ca-serial per-DC vr1-dc1 headend controller-ca.cert.srl none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:542 SEC-023 n-outside-convention +octavia-controller-bundle per-DC vr1-dc1 headend controller.bundle.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:584 SEC-023 n-outside-convention +octavia-controller-cert per-DC vr1-dc1 headend controller.cert.pem none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:580 SEC-023 n-outside-convention +octavia-controller-ca-serial per-DC vr1-dc1 headend controller-ca.cert.srl none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:578 SEC-023 n-outside-convention substrate-tfstate-backup per-DC vr1-dc1 jumphost vr1-dc1-substrate.tfstate.gz none service consolidated stage3 runbook:runbooks/dc-dc-phase2-tofu-dc-substrate.md:783 SEC-023 n-tfstate-backup -octavia-pki-backup per-DC vr1-dc1 jumphost octavia-pki-vr1-dc1.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:658 SEC-023 n-pki-backup -octavia-reissue-backup per-DC vr1-dc1 jumphost vr1-dc1-octavia-pki-.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:817 SEC-023 n-reissue-backup -octavia-pki-overlay per-DC vr1-dc1 headend vr1-dc1-octavia-pki.yaml none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:609 SEC-004 n-overlay-in-clone +octavia-pki-backup per-DC vr1-dc1 jumphost octavia-pki-vr1-dc1.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:694 SEC-023 n-pki-backup +octavia-reissue-backup per-DC vr1-dc1 jumphost vr1-dc1-octavia-pki-.tar.gz none service consolidated stage5 runbook:runbooks/phase-01-bundle-deploy.md:853 SEC-023 n-reissue-backup +octavia-pki-overlay per-DC vr1-dc1 headend vr1-dc1-octavia-pki.yaml none service off-manifest-known stage5 runbook:runbooks/phase-01-bundle-deploy.md:645 SEC-004 n-overlay-in-clone tenant-domain-admin per-tenant - jumphost -domain-admin-cred.txt gui human off-manifest-known tenant-onboard script:scripts/tenant-onboard.sh:64 SEC-023 n-tenant-dir tenant-cluster-user per-tenant - jumphost -cluster-cred.txt api human off-manifest-known tenant-onboard script:scripts/tenant-onboard.sh:90 SEC-023 n-tenant-dir tenant-svc-user per-tenant - jumphost -svc-cred.txt api service off-manifest-known tenant-onboard script:scripts/tenant-onboard.sh:90 SEC-023 n-tenant-dir diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4afa146..3a59084 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2562,6 +2562,25 @@ whose only purpose was to TEST it, and then blocking the heredoc writing the correction that documents its own misfires. Both are the established class: the matcher cannot tell reading secret material from merely naming it. + - **F8 AND F9 ARE NOW FIXED AT SOURCE IN 1.0-GEN.c, 2026-07-30 -- this is the part that + protects FUTURE DC standups.** Everything above repaired the two EXISTING DCs; the + GENERATION path still carried both defects, so the next DC standup would have recreated + them. **F9 at source:** GEN.c baked `omega.dc0.vr0` as a literal in THREE places (CN + both + DNS SANs); it now DERIVES `DC_ZONE` from `$DC` with the same two expansions the tool uses, + and echoes it for confirmation before the sign. **F8 at source:** the `cat > controller.cnf + <<'CNF'` heredoc is replaced by one `printf` per line with values passed as `%s` ARGUMENTS. + That rewrite was deferred on 2026-07-29 as "untested ... reasonable when someone can run a + real generation" -- **the condition is now met**: the identical shape was exercised by two + real mints plus 48 harness cases. **Plus structural assertions before signing** (four + sections present, `subjectAltName` wired, CN equals the derived zone, exactly 2 DNS + entries), because `printf` removes the paste hazard but not the failure CLASS. + **F10 handled deliberately:** editing GEN.c shifts every mint-ref anchored below it, so all + 13 octavia anchors were re-resolved BY MARKER in a SINGLE PASS keyed by row id (never + sequential seds) and each verified against its command; 12 rows rewritten, `creds-matrix` + **S4 CLEAN**, and the block `bash -n` syntax-checked as extracted. + Worth carrying forward: R7 recorded this cert's SAN as "already DERIVED per-DC by design", + which was true of the IP SAN and NOT the DNS names -- **a claim accurate about one half of a + field and read as covering both. That is how F9 survived.** - The certs remain INERT by design -- `os-public-hostname` is still set nowhere (R5 refused setting it at Stage 5 as a D-019 repeat). The names are now correct IN ADVANCE of D-106's Stage-7 work arming them. diff --git a/docs/changelog-20260730-octavia-reissue-tool.md b/docs/changelog-20260730-octavia-reissue-tool.md index 676990a..529d436 100644 --- a/docs/changelog-20260730-octavia-reissue-tool.md +++ b/docs/changelog-20260730-octavia-reissue-tool.md @@ -195,3 +195,91 @@ - `octavia-pki.sh` is still called by NO gate: `preflight.sh` checks only that the overlay EXISTS. So "gauntlet ALL GREEN" remains not-evidence about the live PKI. Wiring `verify` into preflight is a gate change with its own blast radius -- proposed, not smuggled in here. + +--- + +# PART 3 (same session) -- custody, register, and F8/F9 fixed AT SOURCE in GEN.c + +Operator: "Update the matrix as needed. Transfer the artifacts to the vcloud. keep the pin +that during the secrets workflow planning that the creds and certs from this step need to be +included. continue on with the rest of the steps." + +## 9. Backup custody executed (Step 1.0-REISSUE.4) + +Both pre-reissue archives pulled from the headend to the jumphost SEC-009 creds folders, +**sha256 compared and identical at both ends** before anything was deleted (a truncated `scp` +would otherwise leave a verified-looking backup of nothing). 0600; 15 entries each including +the deploy overlay and the CA serial. Headend `~/octavia-pki/backups/` removed entirely; no +`.reissue-*` staging residue. + +## 10. The archives are now REGISTERED (they were an unregistered credential class) + +Two `octavia-reissue-backup` rows (per-DC, jumphost, `consolidated`, templated filename +`-octavia-pki-.tar.gz`), the DERIVED manifests updated to match (D-137 ruling 2 -- +manifests are generated, never hand-authored), and a new `n-reissue-backup` note. + +Deliberately a SEPARATE id from `n-pki-backup`: that one is the generation-time workspace +archive, this is per-ROTATION and additionally carries the deploy overlay, which the generation +archive does not. Restoring only the workspace would leave the half that reaches the charm +unrecoverable. + +Measured after: tier 1 = 101 rows, the SAME 5 pre-existing findings; tier 2 = the same 7, with +**no reissue-attributable finding** -- and tier 2 FOUND both archives at their declared +location, which is the point of registering them. + +**Hazard recorded in the note:** these archives contain `controller-ca.cert.srl`, the CA's +ISSUANCE STATE. Restoring one over a workspace that has issued since rolls the serial counter +BACKWARDS and the next mint reuses a serial the estate already holds. Serials burned +2026-07-30: `...250E` (dc0), `...674DE` (dc1). + +**PINNED (operator-directed):** the note carries the standing requirement that the creds and +certs from this step be included in the secrets-storage workflow planning, with the full scope +enumerated -- both CAs (key, passphrase, cert), the controller LEAF key/cert/bundle, the CA +serial state, and the deploy overlay, at BOTH DCs. The jumphost creds folder is INTERIM only. + +## 11. Q2 withdrawn -- the THIRD stale premise in one findings file + +`docs/audit/queued-findings-20260730.txt` Q2 opens "D-137 OPEN FORK 1 -- still unruled". It was +ruled 2026-07-25 at option (b), "Blocking in preflight", whose CONSEQUENCE block explicitly +declines option (c). Appended correction, not a rewrite. The misfire tally is now SEVEN -- the +two newest collected today, when the guard blocked a command whose only purpose was to TEST it, +and then blocked the heredoc writing the correction that documents its own misfires. + +## 12. F8 AND F9 FIXED AT SOURCE IN 1.0-GEN.c -- the part that matters for future standups + +Everything above repairs the two EXISTING DCs. **The generation path still carried both defects, +so the next DC standup would have recreated them.** Both are now fixed where they originate: + +- **F9 at source.** GEN.c baked `omega.dc0.vr0` as a literal in THREE places (CN + both DNS + SANs). It now DERIVES `DC_ZONE` from `$DC` using the same two expansions the tool uses + (`${DC%%-*}` / `${DC#*-}`), so the runbook and the gate cannot disagree, and echoes the zone + for confirmation before the sign. +- **F8 at source.** The `cat > controller.cnf <<'CNF'` heredoc is replaced by one `printf` per + line, values passed as `%s` ARGUMENTS. This was explicitly deferred on 2026-07-29 as "an + untested rewrite ... reasonable when someone can run a real generation" -- that condition is + now met: the identical shape was exercised end to end by two real mints (both DCs) plus 48 + harness cases. +- **Plus structural assertions before signing**, because `printf` removes the paste hazard but + not the failure CLASS: all four sections present, `subjectAltName` wired, CN equal to the + derived zone, exactly 2 DNS entries. A typo'd section name now stops the step instead of + producing a SAN-less certificate behind a wall of OK output. +- The stale "NOT changed -- outside R7's ruled scope" note is superseded in place, keeping the + reasoning: R7 recorded this cert's SAN as "already DERIVED per-DC by design", which was true + of the IP SAN and NOT the DNS names -- **a claim accurate about one half of a field, read as + covering both. That is how F9 survived.** + +**F10 handled deliberately.** Editing GEN.c shifts every mint-ref anchored below it. All 13 +octavia anchors were re-resolved by MARKER in a SINGLE PASS keyed by row id (never sequential +seds -- a line number can be simultaneously an old value for one row and a new value for +another), then each verified to point at its correct command. 12 rows rewritten; `creds-matrix` +S4 reports clean. The runbook block was `bash -n` syntax-checked as extracted. + +**Revert:** `git checkout -- runbooks/phase-01-bundle-deploy.md creds-matrix.tsv creds-matrix-notes.md creds-manifests/vr1-dc0.manifest creds-manifests/vr1-dc1.manifest`. + +## Verification (part 3) + + bash tests/octavia-pki/run-tests.sh -> 48/48 PASS + bash tests/creds-matrix/run-tests.sh -> 65/65 PASS + python3 scripts/creds-matrix.py -> 101 rows, 5 findings (all pre-existing), S4 CLEAN + bash scripts/run-tests-all.sh -> GAUNTLET: ALL GREEN (89) + bash scripts/repo-lint.sh -> 0 fail diff --git a/runbooks/phase-01-bundle-deploy.md b/runbooks/phase-01-bundle-deploy.md index f490f54..5cb1e48 100644 --- a/runbooks/phase-01-bundle-deploy.md +++ b/runbooks/phase-01-bundle-deploy.md @@ -459,32 +459,68 @@ cd "$WORKDIR/controller" || exit 1 # dir from 1.0-GEN.0 openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out controller.key chmod 600 controller.key - cat > controller.cnf <<'CNF' -[req] -distinguished_name = req_distinguished_name -req_extensions = v3_req -prompt = no - -[req_distinguished_name] -CN = octavia-controller.omega.dc0.vr0.cloud.neumatrix.local -O = Neumatrix - -[v3_req] -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = @alt_names - -[alt_names] -DNS.1 = octavia-controller.omega.dc0.vr0.cloud.neumatrix.local -DNS.2 = octavia.omega.dc0.vr0.cloud.neumatrix.local -CNF - # NOTE (logged 2026-07-29, NOT changed -- outside R7's ruled scope): the CN and the two - # DNS SANs above still carry the `dc0.vr0` identity. R7 ruled only the CA SUBJECT and - # the VIP gate, and recorded that the controller cert's SAN is "already DERIVED per-DC - # by design (DOCFIX-067)" -- which is true of the IP SAN, not of these DNS names. They - # are inert while `os-public-hostname` is unset on every API charm (bundle.yaml:11, - # IP-ONLY endpoints per B5), so nothing resolves them today. Revisit with D-106's - # hostname work, not here. + # THE ZONE IS DERIVED FROM $DC, NEVER TYPED (F9 fixed AT SOURCE, 2026-07-30). + # This block used to bake `omega.dc0.vr0` as a literal in THREE places -- the CN and both + # DNS SANs -- so every DC generated from it inherited VR0's region, and vr1-dc1 inherited + # dc0's LABEL as well. That is exactly what had to be repaired on both live DCs by Step + # 1.0-REISSUE. Leaving the literal here would have recreated the defect at the next DC + # standup, which is the whole reason it is derived now. Same two expansions the tool uses + # (`scripts/octavia-pki.sh`), so the runbook and the gate cannot disagree: + DC_REGION="${DC%%-*}" # vr1-dc0 -> vr1 (D-008 ) + DC_DNS_LABEL="${DC#*-}" # vr1-dc0 -> dc0 (D-008 , 0-indexed per D-117) + DC_ZONE="omega.${DC_DNS_LABEL}.${DC_REGION}.cloud.neumatrix.local" + echo "controller cert zone: $DC_ZONE" # CONFIRM before the sign + # ONE printf PER LINE, APPENDING. NO HEREDOC (F8 fixed at source, 2026-07-30). + # The heredoc this replaces had its body and terminator at column 1 inside an indented + # subshell, and pasting it with the indentation preserved gave a terminator that no longer + # matched: the heredoc never closed and `controller.cnf` ended up with no [alt_names] + # section, producing a certificate with NO SANs while every openssl command still printed + # OK. The operator hit that live on 2026-07-29. A printf's correctness does not depend on + # invisible leading whitespace on a line nobody reads. + # Values are %s ARGUMENTS, never inlined, so a value containing a `%` cannot become a + # format directive. This is the shape `octavia-pki.sh reissue` uses, and it has now been + # exercised end to end by two real mints (both DCs, 2026-07-30) plus the harness. + : > controller.cnf + printf '[req]\n' >> controller.cnf + printf 'distinguished_name = req_distinguished_name\n' >> controller.cnf + printf 'req_extensions = v3_req\n' >> controller.cnf + printf 'prompt = no\n' >> controller.cnf + printf '\n' >> controller.cnf + printf '[req_distinguished_name]\n' >> controller.cnf + printf 'CN = octavia-controller.%s\n' "$DC_ZONE" >> controller.cnf + printf 'O = Neumatrix\n' >> controller.cnf + printf '\n' >> controller.cnf + printf '[v3_req]\n' >> controller.cnf + printf 'keyUsage = critical, digitalSignature, keyEncipherment\n' >> controller.cnf + printf 'extendedKeyUsage = clientAuth, serverAuth\n' >> controller.cnf + printf 'subjectAltName = @alt_names\n' >> controller.cnf + printf '\n' >> controller.cnf + printf '[alt_names]\n' >> controller.cnf + printf 'DNS.1 = octavia-controller.%s\n' "$DC_ZONE" >> controller.cnf + printf 'DNS.2 = octavia.%s\n' "$DC_ZONE" >> controller.cnf + # ASSERT THE CONFIG BEFORE SIGNING. printf removes the paste hazard; it does not remove the + # failure CLASS (a typo'd section name still yields a SAN-less cert, and every openssl + # command would still print OK). So the structure is checked, not trusted. + for s in '[req]' '[req_distinguished_name]' '[v3_req]' '[alt_names]'; do + grep -Fqx "$s" controller.cnf || { echo "FAIL: controller.cnf is missing section $s -- stop"; exit 1; } + done + grep -Fqx 'subjectAltName = @alt_names' controller.cnf || { echo "FAIL: subjectAltName not wired to [alt_names] -- stop"; exit 1; } + grep -Fqx "CN = octavia-controller.$DC_ZONE" controller.cnf || { echo "FAIL: CN is not the derived zone -- stop"; exit 1; } + [ "$(grep -c '^DNS\.[0-9]* = ' controller.cnf)" -eq 2 ] || { echo "FAIL: expected exactly 2 DNS entries -- stop"; exit 1; } + # HISTORY OF THE LINES ABOVE (superseded 2026-07-30 -- kept because the reasoning is the + # lesson). This note used to read "logged 2026-07-29, NOT changed -- outside R7's ruled + # scope: the CN and the two DNS SANs above still carry the `dc0.vr0` identity ... Revisit + # with D-106's hostname work, not here." That was a correct scope call at the time and it + # is now DISCHARGED: the names are derived from $DC above, and the two live certificates + # were reissued into their own zones on 2026-07-30 (Step 1.0-REISSUE; capture + # docs/audit/octavia-reissue-executed-20260730.txt). + # WORTH KEEPING: R7 recorded that this cert's SAN is "already DERIVED per-DC by design + # (DOCFIX-067)", which was true of the IP SAN and NOT of the DNS names -- a claim that was + # accurate about one half of a field and was read as covering both. That is how F9 survived. + # STILL TRUE: the DNS names remain INERT while `os-public-hostname` is unset on every API + # charm (bundle.yaml:11, IP-ONLY endpoints per B5), so nothing resolves them today. They + # are simply correct in ADVANCE of D-106's Stage-7 work arming them, rather than wrong and + # waiting to be noticed. # # (The "no IPv6 IP SAN" gap this block used to log was RULED and CLOSED 2026-07-29 -- # operator: "Yes, add the v6 IP sans" -- see the D-109 ruling note. IP.2 is emitted