diff --git a/bundle.yaml b/bundle.yaml index a232960..e5e0668 100644 --- a/bundle.yaml +++ b/bundle.yaml @@ -22,6 +22,13 @@ # addressing, not an HAProxy VIP, so it doesn't add to the VIP-triple count). VIPs # front-loaded into the MAAS-reserved /26: provider 10.12.4.2-.63, metal # 10.12.8.2-.63 (supersedes .224-.254). (B1) +# VIP VALUES: NOT IN THIS FILE. Ruling 3 (2026-07-25) made this bundle VIP-FREE -- +# topology only. Every VIP arrives via a per-DC overlay, dc0 INCLUDED: +# overlays/vr1-dc0-vips.yaml and overlays/vr1-dc1-vips.yaml, both +# GENERATED by scripts/render-dc-overlays.py. This bundle is therefore +# NOT a deployable input on its own -- deploying it without its overlay +# gives 11 charms of decorative HA. The B1 and B5 tokens above remain +# DEFINED here and are referenced from the overlays' per-app comments. # Vault: single unit, MYSQL storage backend (via vault-mysql-router). etcd + easyrsa # REMOVED -- the etcd backend was never used (live storage = mysql) and is moot at # 1 unit; HA backend (Raft vs etcd) is a Roosevelt rehearsal item. (C1; revises D-006) @@ -197,7 +204,6 @@ num_units: 1 # 3 on Roosevelt (D-009) to: [lxd:0] options: - vip: "10.12.4.50 10.12.8.50 10.12.12.50" # B1 front-loaded VIP; IS the catalog endpoint (B5, no os-public-hostname) use-policyd-override: true # as-built reconcile 2026-06-09 (origin untraced -- Review-later) bindings: '': metal-admin @@ -236,7 +242,6 @@ num_units: 1 to: [lxd:0] options: - vip: "10.12.4.53 10.12.8.53 10.12.12.53" # B1 image-conversion: true # as-built; image conversion enabled (raw on Ceph-backed glance) bindings: '': metal-admin @@ -292,7 +297,6 @@ options: console-access-protocol: novnc network-manager: Neutron - vip: "10.12.4.56 10.12.8.56 10.12.12.56" # B1 bindings: '': metal-admin amqp: metal-internal @@ -360,8 +364,6 @@ channel: 2024.1/stable num_units: 1 to: [lxd:0] - options: - vip: "10.12.4.59 10.12.8.59 10.12.12.59" # B1 bindings: '': metal-admin amqp: metal-internal @@ -397,7 +399,6 @@ enable-ml2-port-security: true flat-network-providers: physnet1 neutron-security-groups: true - vip: "10.12.4.55 10.12.8.55 10.12.12.55" # B1 bindings: '': metal-admin amqp: metal-internal @@ -487,7 +488,6 @@ options: block-device: None glance-api-version: 2 - vip: "10.12.4.52 10.12.8.52 10.12.12.52" # B1 bindings: '': metal-admin amqp: metal-internal @@ -573,7 +573,6 @@ to: [lxd:0] options: source: *ceph-source - vip: "10.12.4.60 10.12.8.60 10.12.12.60" # B1 -- radosgw HA un-deferred for Roosevelt fidelity (decorative HA on testcloud) bindings: '': metal-admin certificates: metal-internal @@ -600,7 +599,6 @@ to: [lxd:0] options: debug: "false" - vip: "10.12.4.58 10.12.8.58 10.12.12.58" # B1 -- browse HTTPS by IP (B5); ALLOWED_HOSTS must permit the VIP IP (verify at deploy) bindings: '': metal-admin application-dashboard: metal-internal @@ -654,7 +652,6 @@ # juju deploy ./bundle.yaml \ # --overlay overlays/vr0-dc0-testcloud.yaml \ # --overlay overlays/octavia-pki.yaml - vip: "10.12.4.57 10.12.8.57 10.12.12.57" # B1 bindings: '': metal-admin amqp: metal-internal @@ -719,7 +716,6 @@ to: [lxd:0] options: openstack-origin: *openstack-origin - vip: "10.12.4.51 10.12.8.51 10.12.12.51" # B1 bindings: '': metal-admin amqp: metal-internal @@ -777,7 +773,6 @@ options: openstack-origin: *openstack-origin region: RegionOne - vip: "10.12.4.54 10.12.8.54 10.12.12.54" # B1 bindings: '': metal-admin amqp: metal-internal diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index e6c5ae9..ae3ce56 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1463,6 +1463,47 @@ CHECK 1 is triple-only AND dc0-only and cannot evaluate dc1 at all; `render-baseline` T3 compares counts not sets; and the **B1/B5 comment tokens are defined in `bundle.yaml`'s own header**, so migrating the inline VIP comments out would strand them. +- **RULING 3 EXECUTED 2026-07-29, COMMIT 1 of 2: `bundle.yaml` IS NOW VIP-FREE.** Operator: + "Approved, continue". The 11 inline `vip:` lines are removed and live in + **`overlays/vr1-dc0-vips.yaml`, GENERATED by `scripts/render-dc-overlays.py`** -- the + renderer's first real output. `placement`'s then-empty `options:` block was dropped rather + than left parsing as `null`. **The dual-stack ADD is ruling 3's commit 2 and is deliberately + NOT done here.** Changelog `docs/changelog-20260728-vip-arity-gate.md`. + **PROVEN NEUTRAL by the ruling's own named check:** `provider-bundle-check` on the + pre-extraction bundle and on `bundle.yaml --overlay overlays/vr1-dc0-vips.yaml --dc vr1-dc0` + produce LINE-FOR-LINE IDENTICAL output, same known designate failure included; a structural + diff separately confirmed only `vip` moved (applications 56 -> 56, relations 108 -> 108, every + other key byte-identical). The 11 inline COMMENTS were carried, extracted verbatim by parser + rather than retyped, via new optional per-app comment support in the renderer -- dc1 still + reproduces byte-for-byte at 1634 bytes. The **B1/B5 tokens stay DEFINED in `bundle.yaml`'s + header** and the overlay header names that, so the references do not dangle. + **THE BLAST-RADIUS REPAIR IS THE BULK OF THIS WORK, and every item was found BEFORE the edit + by the read-only agent sweep.** `preflight.sh` P2 validated the BARE base (11 phantom failures + post-move) -> now the MERGED input. `pre-flight-checks.sh` CHECK 1 read the base as raw text + and would have seen ZERO VIPs -> **this is BLOCKER-1, fixed the ruled way (merged source), NOT + by bumping the count**; a measured trap on the way -- the file sets `IFS=$'\n\t'`, so a + space-joined string reached `grep` as ONE filename, `grep` exited 2, `pipefail` propagated and + the gate DIED SILENTLY mid-check (fixed with a bash array). **`phase-01`'s RUN block gates + `juju deploy` on `11/11/0` and would have read `0/0/0` and ABORTED THE DEPLOY** -- both guard + blocks now span base + overlay, and a second latent bug was fixed in the same edit (`grep -c` + over two files prints one count PER FILE, so the bare `$( )` captured a two-line string and + every numeric test would have failed anyway; now awk-summed). Verified end to end: the gate + reads **11/11/0 -> DEPLOY**. `phase-01`'s Octavia SAN derivation raised KeyError swallowed by + `|| true` into an EMPTY VIP -> now reads the overlay, deriving `10.12.4.57`. Also repaired: + `phase-00-teardown:233`; `dc-dc-phase4`'s "DC1 needs NO VIP/CIDR changes at all" and "deploy + the EXISTING bundle.yaml with NO VIP edits" (superseded in place, D-101 plane reasoning kept) + plus its DC2 "edit bundle.yaml's VIP lines" instruction; appendix-A L3's grep-the-bundle + guidance; and `render-baseline/README.md`'s now-unverifiable `bundle.yaml` hash, marked HISTORY + rather than re-pinned. + **Harness 30 -> 31.** The `provider-bundle-check` fixture BASE is now bundle + the dc0 overlay + (that pair is the deploy input); T16 renamed from "pristine repo bundle" to "the real dc0 + deploy input" -- same assertion, honest name. **NEW T31 asserts ruling 3's new invariant + directly: the BARE base fails for every clustered principal**, so "deployed without its + overlay" is a tested state rather than an assumption. Gauntlet **ALL GREEN (85) on vcloud**, + repo-lint 0 fail / 610 files scanned. + **STILL RED, UNCHANGED AND EXPECTED:** CHECK 1 reports `OK=11 want 13` and + `provider-bundle-check` reports designate -- both are R11's `.61`/`.62` being + RULED-BUT-NOT-BUILT, which is commit 2's job. `preflight` remains exit 1 as it already was. - **THREE PLATFORM BEHAVIOURS GRADUATED to `references/platform-traps.md`** at session close, having been recorded only in this status document (which is consolidated over time, so a durable trap does not belong here alone): MAAS auto-reserves `::1`-`::ffff:ffff` on EVERY diff --git a/docs/changelog-20260728-vip-arity-gate.md b/docs/changelog-20260728-vip-arity-gate.md index f225fd0..ea684b1 100644 --- a/docs/changelog-20260728-vip-arity-gate.md +++ b/docs/changelog-20260728-vip-arity-gate.md @@ -294,3 +294,94 @@ Migrating them into an overlay STRANDS them from their definitions -- `appendix-A:652-661` records the repo's rule for exactly this identifier class. This must be handled when the extraction lands, not discovered then. + +--- + +# 2026-07-29 -- RULING 3, COMMIT 1: the neutral VIP MOVE (+ blast-radius repair) + +Operator: **"Approved, continue"**. This is commit 1 of the two ruling 3 (2026-07-25) +prescribes. **The dual-stack ADD is commit 2 and is deliberately NOT in here.** + +## 9. `bundle.yaml` is now VIP-FREE; `overlays/vr1-dc0-vips.yaml` is GENERATED + +**What.** All 11 inline `vip:` lines removed from `bundle.yaml`; `placement`'s +then-empty `options:` block dropped rather than left parsing as `null`. The values +moved to `overlays/vr1-dc0-vips.yaml`, **rendered by `scripts/render-dc-overlays.py`** +from `render/values/vr1-dc0-vips.yaml` -- the renderer's first real output. + +**The 11 inline comments were carried, not dropped.** Ten are the bare token `# B1` +and three carry real operational content (keystone's catalog-endpoint note, +ceph-radosgw's un-defer note, openstack-dashboard's ALLOWED_HOSTS warning). They were +extracted VERBATIM from `bundle.yaml` by a parser, not retyped, and the renderer grew +optional per-app comment support to carry them. An app with no comment emits none, so +the dc1 artifact still reproduces byte-for-byte (re-verified: 1634 bytes, unchanged). + +**The `B1`/`B5` tokens stay DEFINED in `bundle.yaml`'s header**, where they always +were; the overlay header names that explicitly so the references do not dangle +(`appendix-A:652-661` is the repo's rule for this identifier class). + +**PROVEN NEUTRAL by the ruling's own named check.** `provider-bundle-check` on the +pre-extraction bundle and on `bundle.yaml --overlay overlays/vr1-dc0-vips.yaml --dc +vr1-dc0` produce **line-for-line identical output**, including the same known +`hacluster relation but no vip: designate` failure. Separately, a structural diff +confirmed only `vip` changed: applications 56 -> 56, relations 108 -> 108, every +non-VIP option and every non-option key byte-identical. + +**Revert.** `git revert`; the values file and overlay are additive and the bundle edit +is a pure line removal. + +## 10. Blast-radius repair -- the part that made this bigger than "2 commits" + +Every item below would have been BROKEN by the move. All were found by the read-only +agent sweep BEFORE the edit, not after. + +- **`scripts/preflight.sh` P2** validated the BARE base, which post-move reports 11 + phantom "no vip" failures. Now validates the MERGED dc0 input; the overlay is passed + only when present so a tree without it still runs. +- **`scripts/pre-flight-checks.sh` CHECK 1** read `bundle.yaml` as raw text and would + have seen ZERO VIPs -> hard-fail. Now reads the merged source. **This is BLOCKER-1, + fixed the ruled way (point it at the merged bundle) and explicitly NOT by bumping the + expected count.** A measured trap on the way: the file sets `IFS=$'\n\t'`, so a + space-joined `"$VIP_SRC"` reached `grep` as ONE filename, `grep` exited 2, `pipefail` + propagated and the gate **died silently mid-check**. Fixed with a bash ARRAY. +- **`runbooks/phase-01-bundle-deploy.md`** -- the real hazard. Its RUN block gates + `juju deploy` on `TOT/HI/LO == 11/11/0`; post-move that reads `0/0/0` and **aborts + the deploy**. Both guard blocks now span base + overlay. A second bug had to be fixed + in the same edit: `grep -c` over TWO files prints one count PER FILE, so the bare + `$( )` would have captured a two-line string and every numeric test would have failed + anyway -- the counts are now awk-summed. Verified end to end: the gate reads + **11/11/0 -> DEPLOY**. The deploy line itself now carries + `--overlay overlays/vr1-dc0-vips.yaml`. +- **`phase-01`'s Octavia SAN derivation** (DOCFIX-067) read + `bundle['applications']['octavia']['options']['vip']`, which post-move raises + KeyError -- swallowed by `|| true` into an EMPTY VIP, then a "implausible VIP" abort. + Now reads the overlay. Verified: derives `10.12.4.57`. +- **`runbooks/phase-00-teardown-maas-reset.md:233`** bare checker call -> merged input. +- **`runbooks/dc-dc-phase4-juju-bundle-per-dc.md`** said "**DC1 needs NO VIP/CIDR + changes at all**" and "deploy the EXISTING `bundle.yaml` with **NO VIP edits**". + Followed verbatim post-move that deploys a VIP-free DC1. Superseded in place (the + D-101 plane reasoning is kept, because it is still correct); the DC2 instruction to + "edit `bundle.yaml`'s VIP lines" now points at rendering a new overlay instead. +- **`runbooks/appendix-A-troubleshooting.md` L3** told an operator to grep the bundle + for the metal-side VIP token -- which now returns zero, and the block's `|| true` + makes zero look like a clean result. Note added. +- **`tests/render-baseline/README.md`** records a capture-time `bundle.yaml` sha256 + that no longer re-verifies. Marked HISTORY with what changed and why, rather than + re-pinned -- the fixture hashes themselves are untouched. + +## 11. Harness: 30 -> 31, with the fixture BASE re-pointed + +`tests/provider-bundle-check` now builds its fixture base by MERGING +`overlays/vr1-dc0-vips.yaml` into `bundle.yaml`, because that pair is the deploy input. +T16's label changed from "pristine repo bundle" to "the real dc0 deploy input" -- same +assertion, honest name. **NEW T31 asserts ruling 3's new invariant directly: the BARE +base fails for every clustered principal**, so "deployed without its overlay" is a +tested state rather than an assumption. 31/31. + +**Evidence.** Gauntlet **ALL GREEN (85) on vcloud**; repo-lint 0 fail / 610 files; +`render-dc-overlays` 17/17; `render-baseline` 10/10 (dc1 reproduction still byte-exact +after the comment-support change); `preflight` 16/16. + +**Still red, unchanged and expected:** `pre-flight-checks` CHECK 1 reports `OK=11 want +13` and `provider-bundle-check` reports designate -- both are R11's `.61`/`.62` VIPs +being RULED-BUT-NOT-BUILT, which is commit 2's job, not this one's. diff --git a/overlays/vr1-dc0-vips.yaml b/overlays/vr1-dc0-vips.yaml new file mode 100644 index 0000000..7d54361 --- /dev/null +++ b/overlays/vr1-dc0-vips.yaml @@ -0,0 +1,51 @@ +# overlays/vr1-dc0-vips.yaml +# Per-DC VIP overlay for vr1-dc0. RULING 3 of 2026-07-25 made bundle.yaml +# VIP-FREE (topology only) and moved every per-DC value into an overlay, dc0 +# INCLUDED -- so this file now carries what used to sit inline in the base +# bundle, and the base is no longer any DC's VIP source of truth: +# juju deploy ./bundle.yaml --overlay ./overlays/vr1-dc0-vips.yaml +# +# Mapping (MEASURED from scripts/lib-net.sh lib_net_select_dc vr1-dc0, PLANE_NAME): +# provider-public 10.12.4 metal-admin 10.12.8 metal-internal 10.12.12 +# (host octets .50-.60 UNCHANGED by the move -- this commit is a NEUTRAL +# extraction, proven by provider-bundle-check --overlay --dc vr1-dc0.) +# The three VIPs per service are public / admin / internal API endpoints. +# GENERATED by scripts/render-dc-overlays.py from render/values/vr1-dc0-vips.yaml; +# edit the values file and re-render rather than editing this file by hand. +# The B1 / B5 tokens in the per-app comments below are DEFINED in bundle.yaml's +# own header block (B5 at its "Endpoints:" line, B1 at its "HA chain:" line) -- +# they stayed there when the values moved here, so the references still resolve. +applications: + keystone: + options: + vip: "10.12.4.50 10.12.8.50 10.12.12.50" # B1 front-loaded VIP; IS the catalog endpoint (B5, no os-public-hostname) + barbican: + options: + vip: "10.12.4.51 10.12.8.51 10.12.12.51" # B1 + cinder: + options: + vip: "10.12.4.52 10.12.8.52 10.12.12.52" # B1 + glance: + options: + vip: "10.12.4.53 10.12.8.53 10.12.12.53" # B1 + magnum: + options: + vip: "10.12.4.54 10.12.8.54 10.12.12.54" # B1 + neutron-api: + options: + vip: "10.12.4.55 10.12.8.55 10.12.12.55" # B1 + nova-cloud-controller: + options: + vip: "10.12.4.56 10.12.8.56 10.12.12.56" # B1 + octavia: + options: + vip: "10.12.4.57 10.12.8.57 10.12.12.57" # B1 + openstack-dashboard: + options: + vip: "10.12.4.58 10.12.8.58 10.12.12.58" # B1 -- browse HTTPS by IP (B5); ALLOWED_HOSTS must permit the VIP IP (verify at deploy) + placement: + options: + vip: "10.12.4.59 10.12.8.59 10.12.12.59" # B1 + ceph-radosgw: + options: + vip: "10.12.4.60 10.12.8.60 10.12.12.60" # B1 -- radosgw HA un-deferred for Roosevelt fidelity (decorative HA on testcloud) diff --git a/render/values/vr1-dc0-vips.yaml b/render/values/vr1-dc0-vips.yaml new file mode 100644 index 0000000..56cf3cf --- /dev/null +++ b/render/values/vr1-dc0-vips.yaml @@ -0,0 +1,80 @@ +dc: vr1-dc0 +output: overlays/vr1-dc0-vips.yaml +family: v4 +legs: +- provider-public +- metal-admin +- metal-internal +prefixes_v4: + provider-public: 10.12.4 + metal-admin: 10.12.8 + metal-internal: 10.12.12 +apps: +- name: keystone + octet: 50 + comment: '# B1 front-loaded VIP; IS the catalog endpoint (B5, no os-public-hostname)' +- name: barbican + octet: 51 + comment: '# B1' +- name: cinder + octet: 52 + comment: '# B1' +- name: glance + octet: 53 + comment: '# B1' +- name: magnum + octet: 54 + comment: '# B1' +- name: neutron-api + octet: 55 + comment: '# B1' +- name: nova-cloud-controller + octet: 56 + comment: '# B1' +- name: octavia + octet: 57 + comment: '# B1' +- name: openstack-dashboard + octet: 58 + comment: '# B1 -- browse HTTPS by IP (B5); ALLOWED_HOSTS must permit the VIP IP (verify at deploy)' +- name: placement + octet: 59 + comment: '# B1' +- name: ceph-radosgw + octet: 60 + comment: '# B1 -- radosgw HA un-deferred for Roosevelt fidelity (decorative HA on testcloud)' +header: '# overlays/vr1-dc0-vips.yaml + + # Per-DC VIP overlay for vr1-dc0. RULING 3 of 2026-07-25 made bundle.yaml + + # VIP-FREE (topology only) and moved every per-DC value into an overlay, dc0 + + # INCLUDED -- so this file now carries what used to sit inline in the base + + # bundle, and the base is no longer any DC''s VIP source of truth: + + # juju deploy ./bundle.yaml --overlay ./overlays/vr1-dc0-vips.yaml + + # + + # Mapping (MEASURED from scripts/lib-net.sh lib_net_select_dc vr1-dc0, PLANE_NAME): + + # provider-public 10.12.4 metal-admin 10.12.8 metal-internal 10.12.12 + + # (host octets .50-.60 UNCHANGED by the move -- this commit is a NEUTRAL + + # extraction, proven by provider-bundle-check --overlay --dc vr1-dc0.) + + # The three VIPs per service are public / admin / internal API endpoints. + + # GENERATED by scripts/render-dc-overlays.py from render/values/vr1-dc0-vips.yaml; + + # edit the values file and re-render rather than editing this file by hand. + + # The B1 / B5 tokens in the per-app comments below are DEFINED in bundle.yaml''s + + # own header block (B5 at its "Endpoints:" line, B1 at its "HA chain:" line) -- + + # they stayed there when the values moved here, so the references still resolve. + + ' diff --git a/runbooks/appendix-A-troubleshooting.md b/runbooks/appendix-A-troubleshooting.md index 5030c17..c18e800 100644 --- a/runbooks/appendix-A-troubleshooting.md +++ b/runbooks/appendix-A-troubleshooting.md @@ -307,6 +307,11 @@ - The provider-side VIP guard greps only the first token of each dual `vip:`. The metal side (second token, `10.12.8.5x`) must be eyeballed to confirm all 11 sit in .8.50-.60, clear of metal infra (.8.10 maas / .8.20 lxd / .8.21 capi / .8.30 juju). +- **Since ruling 3 (2026-07-25, executed 2026-07-29) the `vip:` lines are NOT in + `bundle.yaml`** -- it is VIP-free and they live in `overlays/vr1-dc0-vips.yaml` / + `overlays/vr1-dc1-vips.yaml`. Grep the overlay, or the base and its overlay together; + grepping the base alone now returns zero and the guard's `|| true` makes that look + like a clean result rather than a missing file. ================================================================================ ## Vault / secrets (phase-02) diff --git a/runbooks/dc-dc-phase4-juju-bundle-per-dc.md b/runbooks/dc-dc-phase4-juju-bundle-per-dc.md index 00afe4f..33feca7 100644 --- a/runbooks/dc-dc-phase4-juju-bundle-per-dc.md +++ b/runbooks/dc-dc-phase4-juju-bundle-per-dc.md @@ -70,11 +70,21 @@ ## What is genuinely IDENTICAL to VR0/DC0 for DC1, and what is not -**DC1 needs NO VIP/CIDR changes at all.** D-101: DC1 inherits DC0's six-plane -v4 layout UNCHANGED. `bundle.yaml`'s existing VIP values (`.50-.60` band, -per phase-01's own text) are ALREADY correct for DC1 -- deploy the SAME -`bundle.yaml` this repo already carries, in DC1's own separate Juju -controller/model, with no v4-address edits. `phase-01..08`'s command blocks +**SUPERSEDED 2026-07-29 by ruling 3 (2026-07-25) -- the paragraph below was +written when `bundle.yaml` carried DC0's VIPs inline. It no longer does: the +bundle is VIP-FREE (topology only) and EVERY VIP arrives via a per-DC overlay, +dc0 included. So the correct statement is now: DC1 needs no VIP *band* change +-- it keeps the same `.50-.60` host octets -- but it absolutely does need its +OWN overlay, `overlays/vr1-dc1-vips.yaml`, because the prefixes differ +(`10.12.64/68/72` vs dc0's `10.12.4/8/12`). Deploying the bare bundle with no +overlay gives 11 charms of decorative HA and NO gate catches it at deploy time +except `provider-bundle-check`. Both overlays are GENERATED by +`scripts/render-dc-overlays.py`; edit `render/values/-vips.yaml` and +re-render rather than hand-editing an overlay.** + +The original text, kept because its D-101 reasoning about the plane layout is +still correct: DC1 inherits DC0's six-plane v4 layout UNCHANGED, and the host +octets are identical across DCs. `phase-01..08`'s command blocks that reference `lib-net.sh`/`lib-hosts.sh` values will resolve to the exact same numbers for DC1 as they did for the original DC0 rehearsal -- this is expected, not a bug (see the `$DC` selector callout above). @@ -200,18 +210,22 @@ Follow `runbooks/phase-01-bundle-deploy.md` verbatim, with these DC-DC- specific notes, not a rewrite: -- **DC1:** deploy the EXISTING `bundle.yaml` with NO VIP edits (see the - "identical to DC0" section above). The Octavia PKI overlay (Step 1.0) +- **DC1:** deploy `bundle.yaml` WITH its per-DC VIP overlay -- + `--overlay overlays/vr1-dc1-vips.yaml` -- and make no hand edit to either + file (ruling 3, 2026-07-25; the overlay is generated, see the superseded + note above). The base bundle carries no VIPs at all now. The Octavia PKI overlay (Step 1.0) can reuse the existing `overlays/octavia-pki.yaml` if its CAs are still valid (10y validity, per that runbook), OR regenerate fresh per-DC PKI via Step 1.0-GEN if independent per-DC Octavia trust domains are desired -- this is a call the operator should make explicitly (log it as a decision note either way), not silently default to reuse. - **DC2:** BLOCKED until its VIP band exists (see above). Once NetBox - assigns DC2's real supernet and it's imported via `netbox/dc-dc-prefixes- - import.py --dc vr1-dc1` (DOCFIX-152), `bundle.yaml`'s VIP lines need editing - to DC2's real `.50-.60`-equivalent band within DC2's own provider-public - plane -- a genuine, reviewed bundle edit, not a copy-paste of DC1's values. + assigns DC2's real supernet, the artifact is a NEW per-DC overlay rendered + from the apex -- `render/values/-vips.yaml` then + `scripts/render-dc-overlays.py render` -- and NOT an edit to `bundle.yaml`, + which no longer carries VIP lines to edit (ruling 3, 2026-07-25). The + substance of the original warning stands: it must be that DC's own measured + band, never a copy-paste of DC1's values. - Both DCs: `bundle.yaml` explicitly ships "NO designate" per phase-01's own gate text (D-019) -- this is UNCHANGED here; Designate reactivation is Stage 7's job (D-106), do not reactivate it mid-Stage-5. @@ -372,7 +386,8 @@ - [ ] `bash scripts/repo-lint.sh` clean (0 fail) before committing any repo changes made while executing this runbook (the IPv6 overlay file, any - `bundle.yaml` VIP edit for DC2, any DC-qualified `asbuilt/` path fix). + a NEW per-DC VIP overlay rendered for DC2 -- NOT a `bundle.yaml` edit, + which carries no VIPs since ruling 3 -- any DC-qualified `asbuilt/` path fix). - [ ] Changelog entry per real repo change this runbook produces (the overlay, any bundle edit, any script fix) -- next DOCFIX number via `bash scripts/ledger-scan.sh`, same as every other delivery this diff --git a/runbooks/phase-00-teardown-maas-reset.md b/runbooks/phase-00-teardown-maas-reset.md index 39a7800..3ab1ba3 100644 --- a/runbooks/phase-00-teardown-maas-reset.md +++ b/runbooks/phase-00-teardown-maas-reset.md @@ -230,7 +230,7 @@ **CHECK (read-only) -- jumphost** -- bundle invariants ```bash -python3 scripts/provider-bundle-check.py bundle.yaml +python3 scripts/provider-bundle-check.py bundle.yaml --overlay overlays/vr1-dc0-vips.yaml --dc vr1-dc0 ``` **Expect:** `PASS` -- 11 charms `public->provider-public`, `.4/.8/.12` VIP triples, all 4 chassis MACs present (incl openstack0), relations well-formed, mysql at 3 units (D-062), diff --git a/runbooks/phase-01-bundle-deploy.md b/runbooks/phase-01-bundle-deploy.md index 8f1ebc5..ee76d66 100644 --- a/runbooks/phase-01-bundle-deploy.md +++ b/runbooks/phase-01-bundle-deploy.md @@ -95,11 +95,15 @@ ```bash ( { echo "=== CHECK 1: bundle VIPs (quote-tolerant, octet-anchored) ===" - grep -nE '^[[:space:]]+vip:' bundle.yaml - TOT=$(grep -cE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.' bundle.yaml || true) - HI=$(grep -cE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(5[0-9]|60)("|$|[[:space:]])' bundle.yaml || true) - LO=$(grep -cE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(1[0-9]|20)("|$|[[:space:]])' bundle.yaml || true) + grep -nE '^[[:space:]]+vip:' bundle.yaml overlays/vr1-dc0-vips.yaml + TOT=$(grep -hcE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.' bundle.yaml overlays/vr1-dc0-vips.yaml 2>/dev/null | awk '{s+=$1} END{print s+0}') + HI=$(grep -hcE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(5[0-9]|60)("|$|[[:space:]])' bundle.yaml overlays/vr1-dc0-vips.yaml 2>/dev/null | awk '{s+=$1} END{print s+0}') + LO=$(grep -hcE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(1[0-9]|20)("|$|[[:space:]])' bundle.yaml overlays/vr1-dc0-vips.yaml 2>/dev/null | awk '{s+=$1} END{print s+0}') echo " provider VIPs total=$TOT in .50-.60=$HI in .10-.20(stale)=$LO (want 11/11/0)" + # RULING 3 (2026-07-25): the counts span bundle.yaml AND its per-DC VIP overlay, + # because the base is now VIP-FREE. `grep -c` over two files prints one count PER + # FILE, so these sum with awk -- a bare $( ) would capture a two-line string and + # every numeric test below would silently fail. # metal side is the second token of each dual vip; eyeball that all 11 are .8.50-.60, # clear of metal infra .8.10(maas)/.8.20(lxd)/.8.21(capi)/.8.30(juju) -- appendix-A: L3. @@ -145,7 +149,8 @@ echo "ABORT: overlays/octavia-pki.yaml missing (Step 1.0)"; else juju add-model openstack - juju deploy ./bundle.yaml --overlay overlays/octavia-pki.yaml -m openstack --dry-run + juju deploy ./bundle.yaml --overlay overlays/vr1-dc0-vips.yaml \ + --overlay overlays/octavia-pki.yaml -m openstack --dry-run fi } ) ``` @@ -167,11 +172,12 @@ **RUN -- jumphost** ```bash ( { - TOT=$(grep -cE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.' bundle.yaml || true) - HI=$(grep -cE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(5[0-9]|60)("|$|[[:space:]])' bundle.yaml || true) - LO=$(grep -cE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(1[0-9]|20)("|$|[[:space:]])' bundle.yaml || true) + TOT=$(grep -hcE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.' bundle.yaml overlays/vr1-dc0-vips.yaml 2>/dev/null | awk '{s+=$1} END{print s+0}') + HI=$(grep -hcE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(5[0-9]|60)("|$|[[:space:]])' bundle.yaml overlays/vr1-dc0-vips.yaml 2>/dev/null | awk '{s+=$1} END{print s+0}') + LO=$(grep -hcE '^[[:space:]]+vip:[[:space:]]*"?10\.12\.4\.(1[0-9]|20)("|$|[[:space:]])' bundle.yaml overlays/vr1-dc0-vips.yaml 2>/dev/null | awk '{s+=$1} END{print s+0}') if [ "$TOT" = 11 ] && [ "$HI" = 11 ] && [ "$LO" = 0 ]; then - juju deploy ./bundle.yaml --overlay overlays/octavia-pki.yaml -m openstack + juju deploy ./bundle.yaml --overlay overlays/vr1-dc0-vips.yaml \ + --overlay overlays/octavia-pki.yaml -m openstack else echo "ABORT: VIP guard failed (total=$TOT hi=$HI lo=$LO; want 11/11/0)" fi @@ -368,7 +374,10 @@ REPO="${REPO:?REPO not set -- export REPO= (see runbooks/README.md Conventions)}" BUNDLE="${BUNDLE:-$REPO/bundle.yaml}" [ -r "$BUNDLE" ] || { echo "FAIL: bundle not readable at $BUNDLE (set BUNDLE=)"; exit 1; } - OCTAVIA_VIP=$(python3 -c "import yaml;print(yaml.safe_load(open('$BUNDLE'))['applications']['octavia']['options']['vip'].split()[0])" 2>/dev/null || true) + # RULING 3 (2026-07-25): octavia's vip lives in the per-DC OVERLAY now, not in + # $BUNDLE -- reading the base alone yields a KeyError that `|| true` would swallow + # into an empty VIP, so this reads the overlay explicitly. + OCTAVIA_VIP=$(python3 -c "import yaml;print(yaml.safe_load(open('overlays/vr1-dc0-vips.yaml'))['applications']['octavia']['options']['vip'].split()[0])" 2>/dev/null || true) echo "measured octavia provider VIP: ${OCTAVIA_VIP:-}" echo "$OCTAVIA_VIP" | grep -qE '^10\.12\.4\.[0-9]{1,3}$' || { echo "FAIL: implausible VIP -- stop"; exit 1; } printf 'IP.1 = %s\n' "$OCTAVIA_VIP" >> controller.cnf diff --git a/scripts/pre-flight-checks.sh b/scripts/pre-flight-checks.sh index 3c3f52b..60b4aa0 100644 --- a/scripts/pre-flight-checks.sh +++ b/scripts/pre-flight-checks.sh @@ -75,7 +75,23 @@ if [ ! -f bundle.yaml ]; then fail "bundle.yaml not found in $REPO" else - VIPLINES="$(grep -cE '^[[:space:]]+vip:' bundle.yaml || true)" + # BLOCKER-1 (ruling 3, 2026-07-25): bundle.yaml is now VIP-FREE -- every VIP + # arrives via a per-DC overlay, dc0 INCLUDED. Reading the base alone would see + # ZERO VIPs and hard-fail, so this reads the MERGED input. The ruled fix is + # exactly this, and explicitly NOT bumping the expected count to paper over it. + # + # KNOWN AND DELIBERATELY NOT FIXED HERE (logged, hard rule 1): this check is + # still dc0-only -- it never calls lib_net_select_dc, and lib-net.sh UNSETS + # VIP_PREFIX_* for dc1 by design, so it cannot evaluate dc1 at all. It is also + # triple-only (n!=3 below), so it will reject the ruled dual-family sextet that + # provider-bundle-check already accepts. Both are pre-existing VR0-frozen + # defects, widened in scope by this ruling but not created by it. + # An ARRAY, not a space-joined string: under `set -euo pipefail` an unsplit + # "a b" reaches grep as ONE filename, grep exits 2, pipefail propagates it and + # the whole gate dies silently mid-check. Measured here, not theorised. + VIP_SRC=(bundle.yaml) + [ -f overlays/vr1-dc0-vips.yaml ] && VIP_SRC+=(overlays/vr1-dc0-vips.yaml) + VIPLINES="$(grep -hcE '^[[:space:]]+vip:' "${VIP_SRC[@]}" 2>/dev/null | awk '{s+=$1} END{print s+0}')" if [ "$VIPLINES" -eq "$VIP_COUNT_EXPECT" ]; then pass "vip: line count = $VIPLINES"; else warn "vip: line count = $VIPLINES (want $VIP_COUNT_EXPECT)"; fi VIPOUT="$(awk -v pp="$VIP_PREFIX_PROVIDER" -v pa="$VIP_PREFIX_ADMIN" -v pi="$VIP_PREFIX_INTERNAL" \ -v lo="$VIP_OCTET_MIN" -v hi="$VIP_OCTET_MAX" ' @@ -92,7 +108,7 @@ ok++ } END{print "RESULT " ok " " bad} - ' bundle.yaml)" + ' "${VIP_SRC[@]}")" echo "$VIPOUT" | grep -v '^RESULT ' || true VOK="$(echo "$VIPOUT" | awk '/^RESULT /{print $2}')" VBAD="$(echo "$VIPOUT" | awk '/^RESULT /{print $3}')" diff --git a/scripts/preflight.sh b/scripts/preflight.sh index 30f48d3..b011811 100644 --- a/scripts/preflight.sh +++ b/scripts/preflight.sh @@ -78,7 +78,14 @@ bash scripts/repo-lint.sh "$REPO"; note $? "P1 repo lint" warn echo "================ P2: bundle invariants ================" -python3 scripts/provider-bundle-check.py bundle.yaml; note $? "P2 bundle invariants" fail +# Ruling 3 (2026-07-25) made bundle.yaml VIP-FREE, so the BARE base is no longer a +# deployable input -- validating it alone would report 11 phantom "hacluster +# relation but no vip" failures for VIPs that simply moved. P2 therefore validates +# the MERGED dc0 deploy input, which is what actually gets deployed. The overlay is +# passed only when present so this stays runnable on a tree without it. +P2_ARGS="bundle.yaml" +[ -f overlays/vr1-dc0-vips.yaml ] && P2_ARGS="$P2_ARGS --overlay overlays/vr1-dc0-vips.yaml --dc vr1-dc0" +python3 scripts/provider-bundle-check.py $P2_ARGS; note $? "P2 bundle invariants" fail echo "================ P3: channel assert (charmhub) ================" python3 scripts/channel_assert.py bundle.yaml; note $? "P3 channel assert" warn diff --git a/scripts/render-dc-overlays.py b/scripts/render-dc-overlays.py index c3a1be5..3f415f0 100755 --- a/scripts/render-dc-overlays.py +++ b/scripts/render-dc-overlays.py @@ -208,7 +208,13 @@ # prefer-ipv6 and the v6 legs must land together, or HAProxy binds # :::port with no v6 VIP for pacemaker to manage. out.append(" prefer-ipv6: true\n") - out.append(' vip: "%s"\n' % " ".join(addrs)) + # An optional trailing comment, carried so the ruling-3 extraction out of + # bundle.yaml is LOSSLESS -- ten of those eleven comments are the bare + # token `# B1`, whose definition stays in bundle.yaml's header block, and + # three carry real operational content. An app with no comment emits none, + # so the dc1 artifact (which has none) still reproduces byte-for-byte. + cmt = (app.get("comment") or "").rstrip() + out.append(' vip: "%s"%s\n' % (" ".join(addrs), (" " + cmt) if cmt else "")) return "".join(out) diff --git a/tests/provider-bundle-check/run-tests.sh b/tests/provider-bundle-check/run-tests.sh index d8914a4..51f0382 100644 --- a/tests/provider-bundle-check/run-tests.sh +++ b/tests/provider-bundle-check/run-tests.sh @@ -13,22 +13,37 @@ TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT PASS=0; FAIL=0 -# fixture base: real bundle + the policies dir the DOCFIX-071 check resolves. +# fixture base: the real dc0 DEPLOY INPUT + the policies dir DOCFIX-071 resolves. # -# `pristine.yaml` is the repo bundle UNTOUCHED. `good.yaml` is that bundle plus the -# ONE thing R11 (D-020 amendment, RULED 2026-07-27) ruled and that is not built yet: -# designate's VIP at the ruled octet .62. Without it the new hacluster-principal -# check fails the repo bundle, and every rc=0 case below would be asserting "fails -# for the reason we already know" instead of its own invariant. T16 keeps the real -# tree's state honest by asserting that pristine.yaml DOES trip it. +# `pristine.yaml` is bundle.yaml MERGED WITH overlays/vr1-dc0-vips.yaml -- since +# ruling 3 (2026-07-25) the base is VIP-free and the merged pair is what actually +# deploys. `good.yaml` is that plus the ONE thing R11 (D-020 amendment, RULED +# 2026-07-27) ruled and that is not built yet: designate's VIP at the ruled octet +# .62. Without it the new hacluster-principal check fails the real input, and every +# rc=0 case below would assert "fails for the reason we already know" instead of its +# own invariant. T16 keeps the real tree honest by asserting the merged input DOES +# trip it; T31 asserts the bare base is no longer a deploy input at all. mkdir -p "$TMP/policies" -cp "$REPO/bundle.yaml" "$TMP/pristine.yaml" cp "$REPO/policies/domain-manager-policy.yaml" "$REPO/policies/overrides.zip" "$TMP/policies/" -python3 - "$TMP/pristine.yaml" "$TMP/good.yaml" <<'PY' +# RULING 3 (2026-07-25) made bundle.yaml VIP-FREE: every VIP arrives via a per-DC +# overlay, dc0 INCLUDED. So the dc0 DEPLOY INPUT -- and therefore the fixture base +# -- is base + overlays/vr1-dc0-vips.yaml. The bare base is no longer a deploy +# input at all, and T31 asserts exactly that rather than leaving it untested. +python3 - "$REPO/bundle.yaml" "$REPO/overlays/vr1-dc0-vips.yaml" \ + "$TMP/pristine.yaml" "$TMP/good.yaml" <<'PY' import sys, yaml -b = yaml.safe_load(open(sys.argv[1])) -b["applications"]["designate"].setdefault("options", {})["vip"] = "10.12.4.62 10.12.8.62 10.12.12.62" -yaml.safe_dump(b, open(sys.argv[2], "w")) +base = yaml.safe_load(open(sys.argv[1])) +ovl = yaml.safe_load(open(sys.argv[2])) +for n, s in (ovl.get("applications") or {}).items(): + base["applications"].setdefault(n, {}).setdefault("options", {}).update( + (s or {}).get("options") or {}) +yaml.safe_dump(base, open(sys.argv[3], "w")) +# `good.yaml` adds the ONE thing R11 ruled and that is not built yet: designate's +# VIP at the ruled octet .62. Without it the new hacluster check fails the real +# input, and every rc=0 case below would assert a failure we already know about. +base["applications"]["designate"].setdefault("options", {})["vip"] = \ + "10.12.4.62 10.12.8.62 10.12.12.62" +yaml.safe_dump(base, open(sys.argv[4], "w")) PY # dc1 twin of that ruled VIP, so the dc1 overlay case (T14) validates the same shape. cat > "$TMP/dc1-designate.yaml" <<'YAML' @@ -163,7 +178,13 @@ # T16 the REAL repo bundle trips it. This is the ruled-but-not-built designate VIP; # when .62 lands in bundle.yaml this case must be re-pointed, NOT deleted. run 1 'hacluster relation but no vip: designate' \ - "T16 pristine repo bundle FAILS on the R11 designate VIP gap" "$TMP/pristine.yaml" + "T16 the real dc0 deploy input FAILS on the R11 designate VIP gap" "$TMP/pristine.yaml" + +# T31 ruling 3's new invariant: the BARE base is not a deploy input. Validating it +# alone must name every clustered principal, not silently pass -- a VIP-free +# bundle deployed without its overlay is 11 charms of decorative HA. +run 1 'hacluster relation but no vip: barbican' \ + "T31 the VIP-free base ALONE fails for all clustered principals" "$REPO/bundle.yaml" # T17 vault is the second R11 gap and it only appears under the HA overlay -- R6 ruled # the VIPs land BEFORE that overlay, so this is the ordering made executable. diff --git a/tests/render-baseline/README.md b/tests/render-baseline/README.md index b2d98ac..10eaaa3 100644 --- a/tests/render-baseline/README.md +++ b/tests/render-baseline/README.md @@ -45,6 +45,13 @@ 7d6261c55d50f36261bda751bd2159a8508c9c3696313fbc37fafabd6728b219 bundle.yaml ``` +**`bundle.yaml`'s hash above is now HISTORY and will not re-verify** -- ruling 3's VIP +extraction (2026-07-29) removed all 11 inline `vip:` lines and dropped `placement`'s +then-empty `options:` block. That is the divergence this section says is by design; the +values themselves moved intact to `overlays/vr1-dc0-vips.yaml` and are unchanged, proven +by `provider-bundle-check` returning a line-for-line identical verdict across the move. +The `vr1-dc1-vips.yaml` hash is still live-current and IS reproduced by the renderer (T6). + ## How the renderer is meant to use these Two-step validation, in this order. The point is that only DATA changes between the steps,