diff --git a/bundle.yaml b/bundle.yaml index 0bee152..e49be1f 100644 --- a/bundle.yaml +++ b/bundle.yaml @@ -165,8 +165,9 @@ base: ubuntu@22.04/stable charm: rabbitmq-server channel: 3.9/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] + options: {min-cluster-size: 3} # D-009 amendment (2026-07-02): rabbitmq scale-up requires min-cluster-size to avoid split-cluster races bindings: '': metal-admin amqp: metal-internal @@ -184,8 +185,8 @@ # bundle MUST use 1.8/stable. 1.8.8 is EOL (tech debt); getting off it is an OPEN D-068 # problem with 1.16 ruled out. See docs/D-068-vault-1.8-vs-1.16-analysis.md. BUNDLEFIX-010. channel: 1.8/stable - num_units: 1 # 3 on Roosevelt (D-009); HA backend decided there (C1) - to: [lxd:0] + num_units: 3 # 3-unit HA for VR1 (D-121 (v-a)); MySQL backend (C1); Raft-vs-etcd a Roosevelt/D-068 item + to: [lxd:0, lxd:1, lxd:2] bindings: '': metal-admin access: metal-internal @@ -228,8 +229,8 @@ policyd-override: ./policies/overrides.zip charm: keystone channel: 2024.1/stable - num_units: 1 # 3 on Roosevelt (D-009) - to: [lxd:0] + num_units: 3 # 3-unit HA for VR1 (D-121) + to: [lxd:0, lxd:1, lxd:2] options: use-policyd-override: true # as-built reconcile 2026-06-09 (origin untraced -- Review-later) bindings: @@ -268,8 +269,8 @@ base: ubuntu@22.04/stable charm: glance channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: image-conversion: true # as-built; image conversion enabled (raw on Ceph-backed glance) bindings: @@ -324,8 +325,8 @@ base: ubuntu@22.04/stable charm: nova-cloud-controller channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: console-access-protocol: novnc network-manager: Neutron @@ -397,8 +398,8 @@ base: ubuntu@22.04/stable charm: placement channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] bindings: '': metal-admin amqp: metal-internal @@ -430,8 +431,8 @@ base: ubuntu@22.04/stable charm: neutron-api channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: enable-ml2-port-security: true flat-network-providers: physnet1 @@ -551,8 +552,8 @@ base: ubuntu@22.04/stable charm: cinder channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: block-device: None glance-api-version: 2 @@ -642,8 +643,8 @@ base: ubuntu@22.04/stable charm: ceph-radosgw channel: squid/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: source: *ceph-source bindings: @@ -669,8 +670,8 @@ base: ubuntu@22.04/stable charm: openstack-dashboard channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: debug: "false" bindings: @@ -715,8 +716,8 @@ base: ubuntu@22.04/stable charm: octavia channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: debug: false openstack-origin: *openstack-origin @@ -801,8 +802,8 @@ base: ubuntu@22.04/stable charm: barbican channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: openstack-origin: *openstack-origin bindings: @@ -860,8 +861,8 @@ base: ubuntu@22.04/stable charm: magnum channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] options: openstack-origin: *openstack-origin region: RegionOne @@ -920,8 +921,8 @@ base: ubuntu@22.04/stable charm: designate channel: 2024.1/stable - num_units: 1 - to: [lxd:0] + num_units: 3 + to: [lxd:0, lxd:1, lxd:2] bindings: '': metal-admin dns-backend: metal-internal @@ -1008,31 +1009,31 @@ constraints: arch=amd64 # ===================================================================== - # HA Cluster Subordinates (12 active for VR1: 10 v1 API charms + ceph-radosgw + # HA Cluster Subordinates (13 active for VR1: 10 v1 API charms + ceph-radosgw + vault # + designate, reactivated 2026-07-10 per D-106 -- see designate-hacluster below) # ===================================================================== # Channel: 2.4/stable (per Caracal Charm Delivery table, D-002 verified 2026-05-22). - # cluster_count: 1 (decorative on single-unit testcloud, D-009 / BUNDLEFIX-003). + # cluster_count: 3 (real HA for VR1 per D-121; was 1/decorative under D-009/BUNDLEFIX-003, superseded 2026-08-05). # VIPs front-loaded into the MAAS-reserved provider/metal /26 per B1 (.2-.63). - # vault-hacluster stays commented (vault single-unit on mysql, C1 / BUNDLEFIX-002). + # vault-hacluster ACTIVE below (vault 3-unit HA on mysql, D-121 (v-a); BUNDLEFIX-002 de-HA reversed 2026-08-05). # BUNDLEFIX-052 (2026-07-24): removed an orphaned 4-space-indented `bindings:` block here # that YAML take-last had absorbed into ceph-rbd-mirror above, clobbering its real # ceph-local/ceph-remote (replication) + nrpe-external-master bindings (D-108/D-052) and # injecting a phantom `dashboard` binding (a charm with no such endpoint) -> juju deploy # validation reject. Confirmed via yaml.safe_load; committee-20260724-track2-bundle-render. - keystone-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - glance-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - neutron-api-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - nova-cloud-controller-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - placement-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - openstack-dashboard-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - cinder-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - octavia-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - barbican-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - magnum-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} - ceph-radosgw-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} # B1 -- un-deferred - designate-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} # D-106 -- reactivated 2026-07-10, supersedes D-019 deferral - # vault-hacluster: { charm: hacluster, channel: 2.4/stable } # C1: vault single-unit on mysql; HA at Roosevelt + keystone-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + glance-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + neutron-api-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + nova-cloud-controller-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + placement-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + openstack-dashboard-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + cinder-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + octavia-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + barbican-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + magnum-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + ceph-radosgw-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} # B1 -- un-deferred + designate-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} # D-106 -- reactivated 2026-07-10, supersedes D-019 deferral + vault-hacluster: {base: ubuntu@22.04/stable, charm: hacluster, channel: 2.4/stable, options: {cluster_count: 3}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} # D-121 (v-a): vault 3-unit HA on mysql; BUNDLEFIX-002 de-HA reversed 2026-08-05 # memcached: nova-cloud-controller token/cell caching (BUNDLEFIX-004) memcached: @@ -1050,11 +1051,11 @@ relations: - [nova-cloud-controller:memcache, memcached:cache] - # ---- Vault (single unit, MySQL storage backend via vault-mysql-router; C1 -- etcd+easyrsa removed) + # ---- Vault (3-unit HA, MySQL storage backend via vault-mysql-router; D-121 (v-a); C1 -- etcd+easyrsa removed) - [vault-mysql-router:db-router, mysql-innodb-cluster:db-router] - [vault:shared-db, vault-mysql-router:shared-db] - [mysql-innodb-cluster:certificates, vault:certificates] - # - [vault:ha, vault-hacluster:ha] # vault de-HA'd on testcloud (C1/BUNDLEFIX-002); HA backend a Roosevelt item + - [vault:ha, vault-hacluster:ha] # D-121 (v-a): vault 3-unit HA on mysql backend; BUNDLEFIX-002 reversed 2026-08-05 # ---- Keystone (identity, hub of all OS service relations) - [keystone-mysql-router:db-router, mysql-innodb-cluster:db-router] diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4c9f94f..fb73967 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2315,6 +2315,37 @@ "Requesting access to vault (None)" and never completes. NOT deploy-blocking (barbican/0 is `active` on its software backend; this only gates the vault-backed secret store). Distinct follow-up triage, not the ceph class. + **>>> 2026-08-05 -- D-121 HA SCALE-UP IS BEING BUILT ON vr1-dc0 (the "decorative single-unit + control plane -> 3" scale-up). Stage 5 remains OPEN. <<<** Operator ruling 2026-08-05: + **"We have superseded the HA removal from all items ... this run is to stand up all remaining + HA apps ... HA is a blocker for deployment sign off"** + **"testing as if this is Roosevelt."** + This executes the already-ADOPTED D-121 (design-decisions.md:4354) incl. its (v-a) vault + sub-ruling. The barbican-vault `secrets-storage` finding above is RESOLVED BY THIS (vault-hacluster + will host vault's `.61` VIP -> vault publishes `vault_url`). TWO halves: + **(a) BUNDLE-OF-RECORD (BUNDLEFIX-053, committed this session):** `bundle.yaml` reversed from + the D-009/BUNDLEFIX-002/003 single-unit posture to real 3-unit HA -- `num_units 1->3` on 14 HA + apps + `to:[lxd:0,1,2]`; hacluster `cluster_count 1->3` x12; **vault-hacluster + `[vault:ha, + vault-hacluster:ha]` uncommented (BUNDLEFIX-002 reversed, D-121 (v-a))**; rabbitmq + `min-cluster-size:3` (D-009 amendment). Overlays unchanged (VIPs already set). Each change + VERIFIED against charm/juju docs or the deployed charm (hacluster cluster_count default 3; + rabbitmq min-cluster-size; vault `ha` endpoint = metal-internal like keystone; vault MySQL-backend + HA per HashiCorp docs). VALIDATED: `provider-bundle-check.py --dc vr1-dc0` PASS (13 hacluster subs + cluster_count==num_units; 13 principals carry a VIP; 109 relations well-formed); repo-lint 0 fail. + **(b) LIVE SCALE-UP: WAVE 1 COMPLETE (8 apps).** neutron-api, glance, cinder, magnum, designate, + openstack-dashboard, octavia, ceph-radosgw all scaled 1->3 with 3-node hacluster quorum (crm "3 + nodes configured", VIPs Started). designate + octavia remain workload-`blocked` on DEFERRED-by-design + config (Stage-7 DNS / D-021 enablement) -- NOT scale failures. ceph-radosgw reports "Services not + running: radosgw" -- OWED a verify (likely the new-unit stale-status class, HA structure is formed). + Procedure (pilot-proven on placement): per app `add-unit -n 2 --to lxd:1,lxd:2` -> `config + -hacluster cluster_count=3` -> converge, each an individual gated command. FINDINGS: (i) the + apt-cacher-ng proxy hangs ~2 of 6 containers provisioned at once (today's CLOSE-WAIT class) -- + remediated per appendix-A + switched to ONE app / 2 containers at a time (clean thereafter); (ii) + hacluster scale-up briefly STOPS the leader VIP until cluster_count=3 reforms the cluster (~1-2 min) + -- harmless for stateless API apps, sequence keystone/vault deliberately. **REMAINING: Wave 2 + keystone (cloud-wide auth VIP blip) + nova-cloud-controller; Wave 3 rabbitmq; Wave 4 VAULT LAST -- + operator unseals each new unit (SEC-003, guard-hook-blocked for the agent) + cert-SAN check before + the VIP flip.** OWED: D-121 execution runbook (dc1 reuse); post-wave bundle/overlay review vs the + live HA installs (pinned task). Body: `docs/changelog-20260805-d121-ha-scaleup.md`. **NAMED-GATE DEFECT found by measurement -- `phase-03-core-verify.md` Step 3.1 asserts expected non-active/idle = 1 (octavia only); the VR1 roster yields 4 deferred-by-design + gss.** That gate is STALE for VR1 and a DOCFIX is owed (also owed on that runbook: `-m openstack` -> `-m vr1-dc0` diff --git a/docs/changelog-20260805-d121-ha-scaleup.md b/docs/changelog-20260805-d121-ha-scaleup.md new file mode 100644 index 0000000..23ea32f --- /dev/null +++ b/docs/changelog-20260805-d121-ha-scaleup.md @@ -0,0 +1,88 @@ +# Changelog 2026-08-05 (D-121 HA scale-up: bundle + live Wave 1) -- IN PROGRESS + +Session changelog. Branch `dc-dc-stage5-preconditions`. DISCLOSURE (GA-R2/D1): third +same-day changelog in this continuous session, kept separate as a distinct work-stream +from `changelog-20260805-skill-close-convention-docfix.md` and +`changelog-20260805-stage5-dc0-ceph-remediation.md` (both committed). Flagged for the close +bookend to cite all three. `ledger-scan` at this work's open: 4 decisions, 29 SEC, next-free +BUNDLEFIX-053. Status claims live in `docs/CURRENT-STATE.md` ONLY. + +Scope: execute **D-121** ("VR1 makes HA real -- scale the decorative single-unit control +plane to 3") on vr1-dc0, per operator ruling 2026-08-05 ("we have superseded the HA removal +from all items ... stand up all remaining HA apps ... HA is a blocker for deployment sign +off" + "testing as if this is Roosevelt"). Two halves: the running cloud (live add-unit) and +the bundle-of-record (so dc1/Roosevelt reproduce it). + +--- + +## Item 1 -- BUNDLEFIX-053: bundle.yaml + comments updated to the full 3-unit HA posture + +**WHAT.** `bundle.yaml` reversed from the D-009 "decorative single-unit" posture to real +3-unit HA. Applied via an operator-approved one-off script (repetitive lines) + hand-Edits +(unique lines); scripted in-place edit of the tracked bundle is otherwise classifier-walled +(CLAUDE.md caveman guardrail). +- `num_units: 1 -> 3` on the 14 HA apps: keystone, glance, nova-cloud-controller, placement, + neutron-api, cinder, ceph-radosgw, openstack-dashboard, octavia, barbican, magnum, + designate, rabbitmq-server, vault. `to: [lxd:0] -> [lxd:0, lxd:1, lxd:2]` (one-per-control-node). +- `options: {cluster_count: 1 -> 3}` on the 12 hacluster subordinates. +- **vault-hacluster** uncommented/added (cluster_count:3, bindings mirror the others) + + relation `[vault:ha, vault-hacluster:ha]` uncommented -- the **BUNDLEFIX-002 reversal**, + governed by **D-121 (v-a)** (MySQL-backed vault HA, re-add vault:ha, scale to 3). +- `rabbitmq-server` gains `options: {min-cluster-size: 3}` (**D-009 amendment 2026-07-02**: + rabbitmq scale-up is not mechanical without it). +- Governing comments reconciled (D-009/BUNDLEFIX-002/003 single-unit notes -> D-121 built). +- Overlays UNCHANGED (VIPs, incl. vault .61, already set). + +**VERIFICATION (this is the part the operator gated -- "verified against online docs?").** +Each change checked against the charm/juju docs or the deployed charm, not assumed: +- hacluster `cluster_count`: charmhub -- int, default 3, *"best practice to set to the + expected number of units ... with 3+ units quorum is enforced and services stop on quorum + loss."* +- rabbitmq `min-cluster-size`: charmhub -- *"Minimum number of units expected to exist before + charm will attempt to form a rabbitmq cluster."* +- vault `ha`: the DEPLOYED vault charm exposes an `ha` endpoint bound `metal-internal`, + identical to keystone (a known-good hacluster user). +- vault MySQL-backend HA (D-121 (v-a) gate): HashiCorp docs -- *"The MySQL storage backend + supports high availability"* (`ha_enabled` + `lock_table`). Caveat: MySQL lock lost on + connection death -> possible leader flap; mitigate via `interactive_timeout`/`wait_timeout`. +- `num_units:3` / `to:[0,1,2]`: standard juju, proven live this session. + +**VALIDATED.** `provider-bundle-check.py --dc vr1-dc0` **PASS (exit 0)** -- notably +"13 hacluster subordinate(s) declare cluster_count == principal num_units", "13 hacluster +principal(s) all carry a VIP", "109 relations well-formed". `repo-lint` 0 fail. + +**Revert.** `git revert` this commit (bundle change is 47+/46-). Restores the decorative +single-unit posture -- do NOT without re-opening D-121, or the bundle diverges from the +built HA cloud. + +## Item 2 -- D-121 live execution on vr1-dc0 (Wave 1) -- IN PROGRESS + +**Procedure (pilot-proven on `placement`):** per app, `juju add-unit -n 2 --to +lxd:1,lxd:2` -> `juju config -hacluster cluster_count=3` -> converge (3-node crm, VIPs +Started, units active). Each an individual gated command (the classifier enforces +"never batch destructive steps"). + +**Findings (live):** +- **apt-cacher-ng concurrency limit:** 6 containers at once (3-app batch) hung 2 in + `apt-get update` CLOSE-WAIT (today's outage class). Remediated per appendix-A (remove-unit + -> remove-machine --force -> re-add) and switched to **one app / 2 containers at a time** -- + clean thereafter. A *transient* cw>0 during active apt is normal; only a persistent stall + is the hang. +- **transient VIP outage on hacluster scale-up:** adding the 2 new hacluster units while + cluster_count is still 1 briefly STOPS the leader's VIP resource; it recovers once + cluster_count=3 reforms the cluster (~1-2 min). Harmless for stateless API apps; sequence + keystone/vault deliberately. + +**Wave 1 (8 apps) status at close of this entry:** neutron-api, glance, cinder, magnum, +designate, openstack-dashboard, octavia, ceph-radosgw scaled to 3 with 3-node hacluster +(designate/octavia remain workload-`blocked` on their DEFERRED-by-design config -- Stage-7 +DNS / D-021 octavia enablement -- NOT a scale problem). Live status authority: CURRENT-STATE. + +**Revert.** Live-cloud; not a git revert. To undo, remove-unit each app back to 1. + +## Owed (not done this entry) +- **Waves 2-4:** keystone + nova-cloud-controller (auth-path, VIP-blip); rabbitmq + (min-cluster-size handling); **vault LAST -- operator unseals each new unit (SEC-003, + guard-hook-blocked for the agent); cert-SAN check before the VIP flip**. +- **D-121 execution runbook** (hard-rule-4 gap -- needed verbatim for dc1). +- CURRENT-STATE "D-121 (v-a) now BUILT 2026-08-05" note (this commit).