diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 917a4bb..0666199 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4010,6 +4010,18 @@ pristine `opnsense-26.7-nano.qcow2` in the same pool. That needs `boot -s`, a mutation, so the read-only agent stopped there. Neither edge disk has a backingStore: **no rollback point exists on either edge today.** + **>>> RESOLVED 2026-08-07 -- SEC-031 CLOSED, dc1 EDGE REBUILT, DC1 EGRESS RESTORED. <<<** + Operator-directed to the PROVEN dc0 procedure ("look back at the steps that were successful with + dc0" -- dc0's edge was REBUILT, not single-user-repaired). Path: `tofu -replace` destroy+recreate + the edge onto a fresh `opnsense-26.7-nano.qcow2` (edge-only plan MACHINE-ASSERTED 2/0/2, the 9 + nodes protected) -> D-112(c) console bootstrap (`scripts/opnsense-console-rebuild.py`) -> API mint + -> WAN `172.30.3.2/24` then LAN `10.12.64.1/22` (pf reload produced automatic outbound NAT). + VERIFIED: `dc-egress-check dc1` **pass=8 fail=0**; `.6` region `ping 1.1.1.1` OK + `curl + images.maas.io` 200 (unblocks jammy image sync). A rollback point WAS taken first (disk copy + `...pre-sec031-repair-20260807` + tfstate backup). Procedure is now a first-class RUNBOOK + (`runbooks/dc-edge-rebuild.md`) + site-agnostic TOOL (`scripts/opnsense-console-rebuild.py`) so it + is not reconstructed again -- the exact gap the operator flagged (the dc0 run existed only as an + audit capture). Body: `docs/changelog-20260807-dc1-region-sequence.md` Item 4. **>>> TWO FINDINGS THAT OUTLIVE THIS INCIDENT. <<<** **(1) `dc-egress-check.sh`'s A2 is STRUCTURALLY BLIND TO THIS FAILURE MODE** -- it asserts the edge ANSWERS, and dc1's edge does; the gate therefore passes A2 and fails A3/A4, which cannot distinguish "no NAT" from "upstream diff --git a/docs/audit/dc1-dhcp-handover-20260807.txt b/docs/audit/dc1-dhcp-handover-20260807.txt new file mode 100644 index 0000000..a0ddb34 --- /dev/null +++ b/docs/audit/dc1-dhcp-handover-20260807.txt @@ -0,0 +1,43 @@ +OK profile 'vr1-dc1-region' -> region with racks [vr1-dc1-maas-01] (as expected) +dc-region-topology check: site=vr1-dc1 profile=vr1-dc1-region + [ok] fabric 'vr1-dc1-provider-public' exists + [ok] fabric 'vr1-dc1-metal-internal' exists + [ok] fabric 'vr1-dc1-data-tenant' exists + [ok] fabric 'vr1-dc1-storage' exists + [ok] fabric 'vr1-dc1-replication' exists + [ok] space 'provider-public' exists + [ok] space 'metal-admin' exists + [ok] space 'metal-internal' exists + [ok] space 'data-tenant' exists + [ok] space 'storage' exists + [ok] space 'replication' exists + [ok] subnet 10.12.64.0/22 (provider-public) exists + [ok] subnet 10.12.64.0/22 is on fabric 'vr1-dc1-provider-public' (got 'vr1-dc1-provider-public') + [ok] subnet 10.12.64.0/22 VLAN is bound to space 'provider-public' (got 'provider-public') + [ok] subnet 10.12.64.0/22 gateway '10.12.64.1' (got '10.12.64.1') + [ok] subnet 10.12.68.0/22 (metal-admin) exists + [ok] metal-admin 10.12.68.0/22 is on MAAS's own fabric 'fabric-0' (by design) + [ok] subnet 10.12.68.0/22 VLAN is bound to space 'metal-admin' (got 'metal-admin') + [ok] subnet 10.12.68.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.72.0/22 (metal-internal) exists + [ok] subnet 10.12.72.0/22 is on fabric 'vr1-dc1-metal-internal' (got 'vr1-dc1-metal-internal') + [ok] subnet 10.12.72.0/22 VLAN is bound to space 'metal-internal' (got 'metal-internal') + [ok] subnet 10.12.72.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.76.0/22 (data-tenant) exists + [ok] subnet 10.12.76.0/22 is on fabric 'vr1-dc1-data-tenant' (got 'vr1-dc1-data-tenant') + [ok] subnet 10.12.76.0/22 VLAN is bound to space 'data-tenant' (got 'data-tenant') + [ok] subnet 10.12.76.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.80.0/22 (storage) exists + [ok] subnet 10.12.80.0/22 is on fabric 'vr1-dc1-storage' (got 'vr1-dc1-storage') + [ok] subnet 10.12.80.0/22 VLAN is bound to space 'storage' (got 'storage') + [ok] subnet 10.12.80.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.84.0/22 (replication) exists + [ok] subnet 10.12.84.0/22 is on fabric 'vr1-dc1-replication' (got 'vr1-dc1-replication') + [ok] subnet 10.12.84.0/22 VLAN is bound to space 'replication' (got 'replication') + [ok] subnet 10.12.84.0/22 gateway 'none' (got 'none') + [ok] metal-admin carries node-facing dns_servers (got '10.12.68.3') -- D-131 + [ok] metal-admin allow_dns is False (got 'False') -- D-131 + [ok] metal-admin has a DHCP dynamic range (got 1) + [ok] tag 'openstack-vr1-dc1' exists + [ok] no stranded interface links (every link is on its subnet's VLAN) +dc-region-topology: 40 passed, 0 failed diff --git a/docs/audit/dc1-region-topology-20260807.txt b/docs/audit/dc1-region-topology-20260807.txt new file mode 100644 index 0000000..574e924 --- /dev/null +++ b/docs/audit/dc1-region-topology-20260807.txt @@ -0,0 +1,43 @@ +OK profile 'vr1-dc1-region' -> region with racks [vr1-dc1-maas-01] (as expected) +dc-region-topology check: site=vr1-dc1 profile=vr1-dc1-region + [ok] fabric 'vr1-dc1-provider-public' exists + [ok] fabric 'vr1-dc1-metal-internal' exists + [ok] fabric 'vr1-dc1-data-tenant' exists + [ok] fabric 'vr1-dc1-storage' exists + [ok] fabric 'vr1-dc1-replication' exists + [ok] space 'provider-public' exists + [ok] space 'metal-admin' exists + [ok] space 'metal-internal' exists + [ok] space 'data-tenant' exists + [ok] space 'storage' exists + [ok] space 'replication' exists + [ok] subnet 10.12.64.0/22 (provider-public) exists + [ok] subnet 10.12.64.0/22 is on fabric 'vr1-dc1-provider-public' (got 'vr1-dc1-provider-public') + [ok] subnet 10.12.64.0/22 VLAN is bound to space 'provider-public' (got 'provider-public') + [ok] subnet 10.12.64.0/22 gateway '10.12.64.1' (got '10.12.64.1') + [ok] subnet 10.12.68.0/22 (metal-admin) exists + [ok] metal-admin 10.12.68.0/22 is on MAAS's own fabric 'fabric-0' (by design) + [ok] subnet 10.12.68.0/22 VLAN is bound to space 'metal-admin' (got 'metal-admin') + [ok] subnet 10.12.68.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.72.0/22 (metal-internal) exists + [ok] subnet 10.12.72.0/22 is on fabric 'vr1-dc1-metal-internal' (got 'vr1-dc1-metal-internal') + [ok] subnet 10.12.72.0/22 VLAN is bound to space 'metal-internal' (got 'metal-internal') + [ok] subnet 10.12.72.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.76.0/22 (data-tenant) exists + [ok] subnet 10.12.76.0/22 is on fabric 'vr1-dc1-data-tenant' (got 'vr1-dc1-data-tenant') + [ok] subnet 10.12.76.0/22 VLAN is bound to space 'data-tenant' (got 'data-tenant') + [ok] subnet 10.12.76.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.80.0/22 (storage) exists + [ok] subnet 10.12.80.0/22 is on fabric 'vr1-dc1-storage' (got 'vr1-dc1-storage') + [ok] subnet 10.12.80.0/22 VLAN is bound to space 'storage' (got 'storage') + [ok] subnet 10.12.80.0/22 gateway 'none' (got 'none') + [ok] subnet 10.12.84.0/22 (replication) exists + [ok] subnet 10.12.84.0/22 is on fabric 'vr1-dc1-replication' (got 'vr1-dc1-replication') + [ok] subnet 10.12.84.0/22 VLAN is bound to space 'replication' (got 'replication') + [ok] subnet 10.12.84.0/22 gateway 'none' (got 'none') + [FAIL] metal-admin carries node-facing dns_servers (got 'none') -- D-131 + [FAIL] metal-admin allow_dns is False (got 'True') -- D-131 + [FAIL] metal-admin has a DHCP dynamic range (got 0) + [ok] tag 'openstack-vr1-dc1' exists + [ok] no stranded interface links (every link is on its subnet's VLAN) +dc-region-topology: 37 passed, 3 failed diff --git a/docs/changelog-20260807-dc1-region-sequence.md b/docs/changelog-20260807-dc1-region-sequence.md index 97f1c55..b4d8efb 100644 --- a/docs/changelog-20260807-dc1-region-sequence.md +++ b/docs/changelog-20260807-dc1-region-sequence.md @@ -54,3 +54,117 @@ dns_names), e.g. `DELETE /api/ipam/ip-addresses//`. The importer itself and its harness are removable with `git rm netbox/dc-util-hosts-import.py tests/dc-util-hosts-import/` + `bash scripts/run-tests-all.sh --record-manifest`. + +--- + +## Item 2 -- Stage-5/dc1 region STANDUP: topology + IPAM + DHCP cutover (live, gated) + +All against `--profile vr1-dc1-region --expect-rack vr1-dc1-maas-01`, run from voffice1; +each mutation profile-asserted and read back. Operator-approved ("I approve you running +these commands"; "Full cutover first"). + +**B -- topology apply** (`dc-region-topology.sh apply vr1-dc1 --commit`). Built the 5 named +plane fabrics, 6 spaces, 4 missing plane subnets (10.12.72/76/80/84.0/22), space bindings, +gateways, and the `openstack-vr1-dc1` placement tag. Verified `check` = 37/40 (the 3 +remaining = DHCP/DNS, deliberately deferred to the handover). Capture: +`docs/audit/dc1-region-topology-20260807.txt`. (The apply's own run exceeded the client's +120s window during output; the writes all landed -- the check is the evidence, no half-write.) + +**C-IPAM** (`dc-plane-ipam.sh`, `MAAS_PROFILE=vr1-dc1-region`). `carve-v6 --commit` = 6 v6 +plane subnets on their v4 twins' VLANs, created + read back (0 errors). `reserve --commit` += 13 v4 reservations (6 utility + 6 VIP bands + FIP pool `10.12.65.0-10.12.67.254`). +Read-only `check` = 24/0 PASS. (Note: reserve reported `skipped=6` on v6 bands though +carve-v6 had landed; the 24/0 check confirms the ruled state is fully present -- glance +owed, non-blocking.) + +**C-DHCP/DNS cutover** (manual per the dc0 2026-07-30 precedent; `dc-region-topology` +deliberately excludes DHCP and no handover TOOL exists -- a Roosevelt-transferable gap, +logged not built mid-cutover). Measured exact values (nothing inferred): OLD = +Office1 `admin` subnet 11, vlan fabric 142 vid 0, primary nmpcq4; NEW = `vr1-dc1-region` +subnet 3, vlan fabric 0 vid 0, rack qtw8pm. As-built config replicated verbatim +(dns_servers `10.12.68.3`, allow_dns false, dynamic `10.12.68.201-.254` -- MEASURED, my +recollection of `.100-.200` was WRONG). Sequence: OFF admin (dhcp_on=false) -> region is +sole server -> set subnet-3 dns/range -> ON region (dhcp_on=true primary=qtw8pm). +Verified: topology check 40/0; `.6` runs the sole metal-admin dhcpd (`ss :67` on enp1s0); +vvr1-dc1 has NO `:67` on the node segment. FALSE ALARM resolved: a bare `pgrep -c dhcpd` +self-matched its own command shell and read 1 -- `ss :67`/`pgrep -af` are authoritative. +Capture: `docs/audit/dc1-dhcp-handover-20260807.txt`. + +**Revert.** Region-side: `maas vr1-dc1-region vlan update 0 0 dhcp_on=false`; re-enable +admin: `maas admin vlan update 142 0 dhcp_on=true`. Topology/IPAM are additive (idempotent +re-apply / MAAS delete by object). No repo change in this item -- all live MAAS state. + +--- + +## Item 3 -- Stage-5/dc1: power-key installed; image sync BLOCKED on SEC-031 (edge repair) + +**C-powerkey DONE** (`maas-region-power-key.sh install vr1-dc1 qemu+ssh://jessea123@10.12.68.2/system +--commit`, run ON the .6 via office1_svc, key piped on stdin from vcloud -- never argv/intermediate, +shredded from the .6's mktemp). Verify 9/9 incl. the artifact test: REAL virsh (snap) connect to the +rack succeeds + enumerates 13 domains. SEC-016 per-DC isolation intact (dc1's own key, never dc0's). +FRAGILITY (SEC-016): re-assert after any MAAS snap refresh (`/var/snap/maas/current/`). + +**C-images BLOCKED -- root-caused to SEC-031, not a new problem.** The region has no jammy resources +(boot-source `images.maas.io`, only noble selected). The .6 cannot reach images.maas.io: routing is +correct (default via edge 10.12.64.1, edge gw pingable, DNS resolves) but `ping 1.1.1.1` FAILS. Measured +that the RACK is identical (direct egress also fails; its :8000 proxy only works via an Office1-proxy +chain). PRIOR ART (CURRENT-STATE ~3979, SEC-031, capture dc1-edge-assessment-20260802.txt) already +diagnosed this: the 2026-08-01 in-place tofu resize bounced the containment VM = hard power cut; fsck on +the dc1 OPNsense edge destroyed the FreeBSD base-system user DB (`/etc/master.passwd`, `/etc/group`), +so `Configuring firewall.....failed` every boot -> NO pf ruleset -> the edge forwards but does NOT +translate (tcpdump: packet leaves WAN with source still 10.12.64.2, untranslated). config.xml is INTACT; +verdict REPAIR-not-rebuild. + +**REFRAME for the egress decision.** dc0 (measured) reaches images.maas.io directly via its WORKING +edge NAT (http_proxy=null, jammy present). So the operator's choice "fix .6 egress to match dc0" = +**repair the dc1 edge (SEC-031)**: regenerate the user DB from config.xml via the proven D-112(c) +`local_user_set()` path (or donate from the pristine `opnsense-26.7-nano.qcow2` in the same pool) -> +firewall applies -> NAT restored. This also closes the SEC-031 security exposure (edge currently an +open router). CAVEAT: the edge has NO working credential path (sshd down, authenticated API hangs, +console login likely fails on the unparseable passwd DB). HELD for operator direction before touching +the live edge -- not bundled into image sync. + +**Revert.** No mutation taken for images (diagnosis only). Power-key revert: remove +`/var/snap/maas/current/root/.ssh/id_dc1_power` + its `config` Host stanza on the .6. + +--- + +## Item 4 -- SEC-031 CLOSED: dc1 OPNsense edge REBUILT, DC egress restored (proven dc0 procedure) + +**What.** Rebuilt the dc1 edge via the proven dc0-2026-08-02 procedure (operator-directed: +"look back at the steps that were successful with dc0" -- dc0's edge was REBUILT, not +repaired). Result: `dc-egress-check dc1` **pass=8 fail=0**; the `.6` region reaches the +internet (`ping 1.1.1.1` OK, `curl images.maas.io` 200), which unblocks jammy image sync. + +**Steps (now the runbook `runbooks/dc-edge-rebuild.md`):** +1. Rollback prep -- edge `virsh shutdown`, disk copied `...pre-sec031-repair-20260807`; + inner tfstate backed up (sha256-identical); TARGETED `tofu plan -replace` (disk+domain) + MACHINE-ASSERTED edge-only (2 non-no-op, both `vr1_dc1_opnsense`, 0 others -- the 9 nodes + protected); base image confirmed present on voffice1. +2. `tofu apply` -> 2 added / 2 destroyed (edge only), fresh pristine disk, factory boot. +3. Console bootstrap -- generalised `scripts/opnsense-console-rebuild.py --domain vr1-dc1-opnsense + --pubkey ` on the rack: `CONFIG-WRITTEN`, `AK-IN-CONFIG=152`, + `USER-MATERIALIZED`, sshd `OK`. +4. Cleared rack known_hosts (rebuilt edge = new host keys), interim reach leg on virbr4, + proved key-only SSH (`OPNsense 26.7`, `15.1-RELEASE-p1`). +5. Minted API key (`opnsense-bootstrap-apikey.sh`); set WAN `172.30.3.2/24` gw `172.30.3.1` + (pf reload produced automatic outbound NAT here); proved edge egress (ping 1.1.1.1, + fetch jammy Release) BEFORE the LAN move; set LAN `10.12.64.1/22` (session drop expected), + `configctl interface reconfigure lan`, NAT confirmed `from (vtnet0:network) to any`. +6. `dc-egress-check dc1` 8/8. Cleanup: shredded the transient rack edge key (SEC-015 pattern), + removed the reach leg, consolidated the API key to `~/vr1-dc1-creds/` (0600, sha256-verified). + +**Why a runbook + tool now (operator instruction).** The dc0 rebuild lived ONLY as an audit +capture (`docs/audit/dc0-edge-rebuild-20260802.txt`), so dc1 had to reconstruct the whole +procedure by reading 683 lines + adapting a one-off driver. Fixed: `runbooks/dc-edge-rebuild.md` +(site-parameterised procedure) + `scripts/opnsense-console-rebuild.py` (site-agnostic driver, +replaces the per-DC `d112c-console--rebuild.py` copies). SEC-031 CLOSED in the ledger. + +**Lessons baked in:** rebuilt edge invalidates rack known_hosts; the LAN move drops its own +session (config saves+applies first); edge root shell is tcsh (no `2>&1`); assert tofu scope on +ACTIONS not the drift note; `opnsense-set-interface-v4.sh` uses the legacy interfaces page by +design (D-113 amdt). + +**How to revert.** Rebuild is forward-only; the pre-rebuild disk copy +(`...pre-sec031-repair-20260807`) + tfstate backup on the rack/voffice1 are the rollback point. +Repo revert: `git rm scripts/opnsense-console-rebuild.py runbooks/dc-edge-rebuild.md` + reopen SEC-031. diff --git a/docs/security-ledger.md b/docs/security-ledger.md index e9bcce4..87e2446 100644 --- a/docs/security-ledger.md +++ b/docs/security-ledger.md @@ -81,6 +81,6 @@ | SEC-028 | 2026-07-30 | **Per-DC JUJU SERVICE credential, and the RACK becomes a credential-bearing host for the first time.** D-132 q1 gives each DC its own MAAS region with its own database, so the Office1 `juju-vr1-dc0` user and its API key do NOT authenticate against dc0's region -- a fact that reads as a network fault at bootstrap rather than as an auth failure. dc0's `juju-vr1-dc0` was therefore MINTED IN THE NEW REGION 2026-07-30 (superuser, mirroring the measured Office1 shape, which MAAS 3.7 requires for machine allocate/deploy) together with its 3-part API key and account password. Minted ON the region VM, never printed, moved host-to-host with all three sha256 digests compared, staging shredded. Consolidated to `~/vr1-dc0-creds/` on the jumphost (0600) and distributed to the dc0 RACK (0600 in a 0700 folder) because **D-138 moved the cloud-facing Juju client INTO the DC** -- so `rack` is now a credential-bearing host-role, which it had never been. dc1's is NOT YET MINTED (its region VM is authored, not applied). | 2026-07-30 Stage-5 Juju re-point; D-132 q1 + D-138; captures in `docs/audit/dc0-node-carve-20260730.txt` and this session's changelog | operator ("Mint juju-vr1-dc0 on the new region (Recommended)") | **OPEN -- rotation obligation + three notes.** (1) ROTATE at v1 close, or immediately if the rack or region VM is rebuilt or shared. (2) **BLAST RADIUS IS DC-LOCAL BY CONSTRUCTION and this is strictly BETTER than the alternative considered:** the rejected option was pointing juju at the region ADMIN key already consolidated on vcloud, which would have placed an admin-scoped credential on the rack -- the residency SEC-026 exists to constrain. This key administers dc0's region ONLY; no cross-DC reuse, per SEC-012/-016/-018/-019. (3) **`vm-secret-locations` gained its FIRST `rack` rows** (four, both DCs) -- definition-of-done for a new mint site, since an unlisted location is not audited (how SEC-022 happened). It also now covers the rack's `~/.local/share/juju/*` credential store, where juju keeps its own runtime copy. (4) dc1's four matrix rows are the FORWARD REGISTER and FAIL S2 by design until dc1's region exists -- that failure is D-137 working, and the rows must NOT be deleted to go green. **These two NEW S2 findings are NOT covered by the operator's 2026-07-30 P5 acceptance, which enumerated six and said so explicitly.** | | SEC-029 | 2026-07-30 | **Octavia PKI overlay resident on the DC RACK.** The overlay carries the Octavia controller CERT and its PRIVATE KEY, and `juju deploy` consumes it at deploy time. D-138 moved the cloud-facing juju client INTO the DC, and the deploy CANNOT run from `voffice1` at all -- there is no L3 path from the headend to the controller API on the provider plane. dc0's overlay was therefore copied to the dc0 rack 2026-07-30 (`~/repo-stage/overlays/vr1-dc0-octavia-pki.yaml`, `0600`, sha256 compared end to end and equal). dc1's is NOT yet copied. | 2026-07-30 Stage-5 bundle deploy; D-138; this session's changelog | operator ("Copy dc0's PKI overlay to the dc0 rack (Recommended)") | **OPEN -- residency, not a new key.** (1) This is a COPY of material that already exists on the headend (SEC-004 `n-overlay-in-clone`), not a new mint, so the rotation obligation remains the existing `octavia-pki.sh reissue` path -- **and that path must now update BOTH copies, or the rack will deploy a superseded cert.** (2) Per-DC isolation holds BY CONSTRUCTION: each rack receives ONLY its own DC's overlay, never the pair; a cross-DC copy would destroy the independence that `octavia-pki.sh verify`'s A11 assertion exists to prove. (3) `~/repo-stage/` on the rack is a STAGING directory created because the rack has NO REPO CLONE -- the same D-138 tooling gap as queued finding F1. If a clone is later placed on DC client hosts, this path moves and the `vm-secret-locations` row must move with it. | | SEC-030 | 2026-07-31 | **Local permission rules bypass the committed ask-gating -- ACCEPTED, KNOWN posture, recorded so it is explicit rather than implicit.** `.claude/settings.json` (committed team policy) deliberately puts `juju deploy`, `maas admin machine delete` and `maas admin * update/create/release` in **ask**, and the controller-destruction command in **deny**. `.claude/settings.local.json` (GITIGNORED, 286 allow rules accumulated across sessions, largely from interactive don't-ask-again approvals) contains `Bash(ssh *)`, `Bash(ssh voffice1 *)`, `Bash(ssh vr1-dc0-rack *)`, `Bash(ssh vr1-dc0-maas *)` and `Bash(python3 *)`. Because an `ssh ''` invocation matches on the OUTER ssh command, a destructive `maas admin machine delete` issued over ssh never reaches the ask rule; `python3 *` permits arbitrary execution outright. **CLAUDE.md hard rule 3 states the ask rules enforce per-mutation approval and are not to be worked around, so this is a real divergence between stated and enforced posture.** | 2026-07-31 permission carry-over review; continuation of `queued-findings-20260730-dc0-region-migration` F1 | operator ("Leave them as they are") | **OPEN -- ACCEPTED BY RULING, NOT AN OVERSIGHT.** (1) The operator was shown the five rules verbatim and the gating they bypass, and ruled to keep them: the mutation gate in practice is the operator's conversational approval of each presented command, not the rule engine. That has held all session. (2) **The compensating control is the PRESENTATION discipline** -- every mutation is still stated, justified as minimal, and approved before running. If that discipline lapses, nothing else catches it. (3) **NOT promoted to committed team policy** (ruling: "Promote a curated safe subset"): 36 read-only rules were promoted to `settings.json`; these five stay local-only so the bypass is NOT baked into team policy or carried to Roosevelt. (4) **`Read(//home/jessea123/**)` was deliberately EXCLUDED from promotion** -- it grants read of `~/vr1-*-creds/`, `~/vault-init/`, `~/as-executed/` and `~/tenant-*`, every one of which CLAUDE.md forbids reading into context. Promoting it would have put a secrets-read permission into team policy. (5) Four local wildcards were NARROWED before promotion because they permit mutation: `ip route *`, `ip neigh *`, `bridge fdb *`, `tc qdisc *` all allow add/del on the host running every DC node; only the `show`/`get` verbs were promoted. (6) A rebuild still loses the ~250 unpromoted local rules -- that half of F1 remains open by choice. | -| SEC-031 | 2026-08-02 | **The vr1-dc1 OPNsense edge is currently an OPEN ROUTER serving its management GUI to the simulated ISP.** The 2026-08-01 hard power cut (in-place tofu resize bouncing the containment VM) destroyed the edge's FreeBSD base-system user database -- `/etc/master.passwd` (`pwd_mkdb: line #1 too long`) and `/etc/group` (`pw: group 'wheel' does not exist`) -- so `Configuring firewall.....failed.` on every boot since and NO pf ruleset is in force. MEASURED against a control, which is what makes this a finding rather than a worry: from vcloud, the healthy office1 edge (same 26.7 image, same build path) answers WAN ICMP with 100% loss and its GUI with `000`; the dc1 edge answers WAN ICMP at **0% loss** and its GUI **200 in 0.014s**. It also forwards without translating -- simultaneous tcpdump on both taps shows the same packet leaving the WAN leg with source still `10.12.64.2`. Bounded by lab topology: the "ISP" is the `vr1-dc1-uplink` libvirt NAT net on vcloud, not the internet, and D-125 egress isolation is unaffected. Discovered by the read-only dc1 edge assessment, `docs/audit/dc1-edge-assessment-20260802.txt`. | docs/audit/dc1-edge-assessment-20260802.txt | operator | OPEN -- closes when the edge's pf ruleset is restored (the repair regenerates the user DB from config.xml via the proven D-112(c) `local_user_set()` path, or donates it from the pristine `opnsense-26.7-nano.qcow2` in the same pool). Config.xml itself is INTACT -- fsck names every inode it deletes and `/conf/config.xml` is not among them. NOT rebuild-blocking; the exposure ends with the repair. | +| SEC-031 | 2026-08-02 | **The vr1-dc1 OPNsense edge is currently an OPEN ROUTER serving its management GUI to the simulated ISP.** The 2026-08-01 hard power cut (in-place tofu resize bouncing the containment VM) destroyed the edge's FreeBSD base-system user database -- `/etc/master.passwd` (`pwd_mkdb: line #1 too long`) and `/etc/group` (`pw: group 'wheel' does not exist`) -- so `Configuring firewall.....failed.` on every boot since and NO pf ruleset is in force. MEASURED against a control, which is what makes this a finding rather than a worry: from vcloud, the healthy office1 edge (same 26.7 image, same build path) answers WAN ICMP with 100% loss and its GUI with `000`; the dc1 edge answers WAN ICMP at **0% loss** and its GUI **200 in 0.014s**. It also forwards without translating -- simultaneous tcpdump on both taps shows the same packet leaving the WAN leg with source still `10.12.64.2`. Bounded by lab topology: the "ISP" is the `vr1-dc1-uplink` libvirt NAT net on vcloud, not the internet, and D-125 egress isolation is unaffected. Discovered by the read-only dc1 edge assessment, `docs/audit/dc1-edge-assessment-20260802.txt`. | docs/audit/dc1-edge-assessment-20260802.txt | operator | **CLOSED 2026-08-07 -- edge REBUILT, pf ruleset restored, exposure ended.** The stated closure condition (pf ruleset restored) is measurably met. Path taken was REBUILD not in-place repair (operator-directed to the proven dc0 procedure): tofu destroy+recreate the edge onto a fresh `opnsense-26.7-nano.qcow2` (edge-only plan, machine-asserted 2/0/2), D-112(c) console bootstrap (`scripts/opnsense-console-rebuild.py`), API mint, WAN-then-LAN addressing -> pf reload produced automatic outbound NAT. VERIFIED: `dc-egress-check dc1` **pass=8 fail=0**; the edge no longer serves its GUI on the WAN and the `.6` region reaches the internet (`ping 1.1.1.1` OK, `curl images.maas.io` 200). Procedure now a first-class runbook (`runbooks/dc-edge-rebuild.md`) + tool so it is not re-derived. Evidence: `docs/changelog-20260807-dc1-region-sequence.md` Item 4; `docs/audit/dc1-dhcp-handover-20260807.txt` sibling captures. Config.xml itself was INTACT (fsck names every inode it deletes; `/conf/config.xml` not among them) but the rebuild recreates the config via addressing regardless. | | SEC-032 | 2026-08-02 | **dc0 edge credentials have been RESIDENT ON THE RACK, undisclosed, since 2026-07-20 -- the dc0 analog of SEC-015, which dc1 opened and remediated the same phase.** Found while staging the dc0 edge rebuild: `ls` (never a read) of `vr1-dc0-rack:~/vr1-dc0-creds/` shows `vr1-dc0-edge_ed25519` (the PRIVATE half, 432 B, 0600) and `opnsense-api.txt` (173 B, 0600), both dated 2026-07-20 -- i.e. left in place at the original edge build. dc1's build shredded both equivalents in the same phase and disclosed the transient as SEC-015; dc0's never did, and 13 days passed with nothing noticing. **THIS ALSO ANSWERS SEC-021(a), which asked for "a decision between RE-MINT and locating an off-jumphost copy": the off-jumphost copy is this one.** `creds-manifests/vm-secret-locations:123` already declares `rack vr1-dc0 ssh vr1-dc0-rack:~/vr1-dc0-creds/*` as an audited location, so this was visible to a `--remote` sweep and simply had not been reconciled -- the gap is reconciliation, not detection. **The 2026-07-20 `opnsense-api.txt` is now DEAD MATERIAL:** the rebuild destroyed the config.xml that held its hash, so it authenticates to nothing. It was deliberately NOT deleted (hard rule 1 -- a finding is logged, not actioned mid-step, and it is the physical evidence behind SEC-021(a)). The edge PRIVATE KEY was likewise left in place: `opnsense-bootstrap-apikey.sh` and `opnsense-set-interface-v4.sh` both SSH the edge DIRECT with no ProxyJump, and the edge LAN is reachable only from the rack, so rack residency is a structural requirement of the current tooling, not an oversight to be quietly cleaned up. **REMEDIATED IN PASSING:** SEC-021(a)'s access gap is closed -- a freshly minted key is now consolidated at `~/vr1-dc0-creds/opnsense-api.txt` on the jumphost (0600, 173 B, shape-verified 1x`key=`/1x`secret=` at 80 chars each, sha256-compared to the source, secret never printed). `creds-matrix.py` E1 for `dc0-edge-api` is consequently CLEAN where it previously read EXPECTED-BUT-ABSENT. | measured read-only during the dc0 edge rebuild, `docs/audit/dc0-edge-rebuild-20260802.txt`; SEC-015 (the dc1 precedent); SEC-021(a); `creds-manifests/vm-secret-locations:123` | operator | **OPEN -- three items, none actioned here.** (1) Rule on the dead 2026-07-20 `opnsense-api.txt` on the rack: shred (the dc1 precedent) or retain as SEC-021 evidence. (2) The `S2 vr1-dc0 EXPECTED-BUT-ABSENT: 'opnsense-api.txt'` finding is STILL RED and is now red for a DIFFERENT reason than when the operator accepted it on 2026-07-30 -- then the credential did not exist; now it exists and `creds-manifests/vr1-dc0.manifest` simply does not declare it. The manifest is DERIVED (`creds-matrix.py --render`), so this is a one-line regeneration, deliberately left to the operator rather than rendered inside a rebuild task. `creds-matrix.tsv:76`'s note key `n-dc0-edge-api-absent` and its `creds-matrix-notes.md:123` text are now factually stale for the same reason -- REPLACE the wording with the new invariant, never delete the row to go green (the note itself says so). (3) Decide whether rack-resident edge keys are the accepted steady state for BOTH DCs (making SEC-015's remediation the outlier) or whether the tooling should gain a ProxyJump path so the key never leaves the jumphost. Custody detail off-repo per D-069. | | SEC-033 | 2026-08-04 | **The tls-certificates relation databag exposes vault's GLOBAL-CLIENT private key in cleartext to any juju model reader.** `juju show-unit ` (measured identical on ovn-central/0, mysql-innodb-cluster/0, keystone/0) prints vault's `ca` + `client.cert` + `client.key` from the relation databag -- the last being the SHARED global-client PRIVATE key. Inherent to the interface-tls-certificates GLOBAL-CLIENT model (vault distributes one shared client key to every consumer over the relation); NOT newly introduced and NOT deploy-specific -- the same holds on any Charmed-OpenStack + vault deploy. Impact: the global client key is only as protected as juju model access; a read-only juju user can extract it. Discovered while reading the certificates relation databag during the ovn-central cert re-evaluation. | measured read-only 2026-08-04, `docs/audit/ovn-central-cert-reeval-20260803.md`; interface-tls-certificates global-client model | operator | OPEN -- interface/upstream-level, not a deploy defect. Mitigate via juju model RBAC (bound who holds admin/read on the model). Roosevelt analog: identical on every DC's cloud. Accept-or-mitigate is an operator ruling; nothing to action in the deploy itself. | diff --git a/runbooks/README.md b/runbooks/README.md index b4b87f7..23adc6b 100644 --- a/runbooks/README.md +++ b/runbooks/README.md @@ -94,4 +94,5 @@ - ops-capi-recovery.md -- parking, restart, and LB repair for the CAPI/Magnum stack (post-deploy operations companion; not a deploy phase). Added 2026-06-10. - ops-restart-procedure.md -- full-cloud restart / recovery for planned windows and full power/network-loss events (DOCFIX-075). Indexed 2026-07-04 (was missing from this list since commit). +- dc-edge-rebuild.md -- DC OPNsense edge REBUILD after filesystem/power-cut damage (edge forwards but does not NAT; the SEC-031 class). Site-parameterised; proven on dc0 (2026-08-02) + dc1 (2026-08-07). Uses scripts/opnsense-console-rebuild.py. Added 2026-08-07 so the procedure is not reconstructed from an audit capture again. - ops-update-procedure.md -- routine update window: Juju controller patch, model agents, in-channel charm refreshes (DOCFIX-086; policy: D-071). Added 2026-07-04. diff --git a/runbooks/dc-edge-rebuild.md b/runbooks/dc-edge-rebuild.md new file mode 100644 index 0000000..1ba0be6 --- /dev/null +++ b/runbooks/dc-edge-rebuild.md @@ -0,0 +1,129 @@ +# Runbook: DC OPNsense edge REBUILD (filesystem / power-cut damage) + +**Use when** a DC edge forwards but does not NAT, or its GUI answers on the WAN, or +`dc-egress-check ` fails A3/A4 while the edge answers A2 -- i.e. the edge lost its pf +ruleset (typically fsck damage to `/etc/master.passwd` + `/etc/group` after an unclean +power cut; the SEC-031 class). **Verdict is REBUILD, not in-place repair:** a fresh +pristine disk + this bootstrap is faster and more reliable than single-user user-DB +surgery, and the edge config is fully recreated by the addressing step below. + +**Proven twice:** dc0 2026-08-02 (`docs/audit/dc0-edge-rebuild-20260802.txt`, the +worked reference) and dc1 2026-08-07 (`docs/changelog-20260807-dc1-region-sequence.md` +Item 4). Both ended `dc-egress-check ` **8/8**. This runbook GENERALISES those; it +exists because the dc0 run lived only as an audit capture, so dc1 had to reconstruct it. + +**Tools (all in-repo -- do NOT reinvent):** `scripts/opnsense-console-rebuild.py` +(site-agnostic D-112(c) console driver), `scripts/opnsense-bootstrap-apikey.sh`, +`scripts/opnsense-api.sh`, `scripts/opnsense-set-interface-v4.sh`, +`scripts/dc-egress-check.sh`. + +--- + +## Per-DC values (fill in, measured -- never inferred) + +| var | dc0 | dc1 | source | +|---|---|---|---| +| `RACK` (transit IP) | 172.31.0.2 | 172.31.0.6 | `scripts/lib-hosts.sh` | +| `DOMAIN` | vr1-dc0-opnsense | vr1-dc1-opnsense | inner tofu module | +| inner root | `opentofu/vr1-dc0-substrate` | `opentofu/vr1-dc1-substrate` | on voffice1 | +| `WAN` addr/pl gw | 172.30.2.2/24 172.30.2.1 | 172.30.3.2/24 172.30.3.1 | G12 / D-125 | +| `LAN` (provider-public gw) | 10.12.4.1/22 | 10.12.64.1/22 | lib-net `PLANE_GW` | +| rack provider-public leg / bridge | 10.12.4.2 / virbrN | 10.12.64.2 / virbr4 | `ip -4 -o addr` on rack | +| edge key (jumphost) | `~/vr1-dc0-creds/vr1-dc0-edge_ed25519` | `~/vr1-dc1-creds/vr1-dc1-edge_ed25519` | SEC-013/-015 | +| base image (voffice1) | `~/vr1-dc0-images/opnsense-26.7-nano.qcow2` | `~/vr1-dc1-images/opnsense-26.7-nano.qcow2` | present-before-apply | + +All `virsh` on the rack takes `-c qemu:///system` (rack default is `qemu:///session`). + +--- + +## Step 1 -- SAFETY PREP (read-only + backups) + +1. Rollback copy of the edge disk (no backingStore = no other rollback point): + `virsh -c qemu:///system shutdown $DOMAIN` (ACPI works even with a broken login); + wait for `shut off`; then on the rack `sudo cp -p .qcow2 .qcow2.pre-rebuild-`. +2. Backup the inner tfstate on voffice1: `cp -p terraform.tfstate ~/-substrate-tfstate-pre-edge-rebuild-.json` + sha256-verify identical. +3. Cut a TARGETED plan and **machine-assert it is edge-only** (this is the gate that + protects the 9 node VMs sharing this state): + ``` + cd "$REPO"/opentofu/-substrate/ # $REPO set per session bootstrap + tofu plan -replace=module._opnsense.libvirt_volume.disk \ + -replace=module._opnsense.libvirt_domain.vm -out=/tmp/edge-rebuild.plan + tofu show -json /tmp/edge-rebuild.plan | jq -r '.resource_changes[] | select(.change.actions!=["no-op"]) | .address+" "+(.change.actions|join(","))' + ``` + REQUIRE: exactly 2 non-no-op changes, BOTH the edge domain+disk, each `delete,create`; + 0 non-no-op changes naming anything else. (The refresh reports ~20 resources "changed + outside OpenTofu" -- that is DRIFT, not an action; assert on ACTIONS only.) +4. Confirm the base image the plan will use is PRESENT on voffice1 (the apply destroys + the volume before recreating -- a missing source strands the edge with no disk). + +## Step 2 -- REBUILD (destroy+recreate the edge, edge-only) + +`tofu apply /tmp/edge-rebuild.plan` -> `2 added, 0 changed, 2 destroyed`. Fresh disk is +base-image size / new mtime (measure the artifact, don't trust the log). Domain boots +factory: LAN `192.168.1.1/24`, WAN address-less, root/opnsense login. + +## Step 3 -- CONSOLE BOOTSTRAP (enables ssh + installs the edge key) + +Runs ON the rack (has pexpect). Stage the driver + the site edge PUBKEY (public): +``` +scp scripts/opnsense-console-rebuild.py .pub -> rack +ssh $RACK 'python3 ~/opnsense-console-rebuild.py --domain '$DOMAIN' --pubkey ~/.pub' +``` +Expect `CONSOLE-STATE=0` then `BOOTSTRAP-SEQUENCE-COMPLETE`. Verify the transcript shows +`CONFIG-WRITTEN`, `AK-IN-CONFIG=152`, `USER-MATERIALIZED`, `OK` (sshd restart). + +## Step 4 -- REACH + KEY-ONLY SSH (D-112(c) exit condition) + +A REBUILT edge has NEW host keys -- clear the rack's stale entries or every `BatchMode` +script fails with HOST IDENTIFICATION CHANGED: +``` +ssh $RACK 'cp -p ~/.ssh/known_hosts ~/.ssh/known_hosts.pre-rebuild-; + ssh-keygen -R 192.168.1.1; ssh-keygen -R $LAN_IP; + sudo ip addr add 192.168.1.2/24 dev ; # interim reach to factory LAN + ssh-keyscan -T8 192.168.1.1 >> ~/.ssh/known_hosts' +``` +Stage the edge PRIVATE key to the rack transiently (0600, SEC-015 pattern -- SHRED in +Step 7) OR reach the edge from the jumphost via ProxyCommand. Prove: +`ssh -i root@192.168.1.1 opnsense-version` -> `OPNsense 26.7`. + +## Step 5 -- API KEY + ADDRESSING (WAN first, LAN last) + +``` +cd ~/repo-stage/scripts # sha256-verify the 5 opnsense-*.sh/php against the repo first +OPNSENSE_SSH_KEY= bash opnsense-bootstrap-apikey.sh 192.168.1.1 .txt +# WAN (also REGENERATES the pf ruleset -> automatic outbound NAT appears here): +OPNSENSE_SSH_KEY= bash opnsense-set-interface-v4.sh --commit 192.168.1.1 wan $WAN_ADDR $WAN_PL $WAN_GW +# PROVE egress FROM THE EDGE before the LAN move (after it, the reach path is gone): +ssh -i root@192.168.1.1 'ping -c3 1.1.1.1; fetch -o /dev/null http://archive.ubuntu.com/ubuntu/dists/jammy/Release' +# LAN LAST -- the apply drops the session it rides on. EXPECTED to time out; config saves+applies. +OPNSENSE_SSH_KEY= bash opnsense-set-interface-v4.sh --commit 192.168.1.1 lan $LAN_ADDR 22 +``` +After the LAN move the edge is at `$LAN_ADDR`, reachable from the rack's PERMANENT +provider-public leg (no interim leg needed). If the script version only SAVES lan, apply +it: `ssh -i root@$LAN_ADDR configctl interface reconfigure lan`. Then +`configctl filter reload` and confirm `pfctl -s nat` shows +`nat on vtnet1 ... from (vtnet0:network) to any -> (vtnet1:0)`. +**tcsh trap:** the edge root shell is tcsh -- NO `2>&1` in edge commands ("Ambiguous +output redirect"). + +## Step 6 -- VERIFY (the gate) + +`ssh $RACK 'sudo -n bash -s -- check ' < scripts/dc-egress-check.sh` -> **pass=8 fail=0**. +Confirm the region VM egresses: from the `.6`, `ping 1.1.1.1` and `curl images.maas.io` 200. + +## Step 7 -- CLEANUP + RECORD + +- SHRED the transient edge private key on the rack (`shred -u`); remove the interim + `192.168.1.2/24` reach leg. Consolidate the API key to the jumphost `~/-creds/` + (0600, sha256-verified, secret never printed). +- SEC row: the open-router exposure ENDS with the pf ruleset restored -- close it. +- Changelog + `docs/CURRENT-STATE.md` (status). A REBUILT edge invalidated the rack + known_hosts (Step 4) -- fold any new lesson here. + +## Gotchas (measured, do not relearn) +- A rebuilt edge = new host keys -> clear rack known_hosts (Step 4). +- The LAN move DROPS the session mid-apply; config is saved+applied first (Step 5). +- Edge root shell is tcsh: no `2>&1`, no nested-quoted one-liners. +- Assert tofu scope on ACTIONS, not the drift note (Step 1.3). +- `opnsense-set-interface-v4.sh` uses the LEGACY interfaces page, not REST (D-113 amdt) -- + do not "fix" it to the API. diff --git a/scripts/opnsense-console-rebuild.py b/scripts/opnsense-console-rebuild.py new file mode 100644 index 0000000..a673a97 --- /dev/null +++ b/scripts/opnsense-console-rebuild.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# scripts/opnsense-console-rebuild.py --domain --pubkey [--log ] +# +# D-112(c) console bootstrap for a FRESHLY-REBUILT OPNsense DC edge. Drives the serial +# console (virsh console) of a just-deployed pristine edge: catches the factory login +# (root/opnsense), drops to a shell, and runs one PHP payload that +# (1) enables sshd + permitrootlogin + group admins, +# (2) installs the site's edge SERVICE pubkey on root, and +# (3) materialises it with local_user_set() (write_config alone does NOT create +# /root/.ssh/authorized_keys -- dc0 lesson v). +# After this, key-only SSH to the factory LAN (192.168.1.1) works and the edge can be +# addressed by scripts/opnsense-set-interface-v4.sh. +# +# RUNS ON THE DC RACK HOST (the libvirt host of the inner edge VM). Needs python3 + +# pexpect there (measured present on the VR1 racks). It is SITE-AGNOSTIC: the only +# per-DC inputs are the libvirt domain name and the edge pubkey file -- NOTHING is +# hardcoded, so it replaces the per-DC one-off drivers (d112c-console--rebuild.py) +# the dc0 2026-08-02 and dc1 2026-08-07 rebuilds each carried as separate copies. +# +# PROVENANCE: generalised 2026-08-07 from the dc0-rebuild driver +# (docs/audit/dc0-edge-rebuild-20260802.txt SECTION 3) after it was used verbatim (bar +# DOMAIN/PUB) to repair the dc1 edge (SEC-031). Lessons baked in: (i) payload shipped as +# <=160-char base64 chunks -- the serial console wraps long lines; (iii) NO nested-quoted +# one-liners and NO `2>&1` in edge commands -- the edge root shell is tcsh; (iv) the +# bootstrap PHP requires util.inc (shell_safe lives there), not just config.inc. +# +# The edge PUBKEY is PUBLIC material; the private half never touches this tool. +# Exit 0 on BOOTSTRAP-SEQUENCE-COMPLETE; 2 if no known console state is reached. +import argparse, base64, sys, time + +try: + import pexpect +except ImportError: + sys.exit("FAIL: python3 pexpect is required on the rack (pip install pexpect)") + +ap = argparse.ArgumentParser(description="D-112(c) console bootstrap for a rebuilt OPNsense edge") +ap.add_argument("--domain", required=True, help="libvirt domain name of the edge (e.g. vr1-dc1-opnsense)") +ap.add_argument("--pubkey", required=True, help="path to the site edge SERVICE pubkey (public material)") +ap.add_argument("--log", default=None, help="console transcript path (default: ~/opnsense-console-rebuild-.log)") +a = ap.parse_args() + +import os +PUB = open(a.pubkey).read().strip() +if not PUB.startswith("ssh-"): + sys.exit("FAIL: --pubkey %s does not look like an ssh public key" % a.pubkey) +LOG = a.log or os.path.expanduser("~/opnsense-console-rebuild-%s.log" % a.domain) + +PHP = '''&1 | tail -3") +sh("grep -c shell_safe /usr/local/etc/inc/util.inc") # lesson iv: shell_safe lives in util.inc + +b64 = base64.b64encode(PHP.encode()).decode() +sh("rm -f /tmp/b.b64 /tmp/b.php") +for ch in [b64[i:i+160] for i in range(0, len(b64), 160)]: # lesson i: console wraps long lines + sh("echo %s >> /tmp/b.b64" % ch) +sh("openssl base64 -d < /tmp/b.b64 > /tmp/b.php") +sh("php -l /tmp/b.php", "No syntax errors"); c.expect(PROMPT) +c.sendline("php /tmp/b.php") +c.expect("CONFIG-WRITTEN", timeout=60) +c.expect("USER-MATERIALIZED", timeout=60) +c.expect(PROMPT) + +sh("ls -la /root/.ssh/ 2>&1 | tail -3") # AFTER-state, the artifact +sh("grep -c authorizedkeys /conf/config.xml") +sh("configctl openssh restart") +time.sleep(5) +sh("rm -f /tmp/b.b64 /tmp/b.php") +c.sendline("exit"); time.sleep(1); c.sendcontrol("]") +print("BOOTSTRAP-SEQUENCE-COMPLETE (transcript: %s)" % LOG)