diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 31c2bda..cbf7fd6 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -475,7 +475,7 @@ | G9 | DC0 outer apply (deploy step A) | [V] operator-gated, logged (`run-logged.sh`), after G1-G8; audit exit criteria met (charter Phase 6). SEC pre-apply dependency (S2): SEC-010's transit FORWARD-drop is applied+verified at deploy step B via `site-headend-install.sh --host-nodes --check` on vvr1-dc0 (gate G10) -- the ONLY SEC row gated on this apply (register of record: security-ledger). CANONICAL ENTRY DOC (probe hole H1): `runbooks/dc-dc-phase2-tofu-dc-substrate.md`, with `docs/dc0-deploy-readiness.md` section E as the step table | operator | CLOSED 2026-07-19: G8 same-session planes check passed (6x 0 leases, 0 attachments); saved plan == 6/0/6 applied in the logged dc0-deploy window; convergence re-plan = no differences; vvr1-dc0 running, prior guests untouched | | G10 | Deploy steps B-E in-sequence gates: SEC-010 `--host-nodes --check` on vvr1-dc0; depth-4 nested boot; D-125 foreign-MAC egress test; MAAS reachability + `TF_VAR_maas_api_key` before step D; netem placeholder step E | [V] exercised during the gated deploy | session (each mutation operator-approved) | Step B DONE 2026-07-20 (`--check` EXIT 0 incl. SEC-010, `docs/audit/stepB-check-20260720-final.txt`; interfaces enp1s0/enp2s0). Depth-4 nested boot DONE (10 domains running inside vvr1-dc0). D-125 egress isolation test PASS 2026-07-20 (`docs/audit/d125-egress-gate-20260720-matrix.txt`), and the edge itself now egresses 0% loss after the v4 addressing. Step D COMPLETE incl. commissioning: ALL 9 NODES READY 2026-07-21 (two stacked faults diagnosed + fixed -- `docs/audit/commissioning-diag-20260721.txt`; section 1). Step E (netem) DONE 2026-07-21: sudo fragment installed+verified, module local-mode amendment, targeted apply 1/0/0 exact (operator-ruled at the 1/1/0 STOP), placeholder profile live on virbr5, virbr7/virbr3 untouched (`docs/audit/stepE-netem-20260721.txt` + `outer-{plan,apply}-20260721-netem*.txt`). **G10 CLOSED 2026-07-21** | | G11 | Operator signs THIS document | [R] read top-to-bottom; discrepancies resolved in the document | operator | CLOSED: RE-SIGNED 2026-07-19 at audit exit, section 11 (replaces the 2026-07-18 signature) | -| G12 | `vr1-dc1` build | [R] operator rules dc1 transit/rack addressing; then vars + substrate authored | operator + session | OPEN -- [R] leg CLOSED 2026-07-21: addressing RATIFIED (D-124 amendment 2026-07-21, utterance quoted). [V] leg IN PROGRESS (branch `dc-dc-g12-dc1-substrate`): apex confirm-free DONE 2026-07-21 -- planes/uplink already assigned+consistent, transit 172.31.0.4/30 + rack 10.12.68.2 FREE (`docs/audit/dc1-apex-confirm-20260721.txt`); importer per-site dc1 support shipped (harness 117/117) with live dry-run preflight PASS (`docs/audit/dc1-rack-import-dryrun-20260721.txt`). vars + substrate root + lib-net dc1 arm COMMITTED 2026-07-22 (successor session landed the disconnected item 3 + the harness reconcile as changelog item 4): six harnesses reconciled to the ratified dc1 arm, phase-00 PLANES parity guard added, rbd-mirror/radosgw cross-DC reminder fixed; gauntlet **ALL GREEN (76)** (`docs/audit/gauntlet-20260722-g12-reconcile.txt`), repo-lint 0-fail. Apex `--commit` EXECUTED 2026-07-22 (operator-gated): 172.31.0.4/30 + 10.12.68.2/22 CREATED, post-commit read-back idempotent (`docs/audit/dc1-rack-import-commit-20260722.txt`). dc1 svc key minted (creds-audit CLEAN), tfvars authored (local), **outer step-A apply DONE 2026-07-22**: saved plan 5/0/0 exact, converged ZERO DIFF (section 5), vvr1-dc1 RUNNING, prior guests untouched (as-executed log dc1-deploy; changelog-20260722-g12-dc1-build.md). NEXT (gated): dc1 step B analog -- transit leg 172.31.0.5/30 on voffice1, vvr1-dc1 bootstrap (site-headend rack role + SEC-010 both ends), OPNsense 26.7 staging; then inner apply FROM voffice1 (D-128) with first-apply MAC pins + D-131 standup DoD (dc-rack-net install, forwarder 10.12.68.3, maas-node-power) | +| G12 | `vr1-dc1` build | [R] operator rules dc1 transit/rack addressing; then vars + substrate authored | operator + session | OPEN -- [R] leg CLOSED 2026-07-21: addressing RATIFIED (D-124 amendment 2026-07-21, utterance quoted). [V] leg IN PROGRESS (branch `dc-dc-g12-dc1-substrate`): apex confirm-free DONE 2026-07-21 -- planes/uplink already assigned+consistent, transit 172.31.0.4/30 + rack 10.12.68.2 FREE (`docs/audit/dc1-apex-confirm-20260721.txt`); importer per-site dc1 support shipped (harness 117/117) with live dry-run preflight PASS (`docs/audit/dc1-rack-import-dryrun-20260721.txt`). vars + substrate root + lib-net dc1 arm COMMITTED 2026-07-22 (successor session landed the disconnected item 3 + the harness reconcile as changelog item 4): six harnesses reconciled to the ratified dc1 arm, phase-00 PLANES parity guard added, rbd-mirror/radosgw cross-DC reminder fixed; gauntlet **ALL GREEN (76)** (`docs/audit/gauntlet-20260722-g12-reconcile.txt`), repo-lint 0-fail. Apex `--commit` EXECUTED 2026-07-22 (operator-gated): 172.31.0.4/30 + 10.12.68.2/22 CREATED, post-commit read-back idempotent (`docs/audit/dc1-rack-import-commit-20260722.txt`). dc1 svc key minted (creds-audit CLEAN), tfvars authored (local), **outer step-A apply DONE 2026-07-22**: saved plan 5/0/0 exact, converged ZERO DIFF (section 5), vvr1-dc1 RUNNING, prior guests untouched (as-executed log dc1-deploy; changelog-20260722-g12-dc1-build.md). **Step B COMPLETE 2026-07-22**: cloudinit-vm interface_macs port + voffice1 dc1-transit NIC (0/2/0 exact, MACs pinned both domains, post-bounce battery ALL PASS, converged zero diff -- `docs/audit/outer-plan-20260722-voffice1-dc1nic.txt`), transit LIVE (voffice1 .5/30 <-> rack .6/30, dc1-key ssh proven), rack ENROLLED (region lists vvr1-dc1 `nmpcq4`), SEC-010 applied+verified BOTH ends, OPNsense 26.7 base staged via hash-verified copy of dc0's proven artifact; named gate EXIT 0 `docs/audit/dc1-stepB-check-20260722-final.txt` (changelog-20260722 items 5-8, three queued findings). NEXT (gated): inner apply FROM voffice1 (`opentofu/vr1-dc1-substrate/`, 54 MACs pre-pinned), then D-125 egress gate, edge bootstrap (D-112(c)/D-113(a2) on 26.7), rack standup DoD (dc-rack-net install dc1, forwarder 10.12.68.3, region-side DHCP, maas-node-power) | | G13 | D-129 residuals | [R] operator-gated live plugin install on office1-opnsense; qga channel retrofit at that edge's next scheduled restart. All 4 sub-decisions RULED 2026-07-21 (D-129 Status line) -- only the two execution items remain | operator | OPEN (execution only; decision content complete) | | G14 | 9 OPEN SEC rows (SEC-001, -003..-008, plus SEC-012 + SEC-013 opened 2026-07-20 for credentials this deploy created; SEC-010 CLOSED 2026-07-20, operator-ruled, applied+verified both transit ends) | [R] per-row: rotations/flips at v1 close (external to VR1 track); SEC-012 also carries a SCOPE question (libvirt-group grant is broader than the power verbs MAAS needs), SEC-013 is tied to whether `opentofu/vr1-dc0-maas` is retired | operator / external | `docs/security-ledger.md` (register of record, GA-R4/F3); count re-verified vs `bash scripts/ledger-scan.sh` 2026-07-20 | | G15 | D-068 / D-071 rulings | [R] operator rules (section 8); neither blocks the VR1 substrate | operator | D-071 ADOPTED 2026-07-21 (all four points); D-068 remains PROPOSED/OPEN (items 2-3 + the item-1 re-scoped migration plan) | diff --git a/docs/audit/dc1-stepB-check-20260722-final.txt b/docs/audit/dc1-stepB-check-20260722-final.txt new file mode 100644 index 0000000..9af8e62 --- /dev/null +++ b/docs/audit/dc1-stepB-check-20260722-final.txt @@ -0,0 +1,17 @@ +# dc1 step-B gate FINAL: site-headend-install --check on vvr1-dc1 -- 2026-07-22T21:37:37Z +OK: maas snap 3.7.2-17972-g.35e297c4d (3.7/stable) +OK: MAAS rackd running +OK: enrolled to region http://10.10.0.20:5240/MAAS +== node-host (Model B) readiness == + [ok] libvirt/qemu installed + [ok] nested KVM ON (kvm_amd) + [ok] jessea123 in libvirt group + [ok] inner pool dir /var/lib/libvirt/vr1-dc1-inner + [ok] opnsense base /var/lib/libvirt/vr1-dc1-inner/opnsense-26.7-nano.qcow2 + [ok] SEC-010 transit FORWARD-drop present + transit interface 'enp1s0' exists + [ok] D-125 WAN bridge 'br-vr1-dc1-wan' present + uplink 'enp2s0' enslaved + [gate] DEPLOY-TIME (unprovable here, like the depth-4 boot gate): before the OPNsense chain + rides it, prove egress in ISOLATION -- attach a throwaway guest to 'br-vr1-dc1-wan', + confirm it gets a vcloud-ISP address and pings out. FAIL => revert to the D-125 + double-NAT fallback, NOT a redesign. +check-exit=0 diff --git a/docs/audit/dc1-stepB-check-20260722.txt b/docs/audit/dc1-stepB-check-20260722.txt new file mode 100644 index 0000000..e93e407 --- /dev/null +++ b/docs/audit/dc1-stepB-check-20260722.txt @@ -0,0 +1,17 @@ +# dc1 step-B gate: site-headend-install --check on vvr1-dc1 -- 2026-07-22T21:32:50Z +OK: maas snap 3.7.2-17972-g.35e297c4d (3.7/stable) +OK: MAAS rackd running +OK: enrolled to region http://10.10.0.20:5240/MAAS +== node-host (Model B) readiness == + [ok] libvirt/qemu installed + [ok] nested KVM ON (kvm_amd) + [ok] jessea123 in libvirt group + [ok] inner pool dir /var/lib/libvirt/vr1-dc1-inner + [--] opnsense base missing -- run: opnsense-prep-image.sh 26.1 /var/lib/libvirt/vr1-dc1-inner/opnsense-26.7-nano.qcow2 + [ok] SEC-010 transit FORWARD-drop present + transit interface 'enp1s0' exists + [ok] D-125 WAN bridge 'br-vr1-dc1-wan' present + uplink 'enp2s0' enslaved + [gate] DEPLOY-TIME (unprovable here, like the depth-4 boot gate): before the OPNsense chain + rides it, prove egress in ISOLATION -- attach a throwaway guest to 'br-vr1-dc1-wan', + confirm it gets a vcloud-ISP address and pings out. FAIL => revert to the D-125 + double-NAT fallback, NOT a redesign. +check-exit=1 diff --git a/docs/audit/outer-plan-20260722-voffice1-dc1nic.txt b/docs/audit/outer-plan-20260722-voffice1-dc1nic.txt new file mode 100644 index 0000000..c978ca8 --- /dev/null +++ b/docs/audit/outer-plan-20260722-voffice1-dc1nic.txt @@ -0,0 +1,99 @@ +module.netem_vr1_dc0_vr1_dc1.terraform_data.netem: Refreshing state... [id=1ea36d3f-e7af-984c-8157-152724a0b85a] +module.vr1_dc1_storage.libvirt_pool.dc: Refreshing state... [id=4a1df114-ee04-4c80-9233-cc0c140c8556] +module.office1_storage.libvirt_pool.dc: Refreshing state... [id=5f94194c-69c1-4b04-a85f-c18d87303a03] +module.mesh_vr1_dc0_office1.libvirt_network.link: Refreshing state... [id=8318548f-c3d6-4e06-bef4-fe3f11d68125] +module.voffice1.libvirt_cloudinit_disk.seed: Refreshing state... [id=a4694210c663c9ce] +module.vvr1_dc0.libvirt_cloudinit_disk.seed: Refreshing state... [id=ff281478c6083cc3] +module.office1_network.libvirt_network.office1_local: Refreshing state... [id=8fdd2a97-417c-44d4-89e4-ae8d65594135] +module.mesh_vr1_dc0_vr1_dc1.libvirt_network.link: Refreshing state... [id=9cbc8589-9f40-48e6-872e-ef3abfe29a93] +module.mesh_vr1_dc1_office1.libvirt_network.link: Refreshing state... [id=38a20d2d-cd91-4604-a5f4-8e2a6609633c] +module.vr1_dc1_uplink.libvirt_network.site_wan: Refreshing state... [id=4aad75c2-924f-410c-96bb-fa9217f8b4ea] +module.vr1_dc0_uplink.libvirt_network.site_wan: Refreshing state... [id=f3500153-e4de-45f1-8854-9c92974a6094] +module.vvr1_dc1.libvirt_cloudinit_disk.seed: Refreshing state... [id=41e9ec4b712038fc] +module.vr1_dc0_storage.libvirt_pool.dc: Refreshing state... [id=7ce1101c-a89e-40ca-9263-5f572bee40a9] +module.voffice1.libvirt_volume.seed: Refreshing state... [id=/var/lib/libvirt/vr1/office1/voffice1-cloudinit.iso] +module.office1_opnsense.libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1/office1/office1-opnsense-disk.qcow2] +module.ubuntu_noble_base.libvirt_volume.base: Refreshing state... [id=/var/lib/libvirt/vr1/office1/ubuntu-24.04-base.qcow2] +module.vvr1_dc1.libvirt_volume.seed: Refreshing state... [id=/var/lib/libvirt/vr1/vr1-dc1/vvr1-dc1-cloudinit.iso] +module.vvr1_dc0.libvirt_volume.seed: Refreshing state... [id=/var/lib/libvirt/vr1/vr1-dc0/vvr1-dc0-cloudinit.iso] +module.vvr1_dc1.libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1/vr1-dc1/vvr1-dc1-disk.qcow2] +module.voffice1.libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1/office1/voffice1-disk.qcow2] +module.vvr1_dc0.libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1/vr1-dc0/vvr1-dc0-disk.qcow2] +module.office1_opnsense.libvirt_domain.vm: Refreshing state... [name=office1-opnsense] +module.voffice1.libvirt_domain.vm: Refreshing state... [name=voffice1] +module.vvr1_dc0.libvirt_domain.vm: Refreshing state... [name=vvr1-dc0] +module.vvr1_dc1.libvirt_domain.vm: Refreshing state... [name=vvr1-dc1] + +OpenTofu used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + ~ update in-place (current -> planned) + +OpenTofu will perform the following actions: + + # module.voffice1.libvirt_domain.vm will be updated in-place + ~ resource "libvirt_domain" "vm" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:89:e1:19" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:99:04:01" + } + # (2 unchanged attributes hidden) + }, + + { + + mac = { + + address = "52:54:01:d1:fe:01" + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "mesh-vr1-dc1-office1" + } + } + }, + ] + # (1 unchanged attribute hidden) + } + id = 5 + name = "voffice1" + # (10 unchanged attributes hidden) + } + + # module.vvr1_dc1.libvirt_domain.vm will be updated in-place + ~ resource "libvirt_domain" "vm" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:11:39:ce" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:3c:a6:a8" + } + # (2 unchanged attributes hidden) + }, + ] + # (1 unchanged attribute hidden) + } + id = 8 + name = "vvr1-dc1" + # (10 unchanged attributes hidden) + } + +Plan: 0 to add, 2 to change, 0 to destroy. + +───────────────────────────────────────────────────────────────────────────── + +Note: You didn't use the -out option to save this plan, so OpenTofu can't +guarantee to take exactly these actions if you run "tofu apply" now. diff --git a/docs/changelog-20260722-g12-dc1-build.md b/docs/changelog-20260722-g12-dc1-build.md index cc18165..49a1c0e 100644 --- a/docs/changelog-20260722-g12-dc1-build.md +++ b/docs/changelog-20260722-g12-dc1-build.md @@ -75,12 +75,83 @@ the follow-up commit; lint back to 0-fail. The applied-plan record remains the two dated captures. +## Item 5 -- cloudinit-vm interface_macs port + voffice1 dc1-transit NIC (gated apply) + +What: `modules/cloudinit-vm` gained `interface_macs` (ported verbatim-in-intent +from `modules/node-vm`, same validations; harness +4 cases, cloudinit-vm 10/10) +-- the 2026-07-20 voffice1 Kea/MAC-regen incident class, closed at the module +layer. Root wiring: `module.voffice1` NIC3 -> `mesh-vr1-dc1-office1` with all +three MACs pinned (NIC1/NIC2 = measured live values; NIC3 = pre-pinned +`52:54:01:d1:fe:01`, dc1 scheme, fe = region-side); `module.vvr1_dc1` pin-adopts +its two measured step-A MACs (standup DoD invariant). Saved plan +`voffice1-dc1nic-20260722.tfplan` = **0/2/0, zero replaces**, all five MAC +values verified in the diff (`docs/audit/outer-plan-20260722-voffice1-dc1nic.txt`); +applied logged (trap 1e: BOTH domains bounced -- presented as such). Post-bounce +battery ALL PASS: 4/4 domains running, voffice1 MACs exactly as pinned, Kea +lease 10.10.0.20 intact, region dhcpd up, netbox 302 + tailscale up (nested LXD +VMs self-recovered, ~3 min), dc0 rack: dhcpd + dc0-node-dns + dc0-rack-legs +active, forwarder SOA answers. Convergence re-plan ZERO DIFF. + +Revert: remove NIC3 + interface_macs from main.tf, revert the module + +harness (`git checkout` the four files), re-plan/apply (bounces again). + +## Item 6 -- voffice1 dc1 transit leg (in-guest netplan, gated) + +`/etc/netplan/61-transit-dc1.yaml` (0600) on voffice1: enp3s0 static +`172.31.0.5/30` (per-DC drop-in; dc0's `60-transit.yaml` untouched). Verified: +enp3s0 UP with .5/30, ping 172.31.0.6 = 0% loss, first ssh into vvr1-dc1 over +the transit with the dc1 key OK (hostname vvr1-dc1, nested-KVM module +present, region route + 10.10.0.20 reach pre-staged by step-A cloud-init). +Revert: remove the drop-in + `netplan apply`. + +## Item 7 -- rack bootstrap EXECUTED (site-headend-install, dc1-parameterized) + +Enrollment secret staged region->rack as a host-to-host pipe (0600 +`/root/region-enroll.secret`, 32 bytes verified by count, value never in +context). Snap system proxy set to `http://10.10.0.20:8000` (dc0-measured +mechanism; env proxy for apt). Dry-run FIRST exposed that the script's +host-nodes DEFAULTS are dc0-flavored -- all overridable by existing flags; +run with `--wan-bridge br-vr1-dc1-wan --inner-pool-path +/var/lib/libvirt/vr1-dc1-inner --opnsense-base ...26.7...`. First real run +FAILED exit 4: stale base-image apt index -> 404s on superseded debs via the +proxy; fixed with `apt-get update`, idempotent re-run **exit 0**: rack +ENROLLED, nested KVM on, inner pool + AppArmor, SEC-010 rack end, WAN bridge +verified with enp2s0 enslaved. Region-side verify: `maas admin +rack-controllers read` lists **vvr1-dc1 (nmpcq4)**. SEC-010 REGION end: +voffice1's `/etc/nftables-sec010.nft` extended with the enp3s0 drop pair +(dc0 idiom), table reloaded clean, ruleset = both legs dropped. +`--check` capture: `docs/audit/dc1-stepB-check-20260722.txt` (sole [--] = +opnsense base, item 8). + +QUEUED findings (logged, not fixed mid-step): (a) site-headend-install +NOTE/hint prose still says "26.1" and "vr1-dc0-substrate" even when +parameterized for dc1 (cosmetic, misleads operators); (b) +`nftables-sec010.nft` reload is NOT idempotent -- `nft -f` on a live table +appends, so a service restart duplicates rules (observed; cleaned via +`nft delete table` + restart; the file should flush first); (c) +`opnsense-prep-image.sh` dies on `BASH_SOURCE[0]: unbound variable` when +piped via `bash -s` under `set -u` (ran into it before the mirror guard). + +Revert: rack side -- `snap remove maas`, remove nftables-sec010 + +sec010-fw + kvm-nested modprobe + inner pool dir on vvr1-dc1; region side -- +remove the enp3s0 pair from voffice1's nft file + restart unit; delete +the staged secret file. + +## Item 8 -- OPNsense 26.7 base staged on dc1 (proven-artifact copy) + +`opnsense-prep-image.sh` requires `OPNSENSE_MIRROR_BASE` (deliberately not +repo-recorded; mirrors change). Instead of choosing a mirror, the dc0 rack's +OPERATOR-RULED and boot-PROVEN 26.7 base was streamed rack->rack through the +jump path (direct rack-to-rack is SEC-010-dropped, correctly) and +sha256-verified on arrival against the dc0 source hash +(`3981821e3a3c...476627d`). Same bits that passed the D-112 boot path on dc0. +Revert: delete `/var/lib/libvirt/vr1-dc1-inner/opnsense-26.7-nano.qcow2`. + ## Next (gated, not run here) -Step B analog for dc1 per the dc0 sequence of record: transit leg -(region end 172.31.0.5/30 on voffice1), vvr1-dc1 bootstrap -(site-headend-install rack role + SEC-010 both ends), OPNsense 26.7 nano -staging; then the inner apply FROM voffice1 (D-128 Plane 2) with MAC pins -from the FIRST apply, and the D-131/D-124-amendment standup -definition-of-done items (dc-rack-net.sh install, forwarder 10.12.68.3, -maas-node-power per-machine virsh). Runbook + CURRENT-STATE govern. +Inner apply FROM voffice1 (D-128 Plane 2; `opentofu/vr1-dc1-substrate/`, +54 MACs pre-pinned) after the step-B `--check` re-run is EXIT 0; then the +D-125 egress isolation gate on `br-vr1-dc1-wan`, edge bootstrap (D-112(c) + +D-113(a2) on 26.7), rack standup DoD (dc-rack-net.sh install dc1, forwarder +10.12.68.3, region-side DHCP on metal-admin, maas-node-power). Runbook + +CURRENT-STATE govern. diff --git a/opentofu/main.tf b/opentofu/main.tf index badcdbe..80efbee 100644 --- a/opentofu/main.tf +++ b/opentofu/main.tf @@ -186,6 +186,18 @@ network_names = [ module.office1_network.network_name, # NIC1 enp1s0 -> lan (office1-local, DHCP/Kea) module.mesh_vr1_dc0_office1.network_name, # NIC2 -> transit (region end, static .1/30) + module.mesh_vr1_dc1_office1.network_name, # NIC3 -> dc1 transit (region end, static .5/30, D-124 amendment) + ] + # MAC pins (interface_macs, ported to cloudinit-vm 2026-07-22): NIC1/NIC2 = + # MEASURED live values (virsh domiflist voffice1) -- NIC1's MAC keys the Kea + # 10.10.0.20 reservation, the 2026-07-20 regen incident class. NIC3 = NEW, + # pre-pinned from first apply (dc1 standup DoD invariant) in the dc1 + # locally-administered scheme (52:54:01:d1:NN:PP; fe = region-side host, + # outside the node NN range). + interface_macs = [ + "52:54:00:89:e1:19", # NIC1 office1-local (Kea reservation key) + "52:54:00:99:04:01", # NIC2 dc0 mesh transit + "52:54:01:d1:fe:01", # NIC3 dc1 mesh transit (pre-pinned) ] expose_nested_virt = true @@ -538,6 +550,13 @@ module.mesh_vr1_dc1_office1.network_name, # NIC1 enp1s0 -> mgmt (transit; SEC-010 keys here) module.vr1_dc1_uplink.network_name, # NIC2 enp2s0 -> uplink (IP-less port of br-vr1-dc1-wan) ] + # MAC pin-adoption (2026-07-22, step-A first-apply values MEASURED via + # virsh domiflist -- the dc1 standup DoD invariant: pinned before any + # in-place update can regenerate them). + interface_macs = [ + "52:54:00:11:39:ce", # NIC1 dc1 mesh transit + "52:54:00:3c:a6:a8", # NIC2 dc1 uplink (br-vr1-dc1-wan port) + ] user_data = <<-EOT #cloud-config diff --git a/opentofu/modules/cloudinit-vm/main.tf b/opentofu/modules/cloudinit-vm/main.tf index 6f6d907..6aa8e49 100644 --- a/opentofu/modules/cloudinit-vm/main.tf +++ b/opentofu/modules/cloudinit-vm/main.tf @@ -154,10 +154,15 @@ ] interfaces = [ - for net_name in var.network_names : { + for i, net_name in var.network_names : { model = { type = "virtio" } + # MAC pinning (see variables.tf interface_macs): an unpinned MAC is + # provider-owned and an "in-place" apply can regenerate it (measured + # 2026-07-20, voffice1 Kea incident). Same wiring as modules/node-vm, + # verified against dmacvicar/libvirt 0.9.8 schema 2026-07-21. + mac = length(var.interface_macs) > 0 ? { address = var.interface_macs[i] } : null source = { network = { network = net_name diff --git a/opentofu/modules/cloudinit-vm/variables.tf b/opentofu/modules/cloudinit-vm/variables.tf index dfcb498..fb4933d 100644 --- a/opentofu/modules/cloudinit-vm/variables.tf +++ b/opentofu/modules/cloudinit-vm/variables.tf @@ -39,6 +39,30 @@ type = list(string) } +variable "interface_macs" { + description = <<-EOT + Optional ordered list of MAC addresses, one per network_names entry (same + order), PINNING each NIC's MAC in config. Empty (the default) lets libvirt + generate MACs. Ported from modules/node-vm (same incident class): an + unpinned MAC is provider-owned and an "in-place" domain update can + silently REGENERATE it -- measured 2026-07-20 on voffice1, where the + transit-NIC add regenerated the office1-local NIC's MAC and broke its Kea + reservation. Pin any VM whose address is leased/reserved by MAC. + EOT + type = list(string) + default = [] + + validation { + condition = length(var.interface_macs) == 0 || length(var.interface_macs) == length(var.network_names) + error_message = "interface_macs must be empty or exactly one MAC per network_names entry (partial pinning would silently leave some NICs drift-prone)." + } + + validation { + condition = alltrue([for m in var.interface_macs : can(regex("^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$", m))]) + error_message = "Each interface_macs entry must be a colon-separated 6-byte hex MAC (e.g. 52:54:00:ab:cd:ef)." + } +} + variable "user_data" { description = <<-EOT Full #cloud-config YAML content (a real, chosen configuration -- hostname, diff --git a/tests/cloudinit-vm/run-tests.sh b/tests/cloudinit-vm/run-tests.sh index b4ab4f1..089ff6c 100644 --- a/tests/cloudinit-vm/run-tests.sh +++ b/tests/cloudinit-vm/run-tests.sh @@ -34,6 +34,22 @@ && no "T5 data-disk volume has NO ignore_changes" \ || ok "T5 data-disk volume has NO ignore_changes" +# T5a-T5d: interface_macs MAC pinning (ported from node-vm 2026-07-22 -- the +# 2026-07-20 voffice1 Kea/MAC-regen incident class). Same regression intent as +# tests/node-vm: the var, both validations, and the conditional wiring must stay. +grep -q 'variable "interface_macs"' "$MOD/variables.tf" \ + && ok "T5a interface_macs variable present" \ + || no "T5a interface_macs variable present" +grep -q 'exactly one MAC per network_names entry' "$MOD/variables.tf" \ + && ok "T5b count validation present (all-or-nothing pinning)" \ + || no "T5b count validation present (all-or-nothing pinning)" +grep -q '0-9a-fA-F.*{2}' "$MOD/variables.tf" \ + && ok "T5c MAC format validation present" \ + || no "T5c MAC format validation present" +grep -q 'mac = length(var.interface_macs) > 0 ? { address = var.interface_macs\[i\] } : null' "$MOD/main.tf" \ + && ok "T5d interfaces wiring carries the conditional MAC pin" \ + || no "T5d interfaces wiring carries the conditional MAC pin" + # T6: module still validates when tofu is available (init -backend=false is # offline once the provider is in the plugin cache; skip cleanly otherwise) if command -v tofu >/dev/null 2>&1; then