diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 88bbbbf..beb9222 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -157,9 +157,17 @@ D-121 Option C. Committee record closed by addendum (its mechanisms were wrong; its instrument found the cause). D-131 PROPOSED (rack-only node DNS strategy -- Roosevelt-relevant). SEC-014 OPENED (rack cluster - secret exposure during diagnosis). Queued delivery: MAC pinning in - modules/node-vm, forwarder + rack-legs repo-carried persistence, - appendix-A entries, LP report, stale pod object cleanup. + secret exposure during diagnosis). Queued delivery: forwarder + + rack-legs repo-carried persistence, appendix-A entries, LP report, + stale pod object cleanup. **MAC pinning SHIPPED 2026-07-21** (54 MACs + measured via `virsh domiflist` + pinned in modules/node-vm + + vr1-dc0-substrate; harness 15 cases; gauntlet 73 ALL GREEN) together + with an operator-ruled power-ownership guard (`ignore_changes = + [running]` -- MAAS owns node power; the pin-adoption plan had carried + 9 out-of-band power-ons). Verification plan captured + (`docs/audit/inner-plan-20260721-macpin.txt`: 0/9/0, 54 mac adoptions, + ZERO replaces); the inner-root APPLY on voffice1 that adopts the pins + into state is PENDING, operator-gated. History of the diagnosis (superseded; kept for the audit trail): the 2026-07-20 state read "3 nodes Ready, 6 timed out." Established: PXE and the ephemeral handoff WORK, and the ephemeral OS boots diff --git a/docs/audit/inner-plan-20260721-macpin.txt b/docs/audit/inner-plan-20260721-macpin.txt new file mode 100644 index 0000000..11df09e --- /dev/null +++ b/docs/audit/inner-plan-20260721-macpin.txt @@ -0,0 +1,482 @@ +module.inner_storage.libvirt_pool.dc: Refreshing state... [id=62bb75eb-b7b9-4659-bca3-f75825ae2cdf] +module.vr1_dc0_wan.libvirt_network.wan_bridge: Refreshing state... [id=fabdac49-5daf-4cd6-aa0a-40d43ace9b1d] +module.vr1_dc0_planes.libvirt_network.plane["metal-internal"]: Refreshing state... [id=edbc1aa5-5ac0-46e1-b46c-51f6bdc27bc9] +module.vr1_dc0_planes.libvirt_network.plane["provider-public"]: Refreshing state... [id=fcd1c106-1f36-4558-a133-681009962f3b] +module.vr1_dc0_planes.libvirt_network.plane["storage"]: Refreshing state... [id=1a8bd1c4-4f9e-4272-8af4-31f2be77ed9a] +module.vr1_dc0_planes.libvirt_network.plane["replication"]: Refreshing state... [id=6c7f8727-5877-4556-a41d-d44e5535e046] +module.vr1_dc0_planes.libvirt_network.plane["data-tenant"]: Refreshing state... [id=dd75dac8-b51a-4fe0-98d4-2b6e34d7ed4a] +module.vr1_dc0_planes.libvirt_network.plane["metal-admin"]: Refreshing state... [id=4455b805-5fc7-4d25-bd78-2e80d3f472ed] +module.vr1_dc0_opnsense.libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-opnsense-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-compute-01-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-control-01-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-control-02-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-control-03-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-compute-02-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-01-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-04-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-03-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_volume.disk: Refreshing state... [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-02-disk.qcow2] +module.vr1_dc0_opnsense.libvirt_domain.vm: Refreshing state... [name=vr1-dc0-opnsense] +module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-compute-02] +module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-compute-01] +module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-storage-04] +module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-storage-01] +module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-storage-03] +module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-storage-02] +module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-control-02] +module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-control-01] +module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_domain.node: Refreshing state... [name=vr1-dc0-control-03] + +OpenTofu used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + ~ update in-place (current -> planned) + +OpenTofu will perform the following actions: + + # module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:1b:19:e6" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:8c:2a:8c" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:9c:7f:7a" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:33:92:4e" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:40:62:bb" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:b7:d1:a2" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 25 + name = "vr1-dc0-compute-01" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:18:ab:b4" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:50:48:88" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:78:fb:c5" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:cc:84:61" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:e4:ab:df" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:5b:93:c4" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 22 + name = "vr1-dc0-compute-02" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:be:69:c5" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:29:e5:2b" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:05:98:c4" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:96:e8:36" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:5a:dc:91" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:35:cc:01" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 9 + name = "vr1-dc0-control-01" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:02:ff:57" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:b0:17:2a" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:58:17:23" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:1c:ab:44" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:a6:f8:0b" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:2e:09:d8" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 24 + name = "vr1-dc0-control-02" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:4f:de:a9" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:0b:c2:1b" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:c0:d3:e6" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:10:0f:ea" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:53:19:ef" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:86:78:ed" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 23 + name = "vr1-dc0-control-03" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:5f:8d:42" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:4c:69:7b" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:ac:2c:4d" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:28:91:d2" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:fa:7c:53" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:0c:7a:ab" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 7 + name = "vr1-dc0-storage-01" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:48:86:2c" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:2e:8b:55" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:d7:4e:38" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:05:60:af" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:c6:02:bd" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:2c:f8:42" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 21 + name = "vr1-dc0-storage-02" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:b1:94:d1" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:bd:d1:24" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:80:67:87" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:0f:a6:35" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:e0:73:ed" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:62:65:77" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 6 + name = "vr1-dc0-storage-03" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + + # module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_domain.node will be updated in-place + ~ resource "libvirt_domain" "node" { + ~ devices = { + ~ interfaces = [ + ~ { + + mac = { + + address = "52:54:00:2b:ed:ab" + } + # (3 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:90:ca:d0" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:2a:ac:14" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:af:34:fd" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:28:b1:6d" + } + # (2 unchanged attributes hidden) + }, + ~ { + + mac = { + + address = "52:54:00:ab:5c:50" + } + # (2 unchanged attributes hidden) + }, + ] + # (2 unchanged attributes hidden) + } + id = 26 + name = "vr1-dc0-storage-04" + ~ running = false -> true + # (9 unchanged attributes hidden) + } + +Plan: 0 to add, 9 to change, 0 to destroy. + +───────────────────────────────────────────────────────────────────────────── + +Note: You didn't use the -out option to save this plan, so OpenTofu can't +guarantee to take exactly these actions if you run "tofu apply" now. diff --git a/docs/changelog-20260721-close-and-delivery.md b/docs/changelog-20260721-close-and-delivery.md index 863fc61..d3b9647 100644 --- a/docs/changelog-20260721-close-and-delivery.md +++ b/docs/changelog-20260721-close-and-delivery.md @@ -44,3 +44,20 @@ STOP (present to operator, do not apply). - **Revert:** git revert of this delivery commit (module + root + harness together); live domains unaffected until the gated apply. + +## 3. node-vm power-ownership guard (operator-ruled this session) + +- The MAC-pin verification plan on voffice1 (captured: + docs/audit/inner-plan-20260721-macpin.txt -- 0/9/0, 54 mac adoptions, + ZERO replaces) exposed an entanglement: the module hardcodes + `running = true` while MAAS holds all 9 Ready nodes OFF, so the apply + would ALSO have powered on all 9 nodes outside MAAS's control. +- RULING (this session): question = how to proceed with the entangled + apply; operator selection = "ignore_changes on running first + (Recommended)". +- Shipped: `lifecycle { ignore_changes = [running] }` on the node domain + (create still boots for PXE enlistment; after that MAAS owns power -- + virsh here, IPMI on Roosevelt). Harness grew 3 cases (T13-T15: guard + present, exactly one ignore_changes, cites MAAS). 15 cases green. +- **Revert:** remove the lifecycle block + T13-T15; the 9 power-on + changes return to the next plan. diff --git a/opentofu/modules/node-vm/main.tf b/opentofu/modules/node-vm/main.tf index abe6b30..566538c 100644 --- a/opentofu/modules/node-vm/main.tf +++ b/opentofu/modules/node-vm/main.tf @@ -45,6 +45,19 @@ resource "libvirt_domain" "node" { name = var.vm_name + + # POWER STATE BELONGS TO MAAS, NOT OPENTOFU (operator-ruled 2026-07-21). + # `running = true` below is honored at CREATE (wanted: a new node's first + # boot is its PXE enlistment) and ignored forever after. Without this + # guard, any later apply re-asserts running=true against nodes MAAS has + # powered OFF -- measured 2026-07-21: MAAS held all 9 Ready nodes off + # while the config wanted them on, so the MAC-pin adoption plan carried 9 + # out-of-band power-ons (docs/audit/inner-plan-20260721-macpin.txt). + # Same ownership statement as autostart=false (D-127): MAAS drives node + # power (virsh here, IPMI on Roosevelt metal); tofu owns the definition. + lifecycle { + ignore_changes = [running] + } # memory_unit is REQUIRED -- see the opnsense-edge module for the full write-up. On # dmacvicar/libvirt >=0.9, bare `memory` is interpreted in libvirt's default unit (KiB), # NOT MiB, so omitting this gives the guest 1024x too little RAM and it triple-faults diff --git a/tests/node-vm/run-tests.sh b/tests/node-vm/run-tests.sh index 403a18b..9512348 100755 --- a/tests/node-vm/run-tests.sh +++ b/tests/node-vm/run-tests.sh @@ -66,6 +66,21 @@ no "T8-T11 inner root main.tf present" fi +# T13-T15: power-ownership guard (operator-ruled 2026-07-21) -- the domain +# block must ignore_changes on running (MAAS owns node power; an apply must +# never flip it), must NOT ignore anything else there, and must cite MAAS. +DOM_BLOCK="$(awk '/^resource "libvirt_domain" "node" \{/{f=1} f{print; n+=gsub(/\{/,"{"); n-=gsub(/\}/,"}"); if(n==0 && f)exit}' "$MOD/main.tf")" +grep -q 'ignore_changes = \[running\]' <<<"$DOM_BLOCK" \ + && ok "T13 domain ignores running drift (MAAS owns power)" \ + || no "T13 domain ignores running drift (MAAS owns power)" +IGNORE_COUNT="$(grep -c 'ignore_changes' <<<"$DOM_BLOCK")" +[ "$IGNORE_COUNT" -eq 1 ] \ + && ok "T14 exactly one ignore_changes in the domain block (no scope creep)" \ + || no "T14 exactly one ignore_changes in the domain block (found $IGNORE_COUNT)" +grep -q 'MAAS' <<<"$DOM_BLOCK" \ + && ok "T15 guard cites MAAS power ownership" \ + || no "T15 guard cites MAAS power ownership" + # T12: module still validates when tofu is available (init -backend=false is # offline once the provider is in the plugin cache; skip cleanly otherwise) if command -v tofu >/dev/null 2>&1; then