diff --git a/docs/changelog-20260721-close-and-delivery.md b/docs/changelog-20260721-close-and-delivery.md new file mode 100644 index 0000000..863fc61 --- /dev/null +++ b/docs/changelog-20260721-close-and-delivery.md @@ -0,0 +1,46 @@ +# Session changelog 2026-07-21 (successor session) -- disconnect recovery + queued delivery + +One changelog per session (GA-R2/D1). Status lives ONLY in +docs/CURRENT-STATE.md; this file is narrative + reverts. Predecessor +session (ops-commissioning-diag, disconnected before its bookend): +docs/changelog-20260721-commissioning-diag.md. + +## 1. Predecessor session close LANDED (commit 8b3adb3) + +- The ops-commissioning-diag session disconnected before committing its + deliverable. This session re-verified read-only first (9/9 nodes Ready + in MAAS with virsh power; dc0-node-dns active+enabled with the 10.12.8.3 + alias up; dhcpd running on BOTH controllers; diag-serial tag deleted and + serial-console re-attached to hardy-dove = Phase-7 cleanup confirmed + done), then committed the predecessor's 7 files verbatim plus the + bounded session-ledger close entry and a machine-derived block re-seed + from the 2026-07-21 scan (D next-free 132, DOCFIX 197, BUNDLEFIX 052, + 10 open SEC rows). Operator-approved this session. +- **Revert:** git revert 8b3adb3 (returns the record to the + not-yet-durable state; the live cloud is untouched by that commit). + +## 2. MAC pinning in modules/node-vm (queued item 9a of the predecessor) + +- modules/node-vm: new OPTIONAL `interface_macs` variable (list(string), + default [] = unpinned for pre-enlistment use; validations: all-or- + nothing length vs network_names -- partial pinning is rejected -- and + 6-byte hex format). Interfaces comprehension consumes it as + `mac = { address = ... }` -- shape VERIFIED against the provider schema + (dmacvicar/libvirt 0.9.8, `tofu providers schema -json`, this session). +- opentofu/vr1-dc0-substrate: all 9 nodes pinned. 54 MACs MEASURED this + session (`virsh domiflist` on vvr1-dc0, read-only; order = the + vr1_dc0_node_nics plane order) and the 9 metal-admin entries + cross-checked against MAAS boot-interface records 9/9 (machines read). + No value inferred; no value reused from the adjudication file without + re-measurement. +- NEW harness tests/node-vm/run-tests.sh (12 cases: variable shape, + validations, consumption, 9 lists / 54 literals / 0 duplicates in the + inner root, pass-through, tofu validate). Gauntlet 72 -> 73 ALL GREEN; + repo-lint 0 fail; opentofu-validate PASS (both EXTRA roots). +- NOT YET APPLIED: the inner root's state (on voffice1) does not carry + the pins until an operator-gated `tofu apply` there. The plan gate for + that apply: expected NO REPLACE -- pins equal live values, so the plan + must show only benign in-place mac adoption or no-op; ANY replace is a + STOP (present to operator, do not apply). +- **Revert:** git revert of this delivery commit (module + root + harness + together); live domains unaffected until the gated apply. diff --git a/opentofu/modules/node-vm/main.tf b/opentofu/modules/node-vm/main.tf index c4cbf0c..abe6b30 100644 --- a/opentofu/modules/node-vm/main.tf +++ b/opentofu/modules/node-vm/main.tf @@ -117,6 +117,12 @@ model = { type = "virtio" } + # MAC pinning (see variables.tf interface_macs): an unpinned MAC is + # provider-owned and an "in-place" apply can regenerate it, stranding + # the node in MAAS (measured 2026-07-20/21). `mac = { address = ... }` + # verified against the provider schema (dmacvicar/libvirt 0.9.8, + # `tofu providers schema -json`, 2026-07-21). + mac = length(var.interface_macs) > 0 ? { address = var.interface_macs[i] } : null source = { network = { network = net_name diff --git a/opentofu/modules/node-vm/variables.tf b/opentofu/modules/node-vm/variables.tf index ee58e5f..6d7e2d6 100644 --- a/opentofu/modules/node-vm/variables.tf +++ b/opentofu/modules/node-vm/variables.tf @@ -33,6 +33,36 @@ type = list(string) } +variable "interface_macs" { + description = <<-EOT + Optional ordered list of MAC addresses, one per network_names entry (same + order), PINNING each NIC's MAC in config. Empty (the default) lets libvirt + generate MACs -- acceptable ONLY before a node is enlisted anywhere. + + WHY PINNING EXISTS: measured 2026-07-20/21 -- a tofu/libvirt "in-place" + domain update (plan 0/N/0, no replace) can silently REGENERATE every + unpinned NIC MAC. MAAS keys enlisted machines by boot-interface MAC, so + the drift strands the whole fleet: every PXE boot becomes an unknown node + and commissioning times out with NO "Performing PXE boot" event (the + 2026-07-21 commissioning incident; adjudication + docs/audit/commissioning-diag-20260721.txt). Once nodes are enlisted, pin + their MACs to the live measured values so no future apply can drift them. + VR-only trap: Roosevelt metal has fixed NIC MACs. + EOT + type = list(string) + default = [] + + validation { + condition = length(var.interface_macs) == 0 || length(var.interface_macs) == length(var.network_names) + error_message = "interface_macs must be empty or exactly one MAC per network_names entry (partial pinning would silently leave some NICs drift-prone)." + } + + validation { + condition = alltrue([for m in var.interface_macs : can(regex("^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$", m))]) + error_message = "Each interface_macs entry must be a colon-separated 6-byte hex MAC (e.g. 52:54:00:ab:cd:ef)." + } +} + variable "autostart" { description = "Start this domain automatically when the libvirt host boots (D-127 boot matrix). MANUAL (false) for DC containment + node VMs; true for foundational service VMs + edges. Set explicitly per instance in the root -- do not rely on the default." type = bool diff --git a/opentofu/vr1-dc0-substrate/main.tf b/opentofu/vr1-dc0-substrate/main.tf index 1c805e6..a63ffda 100644 --- a/opentofu/vr1-dc0-substrate/main.tf +++ b/opentofu/vr1-dc0-substrate/main.tf @@ -84,16 +84,52 @@ module.vr1_dc0_planes.network_names["replication"], ] + # macs: MEASURED 2026-07-21 (`virsh domiflist` on vvr1-dc0, all 9 domains; + # session changelog 2026-07-21 delivery). Order = vr1_dc0_node_nics order + # (metal-admin, provider-public, metal-internal, data-tenant, storage, + # replication). PINNED because an in-place apply regenerated all 9 unpinned + # boot MACs on 2026-07-20 and stranded the fleet in MAAS (adjudication: + # docs/audit/commissioning-diag-20260721.txt). The metal-admin entries were + # cross-checked against MAAS boot-interface records 9/9 the same day. These + # are identity values, not tunables -- change one and MAAS no longer knows + # the node. vr1_dc0_nodes = { - "vr1-dc0-control-01" = { vcpu = 16, mem = 65536, disk_gib = 150 } - "vr1-dc0-control-02" = { vcpu = 16, mem = 65536, disk_gib = 150 } - "vr1-dc0-control-03" = { vcpu = 16, mem = 65536, disk_gib = 150 } - "vr1-dc0-compute-01" = { vcpu = 12, mem = 49152, disk_gib = 100 } - "vr1-dc0-compute-02" = { vcpu = 12, mem = 49152, disk_gib = 100 } - "vr1-dc0-storage-01" = { vcpu = 8, mem = 24576, disk_gib = 550 } - "vr1-dc0-storage-02" = { vcpu = 8, mem = 24576, disk_gib = 550 } - "vr1-dc0-storage-03" = { vcpu = 8, mem = 24576, disk_gib = 550 } - "vr1-dc0-storage-04" = { vcpu = 8, mem = 24576, disk_gib = 550 } # R-3: 4th OSD host + "vr1-dc0-control-01" = { vcpu = 16, mem = 65536, disk_gib = 150, macs = [ + "52:54:00:be:69:c5", "52:54:00:29:e5:2b", "52:54:00:05:98:c4", + "52:54:00:96:e8:36", "52:54:00:5a:dc:91", "52:54:00:35:cc:01", + ] } + "vr1-dc0-control-02" = { vcpu = 16, mem = 65536, disk_gib = 150, macs = [ + "52:54:00:02:ff:57", "52:54:00:b0:17:2a", "52:54:00:58:17:23", + "52:54:00:1c:ab:44", "52:54:00:a6:f8:0b", "52:54:00:2e:09:d8", + ] } + "vr1-dc0-control-03" = { vcpu = 16, mem = 65536, disk_gib = 150, macs = [ + "52:54:00:4f:de:a9", "52:54:00:0b:c2:1b", "52:54:00:c0:d3:e6", + "52:54:00:10:0f:ea", "52:54:00:53:19:ef", "52:54:00:86:78:ed", + ] } + "vr1-dc0-compute-01" = { vcpu = 12, mem = 49152, disk_gib = 100, macs = [ + "52:54:00:1b:19:e6", "52:54:00:8c:2a:8c", "52:54:00:9c:7f:7a", + "52:54:00:33:92:4e", "52:54:00:40:62:bb", "52:54:00:b7:d1:a2", + ] } + "vr1-dc0-compute-02" = { vcpu = 12, mem = 49152, disk_gib = 100, macs = [ + "52:54:00:18:ab:b4", "52:54:00:50:48:88", "52:54:00:78:fb:c5", + "52:54:00:cc:84:61", "52:54:00:e4:ab:df", "52:54:00:5b:93:c4", + ] } + "vr1-dc0-storage-01" = { vcpu = 8, mem = 24576, disk_gib = 550, macs = [ + "52:54:00:5f:8d:42", "52:54:00:4c:69:7b", "52:54:00:ac:2c:4d", + "52:54:00:28:91:d2", "52:54:00:fa:7c:53", "52:54:00:0c:7a:ab", + ] } + "vr1-dc0-storage-02" = { vcpu = 8, mem = 24576, disk_gib = 550, macs = [ + "52:54:00:48:86:2c", "52:54:00:2e:8b:55", "52:54:00:d7:4e:38", + "52:54:00:05:60:af", "52:54:00:c6:02:bd", "52:54:00:2c:f8:42", + ] } + "vr1-dc0-storage-03" = { vcpu = 8, mem = 24576, disk_gib = 550, macs = [ + "52:54:00:b1:94:d1", "52:54:00:bd:d1:24", "52:54:00:80:67:87", + "52:54:00:0f:a6:35", "52:54:00:e0:73:ed", "52:54:00:62:65:77", + ] } + "vr1-dc0-storage-04" = { vcpu = 8, mem = 24576, disk_gib = 550, macs = [ # R-3: 4th OSD host + "52:54:00:2b:ed:ab", "52:54:00:90:ca:d0", "52:54:00:2a:ac:14", + "52:54:00:af:34:fd", "52:54:00:28:b1:6d", "52:54:00:ab:5c:50", + ] } } } @@ -108,4 +144,5 @@ disk_size_bytes = each.value.disk_gib * 1024 * 1024 * 1024 pool_name = module.inner_storage.pool_name network_names = local.vr1_dc0_node_nics + interface_macs = each.value.macs # pinned -- see the vr1_dc0_nodes comment } diff --git a/tests/node-vm/run-tests.sh b/tests/node-vm/run-tests.sh new file mode 100755 index 0000000..403a18b --- /dev/null +++ b/tests/node-vm/run-tests.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# tests/node-vm/run-tests.sh -- guard for modules/node-vm (MAC pinning). +# Static assertions that the interface MAC pin is present and correctly +# shaped (regression guard: an unpinned node NIC MAC can be silently +# REGENERATED by an in-place apply, stranding the node in MAAS -- the +# 2026-07-21 commissioning incident, docs/audit/commissioning-diag-20260721.txt). +# Also asserts the inner root actually pins all 9 nodes x 6 planes. +# Runs `tofu validate` on the module when the binary is available; skips +# gracefully when it is not. Exit: 0 all pass | 1 any case failed. ASCII + LF. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +MOD="$ROOT/opentofu/modules/node-vm" +INNER="$ROOT/opentofu/vr1-dc0-substrate/main.tf" +PASS=0; FAIL=0 +ok(){ echo " PASS $1"; PASS=$((PASS+1)); } +no(){ echo " FAIL $1"; FAIL=$((FAIL+1)); } + +[ -f "$MOD/main.tf" ] && ok "T1 module main.tf present" || no "T1 module main.tf present" +[ -f "$MOD/variables.tf" ] && ok "T2 module variables.tf present" || no "T2 module variables.tf present" + +# T3: interface_macs variable exists with a safe default (unpinned allowed +# ONLY as an explicit empty default, for pre-enlistment use) +VAR_BLOCK="$(awk '/^variable "interface_macs" \{/{f=1} f{print; n+=gsub(/\{/,"{"); n-=gsub(/\}/,"}"); if(n==0 && f)exit}' "$MOD/variables.tf")" +grep -q 'type *= *list(string)' <<<"$VAR_BLOCK" \ + && ok "T3 interface_macs is list(string)" \ + || no "T3 interface_macs is list(string)" +grep -q 'default *= *\[\]' <<<"$VAR_BLOCK" \ + && ok "T4 interface_macs defaults to empty (opt-in pin)" \ + || no "T4 interface_macs defaults to empty (opt-in pin)" + +# T5/T6: both validations present -- all-or-nothing length, and MAC format +grep -q 'length(var.interface_macs) == length(var.network_names)' <<<"$VAR_BLOCK" \ + && ok "T5 validation: all-or-nothing (no partial pinning)" \ + || no "T5 validation: all-or-nothing (no partial pinning)" +grep -q '0-9a-fA-F' <<<"$VAR_BLOCK" \ + && ok "T6 validation: MAC format regex present" \ + || no "T6 validation: MAC format regex present" + +# T7: the interfaces comprehension actually consumes the pin, guarded on +# emptiness, in the schema-verified shape mac = { address = ... } +IF_LINE="$(grep -n 'mac *= *length(var.interface_macs) > 0 ? { address = var.interface_macs\[i\] } : null' "$MOD/main.tf" || true)" +[ -n "$IF_LINE" ] \ + && ok "T7 interfaces consume the pin (mac = { address = ... } / null)" \ + || no "T7 interfaces consume the pin (mac = { address = ... } / null)" + +# T8: the inner root pins EVERY node: 9 macs lists, 54 MAC literals total, +# and the module call passes interface_macs through. +if [ -f "$INNER" ]; then + MACS_LISTS="$(grep -c 'macs = \[' "$INNER")" + MAC_LITERALS="$(grep -oE '"([0-9a-f]{2}:){5}[0-9a-f]{2}"' "$INNER" | wc -l)" + [ "$MACS_LISTS" -eq 9 ] \ + && ok "T8 inner root: 9 per-node macs lists" \ + || no "T8 inner root: 9 per-node macs lists (found $MACS_LISTS)" + [ "$MAC_LITERALS" -eq 54 ] \ + && ok "T9 inner root: 54 pinned MAC literals (9 nodes x 6 planes)" \ + || no "T9 inner root: 54 pinned MAC literals (found $MAC_LITERALS)" + DUPES="$(grep -oE '"([0-9a-f]{2}:){5}[0-9a-f]{2}"' "$INNER" | sort | uniq -d | wc -l)" + [ "$DUPES" -eq 0 ] \ + && ok "T10 inner root: no duplicate MACs" \ + || no "T10 inner root: no duplicate MACs ($DUPES duplicated)" + grep -q 'interface_macs *= *each.value.macs' "$INNER" \ + && ok "T11 inner root: module call passes interface_macs" \ + || no "T11 inner root: module call passes interface_macs" +else + no "T8-T11 inner root main.tf present" +fi + +# T12: module still validates when tofu is available (init -backend=false is +# offline once the provider is in the plugin cache; skip cleanly otherwise) +if command -v tofu >/dev/null 2>&1; then + TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + cp "$MOD"/*.tf "$TMP"/ + if (cd "$TMP" && tofu init -backend=false -input=false >/dev/null 2>&1 \ + && tofu validate >/dev/null 2>&1); then + ok "T12 module validates (tofu validate)" + else + no "T12 module validates (tofu validate)" + fi +else + echo " SKIP T12 tofu binary not available (static checks above still bind)" +fi + +echo; echo "node-vm: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]