diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index c20cd11..d2ca92c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -38,11 +38,19 @@ KVM + inner pool), SEC-010 applied+verified BOTH transit ends (row CLOSED), OPNsense **26.7** nano base staged (operator ruling; step-C boot REVALIDATES the D-112/D-113 path on 26.7). Named gate check EXIT 0: - `docs/audit/stepB-check-20260720-final.txt`. ACTIVE gate: G10 remaining - items (steps C-E: inner apply + depth-4 boot, D-125 egress isolation - test, MAAS vm-host + `TF_VAR_maas_api_key`, netem). Next mutation: - step C inner apply FROM voffice1 (qemu+ssh, R-5); region-side - metal-admin DHCP (primary_rack) is step-D territory. + `docs/audit/stepB-check-20260720-final.txt`. **Deploy step C (inner + apply) COMPLETE 2026-07-20**: executed FROM voffice1 (D-128 Plane 2 -- + tofu 1.12.4 + repo clone + dc0 key staged there), 28/28 resources, + inner plan CONVERGED zero diff + (`docs/audit/inner-converge-20260720-stepC.txt`); 10/10 domains RUNNING + inside vvr1-dc0 (9 nodes + edge); edge = fresh 26.7 nano, serial log at + the FreeBSD login prompt (D-112 boot path first-datapoint PASS on 26.7). + The INNER tfstate lives ON voffice1 (vr1-dc0-substrate/terraform.tfstate + -- new state-of-record location; add to the site backup set). ACTIVE + gate: G10 remaining -- edge D-112(c) console bootstrap + D-113 API + config + D-129 plugins (26.7 revalidation), D-125 throwaway-guest + egress isolation test, MAAS reach + `TF_VAR_maas_api_key` + region-side + metal-admin DHCP naming this rack primary_rack (step D), netem (E). - The grounding audit is COMPLETE and EXITED (2026-07-19): Phases 1-6 all closed (charter `148dcef`; rulings `docs/audit/ga-rulings.md`; the Phase-5 sweep ran as six operator-gated batches in one session; exit diff --git a/docs/audit/inner-apply-20260720-stepC-run2.txt b/docs/audit/inner-apply-20260720-stepC-run2.txt new file mode 100644 index 0000000..677f9ad --- /dev/null +++ b/docs/audit/inner-apply-20260720-stepC-run2.txt @@ -0,0 +1,13 @@ +module.vr1_dc0_opnsense.libvirt_volume.disk: Creating... +module.vr1_dc0_wan.libvirt_network.wan_bridge: Creating... +module.vr1_dc0_wan.libvirt_network.wan_bridge: Creation complete after 0s [id=fabdac49-5daf-4cd6-aa0a-40d43ace9b1d] + +Error: Failed to Open URL + + with module.vr1_dc0_opnsense.libvirt_volume.disk, + on ../modules/opnsense-edge/main.tf line 55, in resource "libvirt_volume" "disk": + 55: resource "libvirt_volume" "disk" { + +Could not open URL for upload: failed to stat file: stat +/var/lib/libvirt/vr1-dc0-inner/opnsense-26.7-nano.qcow2: no such file or +directory diff --git a/docs/audit/inner-apply-20260720-stepC-run3.txt b/docs/audit/inner-apply-20260720-stepC-run3.txt new file mode 100644 index 0000000..311812d --- /dev/null +++ b/docs/audit/inner-apply-20260720-stepC-run3.txt @@ -0,0 +1,13 @@ +module.vr1_dc0_opnsense.libvirt_volume.disk: Creating... +module.vr1_dc0_opnsense.libvirt_volume.disk: Creation complete after 4s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-opnsense-disk.qcow2] +module.vr1_dc0_opnsense.libvirt_domain.vm: Creating... + +Error: Failed to Start Domain + + with module.vr1_dc0_opnsense.libvirt_domain.vm, + on ../modules/opnsense-edge/main.tf line 87, in resource "libvirt_domain" "vm": + 87: resource "libvirt_domain" "vm" { + +Domain was defined but failed to start: Unable to open file: +/var/lib/libvirt/vr1/staging/vr1-dc0-opnsense-serial.log: No such file or +directory diff --git a/docs/audit/inner-apply-20260720-stepC-run4.txt b/docs/audit/inner-apply-20260720-stepC-run4.txt new file mode 100644 index 0000000..8e8ed6b --- /dev/null +++ b/docs/audit/inner-apply-20260720-stepC-run4.txt @@ -0,0 +1,4 @@ +module.vr1_dc0_opnsense.libvirt_domain.vm: Creating... +module.vr1_dc0_opnsense.libvirt_domain.vm: Creation complete after 1s [name=vr1-dc0-opnsense] + +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. diff --git a/docs/audit/inner-apply-20260720-stepC.txt b/docs/audit/inner-apply-20260720-stepC.txt new file mode 100644 index 0000000..6313036 --- /dev/null +++ b/docs/audit/inner-apply-20260720-stepC.txt @@ -0,0 +1,71 @@ +module.vr1_dc0_planes.libvirt_network.plane["replication"]: Creating... +module.inner_storage.libvirt_pool.dc: Creating... +module.vr1_dc0_planes.libvirt_network.plane["metal-admin"]: Creating... +module.vr1_dc0_planes.libvirt_network.plane["storage"]: Creating... +module.vr1_dc0_planes.libvirt_network.plane["data-tenant"]: Creating... +module.vr1_dc0_planes.libvirt_network.plane["metal-internal"]: Creating... +module.vr1_dc0_planes.libvirt_network.plane["provider-public"]: Creating... +module.vr1_dc0_wan.libvirt_network.wan_bridge: Creating... +module.vr1_dc0_planes.libvirt_network.plane["replication"]: Creation complete after 0s [id=6c7f8727-5877-4556-a41d-d44e5535e046] +module.inner_storage.libvirt_pool.dc: Creation complete after 0s [id=62bb75eb-b7b9-4659-bca3-f75825ae2cdf] +module.vr1_dc0_opnsense.libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_volume.disk: Creating... +module.vr1_dc0_planes.libvirt_network.plane["metal-admin"]: Creation complete after 0s [id=4455b805-5fc7-4d25-bd78-2e80d3f472ed] +module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-02-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-control-02-disk.qcow2] +module.vr1_dc0_planes.libvirt_network.plane["provider-public"]: Creation complete after 0s [id=fcd1c106-1f36-4558-a133-681009962f3b] +module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_volume.disk: Creating... +module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-control-03-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-compute-02-disk.qcow2] +module.vr1_dc0_planes.libvirt_network.plane["storage"]: Creation complete after 0s [id=1a8bd1c4-4f9e-4272-8af4-31f2be77ed9a] +module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-control-01-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-compute-01-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-04-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-03-disk.qcow2] +module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_volume.disk: Creation complete after 0s [id=/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-storage-01-disk.qcow2] +module.vr1_dc0_planes.libvirt_network.plane["metal-internal"]: Creation complete after 0s [id=edbc1aa5-5ac0-46e1-b46c-51f6bdc27bc9] +module.vr1_dc0_planes.libvirt_network.plane["data-tenant"]: Creation complete after 0s [id=dd75dac8-b51a-4fe0-98d4-2b6e34d7ed4a] +module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_domain.node: Creating... +module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_domain.node: Creation complete after 6s [name=vr1-dc0-compute-02] +module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_domain.node: Creation complete after 6s [name=vr1-dc0-control-03] +module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_domain.node: Creation complete after 6s [name=vr1-dc0-control-02] +module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_domain.node: Creation complete after 6s [name=vr1-dc0-compute-01] +module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_domain.node: Creation complete after 6s [name=vr1-dc0-storage-02] +module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_domain.node: Creation complete after 9s [name=vr1-dc0-storage-04] +module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_domain.node: Creation complete after 9s [name=vr1-dc0-storage-01] +module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_domain.node: Creation complete after 9s [name=vr1-dc0-storage-03] +module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_domain.node: Creation complete after 9s [name=vr1-dc0-control-01] + +Error: Failed to Open URL + + with module.vr1_dc0_opnsense.libvirt_volume.disk, + on ../modules/opnsense-edge/main.tf line 55, in resource "libvirt_volume" "disk": + 55: resource "libvirt_volume" "disk" { + +Could not open URL for upload: failed to stat file: stat +/var/lib/libvirt/vr1-dc0-inner/opnsense-26.7-nano.qcow2: no such file or +directory + +Error: Network Creation Failed + + with module.vr1_dc0_wan.libvirt_network.wan_bridge, + on ../modules/wan-bridge/main.tf line 14, in resource "libvirt_network" "wan_bridge": + 14: resource "libvirt_network" "wan_bridge" { + +Failed to define network: XML error: mtu size only allowed in open, route, +nat, and isolated mode, not in bridge (network 'vr1-dc0-wan') diff --git a/docs/audit/inner-converge-20260720-stepC.txt b/docs/audit/inner-converge-20260720-stepC.txt new file mode 100644 index 0000000..7d1419e --- /dev/null +++ b/docs/audit/inner-converge-20260720-stepC.txt Binary files differ diff --git a/docs/audit/inner-plan-20260720-stepC.txt b/docs/audit/inner-plan-20260720-stepC.txt new file mode 100644 index 0000000..2a81181 --- /dev/null +++ b/docs/audit/inner-plan-20260720-stepC.txt @@ -0,0 +1,1406 @@ + +OpenTofu used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + + create + +OpenTofu will perform the following actions: + + # module.inner_storage.libvirt_pool.dc will be created + + resource "libvirt_pool" "dc" { + + allocation = (known after apply) + + available = (known after apply) + + capacity = (known after apply) + + id = (known after apply) + + name = "vr1-dc0-inner-pool" + + target = { + + path = "/var/lib/libvirt/vr1-dc0-inner" + } + + type = "dir" + + uuid = (known after apply) + } + + # module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-compute-01-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 49152 + + memory_unit = "MiB" + + name = "vr1-dc0-compute-01" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 12 + } + + # module.vr1_dc0_node["vr1-dc0-compute-01"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 107374182400 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-compute-01-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-compute-02-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 49152 + + memory_unit = "MiB" + + name = "vr1-dc0-compute-02" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 12 + } + + # module.vr1_dc0_node["vr1-dc0-compute-02"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 107374182400 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-compute-02-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-control-01-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 65536 + + memory_unit = "MiB" + + name = "vr1-dc0-control-01" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 16 + } + + # module.vr1_dc0_node["vr1-dc0-control-01"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 161061273600 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-control-01-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-control-02-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 65536 + + memory_unit = "MiB" + + name = "vr1-dc0-control-02" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 16 + } + + # module.vr1_dc0_node["vr1-dc0-control-02"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 161061273600 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-control-02-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-control-03-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 65536 + + memory_unit = "MiB" + + name = "vr1-dc0-control-03" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 16 + } + + # module.vr1_dc0_node["vr1-dc0-control-03"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 161061273600 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-control-03-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-storage-01-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 24576 + + memory_unit = "MiB" + + name = "vr1-dc0-storage-01" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 8 + } + + # module.vr1_dc0_node["vr1-dc0-storage-01"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 590558003200 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-storage-01-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-storage-02-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 24576 + + memory_unit = "MiB" + + name = "vr1-dc0-storage-02" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 8 + } + + # module.vr1_dc0_node["vr1-dc0-storage-02"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 590558003200 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-storage-02-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-storage-03-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 24576 + + memory_unit = "MiB" + + name = "vr1-dc0-storage-03" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 8 + } + + # module.vr1_dc0_node["vr1-dc0-storage-03"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 590558003200 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-storage-03-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_domain.node will be created + + resource "libvirt_domain" "node" { + + autostart = false + + cpu = { + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + boot = { + + order = 2 + } + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-storage-04-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + boot = { + + order = 1 + } + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-admin" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-metal-internal" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-data-tenant" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-storage" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-replication" + } + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 24576 + + memory_unit = "MiB" + + name = "vr1-dc0-storage-04" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "q35" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 8 + } + + # module.vr1_dc0_node["vr1-dc0-storage-04"].libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = 590558003200 + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-storage-04-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_opnsense.libvirt_domain.vm will be created + + resource "libvirt_domain" "vm" { + + autostart = true + + cpu = { + + features = [ + + { + + name = "svm" + + policy = "disable" + }, + ] + + mode = "host-passthrough" + } + + devices = { + + disks = [ + + { + + driver = { + + type = "qcow2" + } + + source = { + + volume = { + + pool = "vr1-dc0-inner-pool" + + volume = "vr1-dc0-opnsense-disk.qcow2" + } + } + + target = { + + bus = "virtio" + + dev = "vda" + } + }, + ] + + interfaces = [ + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-provider-public" + } + } + }, + + { + + model = { + + type = "virtio" + } + + source = { + + network = { + + network = "vr1-dc0-wan" + } + } + }, + ] + + serials = [ + + { + + log = { + + append = "on" + + file = "/var/lib/libvirt/vr1/staging/vr1-dc0-opnsense-serial.log" + } + + target = { + + port = 0 + } + }, + ] + } + + features = { + + acpi = true + + apic = {} + } + + id = (known after apply) + + memory = 2048 + + memory_unit = "MiB" + + name = "vr1-dc0-opnsense" + + os = { + + type = "hvm" + + type_arch = "x86_64" + + type_machine = "pc" + } + + running = true + + type = "kvm" + + uuid = (known after apply) + + vcpu = 2 + } + + # module.vr1_dc0_opnsense.libvirt_volume.disk will be created + + resource "libvirt_volume" "disk" { + + allocation = (known after apply) + + capacity = (known after apply) + + create = { + + content = { + + url = "/var/lib/libvirt/vr1-dc0-inner/opnsense-26.7-nano.qcow2" + } + } + + id = (known after apply) + + key = (known after apply) + + name = "vr1-dc0-opnsense-disk.qcow2" + + path = (known after apply) + + physical = (known after apply) + + pool = "vr1-dc0-inner-pool" + + target = { + + format = { + + type = "qcow2" + } + + path = (known after apply) + } + } + + # module.vr1_dc0_planes.libvirt_network.plane["data-tenant"] will be created + + resource "libvirt_network" "plane" { + + autostart = true + + domain = { + + name = "data-tenant.vr1-dc0.cloud.neumatrix.local" + } + + id = (known after apply) + + mtu = { + + size = 9000 + } + + name = "vr1-dc0-data-tenant" + + uuid = (known after apply) + } + + # module.vr1_dc0_planes.libvirt_network.plane["metal-admin"] will be created + + resource "libvirt_network" "plane" { + + autostart = true + + domain = { + + name = "metal-admin.vr1-dc0.cloud.neumatrix.local" + } + + id = (known after apply) + + mtu = { + + size = 9000 + } + + name = "vr1-dc0-metal-admin" + + uuid = (known after apply) + } + + # module.vr1_dc0_planes.libvirt_network.plane["metal-internal"] will be created + + resource "libvirt_network" "plane" { + + autostart = true + + domain = { + + name = "metal-internal.vr1-dc0.cloud.neumatrix.local" + } + + id = (known after apply) + + mtu = { + + size = 9000 + } + + name = "vr1-dc0-metal-internal" + + uuid = (known after apply) + } + + # module.vr1_dc0_planes.libvirt_network.plane["provider-public"] will be created + + resource "libvirt_network" "plane" { + + autostart = true + + domain = { + + name = "provider-public.vr1-dc0.cloud.neumatrix.local" + } + + id = (known after apply) + + mtu = { + + size = 9000 + } + + name = "vr1-dc0-provider-public" + + uuid = (known after apply) + } + + # module.vr1_dc0_planes.libvirt_network.plane["replication"] will be created + + resource "libvirt_network" "plane" { + + autostart = true + + domain = { + + name = "replication.vr1-dc0.cloud.neumatrix.local" + } + + id = (known after apply) + + mtu = { + + size = 9000 + } + + name = "vr1-dc0-replication" + + uuid = (known after apply) + } + + # module.vr1_dc0_planes.libvirt_network.plane["storage"] will be created + + resource "libvirt_network" "plane" { + + autostart = true + + domain = { + + name = "storage.vr1-dc0.cloud.neumatrix.local" + } + + id = (known after apply) + + mtu = { + + size = 9000 + } + + name = "vr1-dc0-storage" + + uuid = (known after apply) + } + + # module.vr1_dc0_wan.libvirt_network.wan_bridge will be created + + resource "libvirt_network" "wan_bridge" { + + autostart = true + + bridge = { + + name = "br-vr1-dc0-wan" + } + + forward = { + + mode = "bridge" + } + + id = (known after apply) + + mtu = { + + size = 1500 + } + + name = "vr1-dc0-wan" + + uuid = (known after apply) + } + +Plan: 28 to add, 0 to change, 0 to destroy. + +───────────────────────────────────────────────────────────────────────────── + +Saved the plan to: inner-stepC.tfplan + +To perform exactly these actions, run the following command to apply: + tofu apply "inner-stepC.tfplan" diff --git a/docs/changelog-20260719-dc0-deploy-stepB.md b/docs/changelog-20260719-dc0-deploy-stepB.md index b95fb39..887458e 100644 --- a/docs/changelog-20260719-dc0-deploy-stepB.md +++ b/docs/changelog-20260719-dc0-deploy-stepB.md @@ -184,6 +184,48 @@ - **Revert:** rm the two files + `systemctl disable sec010-fw` on voffice1; `nft delete table inet sec010`; ledger row reopens citing this item. +## 7. Step C: voffice1 becomes the Plane-2 executor; INNER APPLY COMPLETE (28/28) + +- voffice1 setup (gated, logged): tofu 1.12.4 via vcloud's exact apt + source (keyrings byte-copied; version == section-7 pin); repo cloned at + `29cf7bf` (branch); dc0 private key staged to `~/vr1-dc0-creds/` + (0700/0600, piped -- SEC-009 convention now spans hosts); ssh config + entry for 172.31.0.2; known_hosts seeded via ssh-keyscan with the + ED25519 fingerprint cross-checked against the accept-new-trusted entry + (MATCH). +- Inner-root fixes measured at the FIRST REAL dial/apply (each committed): + (i) provider needs `keyfile`+`sshauth=privkey` URI params -- its Go ssh + ignores ~/.ssh/config AND default identities; single-type known_hosts + fails as "key mismatch" (multi-type scan required); + (ii) `modules/wan-bridge`: `` is ILLEGAL in bridge-mode networks -- + removed (MTU belongs to the host bridge; DOCFIX-194 class: parsed, never + applied); + (iii) base-image location premise WRONG under remote provider -- content + UPLOADS from the executing host; 26.7 qcow2 copied rack->voffice1 + (`~/vr1-dc0-images/`), tfvar override + variables.tf description + corrected; + (iv) edge serial-log dir `/var/lib/libvirt/vr1/staging/` is a HARDCODED + vcloud literal in modules/opnsense-edge (:223) -- dir created on the rack + (755); LOGGED findings: parameterize the path; fold staging-dir creation + into site-headend-install --host-nodes; opentofu-validate.sh does not + cover the inner root. +- Applies (captures `docs/audit/inner-{plan,apply}-20260720-stepC*.txt`): + run 1 = 25/28 (nodes+planes+pool; wan-bridge mtu abort); run 2 = wan + bridge (edge disk stat abort); run 3 = edge disk (staging-dir abort; + provider self-cleaned the failed domain -- operator undefine found + nothing); run 4 = edge domain, EXIT 0. CONVERGED: inner plan zero diff + (`docs/audit/inner-converge-20260720-stepC.txt`). +- VERIFIED: 10/10 domains RUNNING inside vvr1-dc0 (9 nodes depth-4 + edge); + edge serial log = FreeBSD login prompt on ttyu0 -- 26.7 boots the D-112 + serial path (first revalidation datapoint; full D-112(c) bootstrap is the + next window). +- NEW STATE OF RECORD: the inner tfstate lives on voffice1 + (`~/openstack-caracal-dc-dc/opentofu/vr1-dc0-substrate/terraform.tfstate`) + -- back it up with the site; follow-item for the backup set. +- **Revert:** `tofu destroy` in the inner root from voffice1 (site-down + alternative: `virsh destroy vvr1-dc0`, D-122); module fixes revert by + commit. + ## 6. ledger-scan hardening: FAIL-OPEN is terminology, not a status - SEC-010's close surfaced a scanner false positive: the OPEN-row detector diff --git a/opentofu/vr1-dc0-substrate/variables.tf b/opentofu/vr1-dc0-substrate/variables.tf index ecf5321..1cec3d7 100644 --- a/opentofu/vr1-dc0-substrate/variables.tf +++ b/opentofu/vr1-dc0-substrate/variables.tf @@ -24,7 +24,7 @@ } variable "opnsense_base_path" { - description = "Path INSIDE vvr1-dc0 to the prepped OPNsense nano qcow2 (staged by the bootstrap step, opnsense-prep-image.sh run on vvr1-dc0)." + description = "Path ON THE EXECUTING HOST (voffice1, D-128 Plane 2) to the prepped OPNsense nano qcow2. CORRECTED 2026-07-20 (measured at the first inner apply): the remote qemu+ssh provider UPLOADS volume content from ITS OWN filesystem -- a path on vvr1-dc0 fails with 'Could not open URL for upload: failed to stat'. The rack-side prep (bootstrap step) still stages the image; copy it to the executing host (or prep there directly) before the inner apply." type = string # 26.7 per operator ruling 2026-07-20 (fresh 26.7 nano base for the DC edge; the # office1 edge's proven 26.1 boot path becomes a REVALIDATION item at the step-C