# Root module: wires the network + storage-pool layer for VR1 (Stage 1-2 of
# docs/dc-dc-deployment-workflow.md). See opentofu/README.md for full scope,
# what is deliberately deferred, and why.
provider "libvirt" {
uri = var.libvirt_uri
}
# ---- The provider "maas" CONFIG block is deliberately DEFERRED to Stage 3
# (DOCFIX-179, decided on the first real `tofu plan`). No module instantiated in
# Stage 1-2 uses the maas provider (modules/maas-vm-host is a Stage-3 module,
# commented out below), yet a configured provider "maas" block here forced
# `tofu plan` to demand maas_api_url AND the SENSITIVE maas_api_key for a plan
# that creates ZERO MAAS resources -- which would bake a fake key into
# plan/state (the DOCFIX-175 plaintext-secret surface). maas stays in
# versions.tf's required_providers (version pinned, lock-file complete); Stage 3
# re-adds this provider block + its two variables when it actually instantiates
# modules/maas-vm-host. modules/netem-link is likewise not instantiated yet
# (needs a real bridge/SSH target + netem params). See gap register items
# 2/4/11 and opentofu/README.md. ----
# ---- DC1: inherits the DC0 six-plane v4 layout unchanged (D-101) ----
# D-123 MODEL B (2026-07-16): the 6 vr1-dc0 planes MOVED to the INNER root
# `opentofu/vr1-dc0-substrate/` -- under Model B the nodes live INSIDE vvr1-dc0, so
# their planes are created on vvr1-dc0's OWN libvirt (via the qemu+ssh inner provider,
# R-5), not on vcloud. A libvirt provider cannot be configured from a resource created
# in the same apply, so the inner substrate is a separate root/state applied AFTER
# vvr1-dc0 is up (the bootstrap gate). See docs/stage3-adversarial-review-20260716.md
# (R3-F01 / Phase C) + docs/model-a-fallback-plan.md.
# NOTE: module.vr1_dc0_planes was applied at vcloud (state serial 21, empty); the outer
# apply will DESTROY those 6 empty networks (gated, verify with `tofu plan`) and the
# inner apply recreates them inside vvr1-dc0.
module "vr1_dc0_storage" {
source = "./modules/dc-storage-pool"
dc_name = "vr1-dc0"
target_path = var.vr1_dc0_pool_path
}
# ---- vr1-dc1 (VR1's SECOND DC): PLANES deferred by SEQUENCING, not a missing
# literal. Its supernet IS assigned -- D-115 moved it to 10.12.64.0/19 (ADOPTED,
# imported to office1-netbox), superseding the old "wait for NetBox" premise. It
# stays out of scope until vr1-dc0 completes (operator ruling 2026-07-10, Option B:
# vr1-dc0-first). When vr1-dc1 is started, add a `vr1_dc1_planes` variable (same
# shape as `vr1_dc0_planes`, CIDRs derived per D-115) and uncomment this block.
# Do NOT uncomment it now. The storage pool + mesh legs are NOT address-dependent,
# so they ARE wired now.
#
# module "vr1_dc1_planes" {
# source = "./modules/dc-planes"
# dc_name = "vr1-dc1"
# domain_suffix = var.domain_suffix
# mtu = var.underlay_mtu
# planes = var.vr1_dc1_planes
# }
module "vr1_dc1_storage" {
source = "./modules/dc-storage-pool"
dc_name = "vr1-dc1"
target_path = var.vr1_dc1_pool_path
}
# ---- Office1 headend: its own storage pool for MAAS-region/NetBox/GitBucket
# service-VM disks (D-103). Office1 does not get a dc-planes module -- the
# six-plane template is per-OpenStack-DC, not per-headend. ----
module "office1_storage" {
source = "./modules/dc-storage-pool"
dc_name = "office1"
target_path = var.office1_pool_path
}
# ---- Office1-local network (closes tooling gap register item #12): the
# shared L2 segment the three Office1 service VMs (MAAS-region, NetBox,
# GitBucket), the vcloud host's own management interface, and Office1's own
# OPNsense edge's LAN side (gap #16) all attach to. See
# modules/office1-network/main.tf's header for the design decision (a new
# module, not a reused host bridge) and opentofu/README.md for the full
# writeup. Instantiated for real here (unlike office1_opnsense below): it
# needs no value beyond domain_suffix/underlay_mtu, both already real,
# required inputs elsewhere in this file -- the same reasoning that already
# lets the three mesh-link legs be instantiated today. ----
module "office1_network" {
source = "./modules/office1-network"
domain_suffix = var.domain_suffix
mtu = var.underlay_mtu
}
# ---- Office1's own OPNsense edge (closes tooling gap register item #16 --
# decision: YES, Office1 gets a fourth modules/opnsense-edge call alongside
# DC1/DC2, and Stage 2's runbook (runbooks/dc-dc-phase1-office1-standup.md)
# owns creating it -- see that runbook's new Step 4b and its "Open questions"
# section for the full reasoning). NOT instantiated below -- commented out on
# purpose, mirroring the identical NOT-YET-REAL status of module
# "vr1_dc0_opnsense"/"vr1_dc1_opnsense" (see runbooks/dc-dc-phase2-tofu-dc-substrate.md
# Step 5's own template, which is ALSO still all placeholders, nothing
# committed here either). Every value below needs to be real before this is
# uncommented -- do not fill in invented specs to wire it in sooner:
#
# Office1's own OPNsense simulated-ISP edge (gap #16 CLOSED). gap #17 resolved
# for Office1 via the `office1-wan` NAT network (created via virsh 2026-07-12 --
# a documented D-103 debt to formalize into an OpenTofu module later, same class
# as the retained `wan`). LAN = the Office1-local segment; WAN = office1-wan (its
# own simulated ISP -> internet). Config is the DOCFIX-185 real-ISP-router config
# (no egress-airgap), rendered to office1-opnsense-config.iso.
module "office1_opnsense" {
source = "./modules/opnsense-edge"
vm_name = "office1-opnsense"
memory_mib = 2048
vcpu = 2
pool_name = module.office1_storage.pool_name
base_volume_path = "/var/lib/libvirt/vr1/office1/opnsense-26.1-nano.qcow2"
# No disk_size_bytes: the disk is a direct copy of the prepped nano, so its size comes
# from scripts/opnsense-prep-image.sh's GROW (default +8G -> 11 GiB). The old
# `disk_size_bytes = 16 GiB` here was SILENTLY IGNORED -- measured: the live volume is
# 11 GiB. Removed in DOCFIX-189.
lan_network_name = module.office1_network.network_name
wan_network_name = "office1-wan"
}
# ---- The D-100 dark-fiber mesh triangle: DC1<->DC2, DC1<->Office1,
# DC2<->Office1 -- mesh, not star. All three legs created now even though DC2
# itself is not yet wired above: the LINK is independent of whether DC2's
# plane networks exist yet, and Phase 0's gate wants "virtual networks present
# and isolated as designed" before Phase 2 needs them. ----
module "mesh_vr1_dc0_vr1_dc1" {
source = "./modules/mesh-link"
link_name = "vr1-dc0-vr1-dc1"
mtu = var.underlay_mtu
}
module "mesh_vr1_dc0_office1" {
source = "./modules/mesh-link"
link_name = "vr1-dc0-office1"
mtu = var.underlay_mtu
}
module "mesh_vr1_dc1_office1" {
source = "./modules/mesh-link"
link_name = "vr1-dc1-office1"
mtu = var.underlay_mtu
}
# =====================================================================
# D-114 -- Office1 site containment VM + its base image.
#
# voffice1 IS the Office1 "facility": MAAS-region runs on it, LXD runs on it, and the
# non-stack service machines (NetBox, GitBucket, Tailscale) are VMs MAAS COMPOSES into
# that LXD host -- VR0's proven `lxd` + `tailscale` pattern, applied per site.
#
# expose_nested_virt = true is LOAD-BEARING, not a nicety: LXD virtual machines are
# qemu/KVM guests, so without the host `svm` feature passed through NO service VM can
# ever boot inside voffice1 and D-114's model fails at its first step. (cloudinit-vm had
# NO cpu block at all before 2026-07-13, which silently gave every guest an emulated CPU
# with no svm -- measured: host is an EPYC 9965, a default-CPU guest sees "Opteron_G3".)
#
# Only network is office1-local: voffice1 reaches the internet THROUGH the OPNsense edge,
# like a real server behind a real site router. It takes its address by DHCP from the
# edge's Kea -- which also makes it the FIRST REAL LEASE that path has ever served.
#
# Cloud-init here is DELIBERATELY MINIMAL (identity + access + guest agent). MAAS and LXD
# are installed as separate GATED steps so they are observable and individually approved,
# not buried in a first-boot script that either silently works or silently does not.
# =====================================================================
module "ubuntu_noble_base" {
source = "./modules/base-image"
image_name = "ubuntu-24.04-base.qcow2"
pool_name = module.office1_storage.pool_name
# Official Ubuntu 24.04 LTS (noble) cloud image. VERIFIED reachable 2026-07-13 (HTTP 200,
# Last-Modified 2026-07-05). 24.04 matches what VR0 deploys for its lxd/juju/tailscale
# machines, so the image choice carries no new delta.
source_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
}
module "voffice1" {
source = "./modules/cloudinit-vm"
vm_name = "voffice1"
vcpu = var.voffice1_vcpu
memory_mib = var.voffice1_memory_mib
disk_size_bytes = var.voffice1_disk_bytes
pool_name = module.office1_storage.pool_name
base_volume_path = module.ubuntu_noble_base.path
network_names = [module.office1_network.network_name]
expose_nested_virt = true
user_data = <<-EOT
#cloud-config
hostname: voffice1
fqdn: voffice1.${var.domain_suffix}
manage_etc_hosts: true
users:
- name: jessea123
groups: [adm, sudo]
shell: /bin/bash
sudo: "ALL=(ALL) NOPASSWD:ALL"
ssh_authorized_keys:
- ${trimspace(file(var.office1_ssh_pubkey_path))}
package_update: true
packages:
- qemu-guest-agent
runcmd:
- [systemctl, enable, --now, qemu-guest-agent]
EOT
meta_data = <<-EOT
instance-id: voffice1-d114
local-hostname: voffice1
EOT
# Match by GLOB, not a guessed interface name: the NIC's kernel name (ens3 / enp1s0 /
# ...) depends on the machine type and PCI topology and is NOT known until first boot.
# Naming it would be an inferred value. dhcp4 is a DELIBERATE choice here, not the
# module's refused default: office1-local genuinely has an authoritative DHCP server
# (Kea on the OPNsense edge, pool .100-.199). A Kea RESERVATION is added afterwards,
# via the OPNsense REST API, to make the address stable.
network_config = <<-EOT
version: 2
ethernets:
lan:
match:
name: "en*"
dhcp4: true
EOT
}
# -----------------------------------------------------------------------------
# D-119 state migration. `moved` rewrites the STATE ADDRESS only -- it cannot
# suppress a ForceNew driven by an attribute, and `dc_name`/`link_name` are
# interpolated straight into each libvirt object's `name` (which IS ForceNew).
# So the libvirt networks/pools ARE replaced; these blocks exist so `tofu plan`
# reads as N clean "must be replaced" lines instead of N unrelated destroy+create
# pairs -- i.e. so the plan is REVIEWABLE.
#
# SAFE TO REPLACE, MEASURED 2026-07-14 BEFORE THE RENAME: all of these objects are
# EMPTY -- no guests are attached (Stage 3 has not run, so no DC node VMs exist)
# and no volumes live in either DC pool (the only volumes in state belong to
# office1_opnsense, ubuntu_noble_base and voffice1, none of which sit in a DC pool).
# Re-verify with `tofu plan` before applying; if a plan shows anything ATTACHED,
# STOP -- standing lesson 2: an apply touching a libvirt_domain's devices is an
# OUTAGE, and "updated in-place" does NOT mean "no restart".
# -----------------------------------------------------------------------------
# D-123 MODEL B: the moved{dc1_planes -> vr1_dc0_planes} block was DELETED here --
# vr1_dc0_planes is no longer in this (outer) root (it moved to the inner root), and a
# `moved` block cannot target an address absent from the config / cross state files.
moved {
from = module.dc1_storage
to = module.vr1_dc0_storage
}
moved {
from = module.dc2_storage
to = module.vr1_dc1_storage
}
moved {
from = module.mesh_dc1_dc2
to = module.mesh_vr1_dc0_vr1_dc1
}
moved {
from = module.mesh_dc1_office1
to = module.mesh_vr1_dc0_office1
}
moved {
from = module.mesh_dc2_office1
to = module.mesh_vr1_dc1_office1
}
# =====================================================================
# STAGE 3 -- vr1-dc0 substrate (runbooks/dc-dc-phase2-tofu-dc-substrate.md).
# Authored 2026-07-15. NOT YET APPLIED -- every block below is a gated Stage-3
# mutation; `tofu apply` is the operator's step (Steps 8/11 of the runbook).
# Governing rulings: D-100 (edge/fabric), D-121 (Option C node layout + sizing),
# D-122 (2-NIC fabric-routed edge; dedicated per-site ISP uplink).
#
# D-123 Model A + MAAS model RULED (region on Office1 + rack per DC) + D-124
# ADOPTED (Scheme A: office1<->dc0 mesh transit; rack sizing 4/8192/80). The
# `vvr1-dc0` rack controller IS wired below -- its two static IPs come from
# office1-netbox via variables (tfvars), NOT invented here. The Step-9
# `maas-vm-host` registration is STILL DEFERRED (DOCFIX-179): adding the
# `provider "maas"` block it needs would force EVERY plan (incl. the Steps 1-8
# substrate apply) to demand MAAS creds, and vr1-dc0's MAAS does not exist until
# vvr1-dc0 is up and its rack is installed. Wire maas-vm-host + the maas provider
# as its own step once the rack MAAS is reachable (runbook Step 9). The blocks
# below are the D-123-independent substrate (D-103/D-114 as-built) plus the rack.
# =====================================================================
# =====================================================================
# D-123 MODEL B (2026-07-16): the vr1-dc0 INNER SUBSTRATE moved to the inner root.
# The site-wan uplink, the 2-NIC OPNsense edge, the R-3 Option-C node layout (3 control
# + 2 compute + 4 storage = 9 nodes/DC) and the 9 node VMs are created INSIDE vvr1-dc0
# by the inner root `opentofu/vr1-dc0-substrate/` (provider = qemu+ssh to vvr1-dc0 over
# the D-124 transit, R-5). Their HCL is UNCHANGED -- the same node-vm / site-wan /
# opnsense-edge / dc-planes modules are reused verbatim; only the provider they run
# against moved from vcloud to vvr1-dc0's inner libvirt. This keeps the whole nested
# substrate one wholesale-deletable Section-9 shim (D-103) that collapses to a single
# root at Roosevelt. Apply order: OUTER (this root -- boots + sizes vvr1-dc0) ->
# BOOTSTRAP GATE (site-headend-install.sh: install nested libvirtd + inner pool +
# kvm nested=1 + stage the opnsense base image) -> INNER root (planes/wan/edge/nodes).
# =====================================================================
# netem (runbook Step 11) -- HELD as a comment. It is a POST-apply step that runs
# SSH-wrapped provisioners from Office1 to the vcloud host, so it needs a real
# `vcloud_host_ssh_target` (a runtime value) and the netem params are an unruled
# D-100 sub-item (gap #11, placeholder only). Measured input ready: the
# vr1-dc0<->office1 mesh leg is bridge `virbr7`. Wire at Step 11, not before.
# module "netem_vr1_dc0_office1" {
# source = "./modules/netem-link"
# link_name = "vr1-dc0-office1"
# bridge_name = "virbr7" # measured 2026-07-15
# vcloud_host_ssh_target = "<real ssh target -- runtime>"
# netem_args = "<PLACEHOLDER: buildout-design S6 same-metro lean; D-100 gap #11 unruled>"
# }
# =====================================================================
# D-125 (Model B bridge-in, closes OBS-3): the vr1-dc0 simulated-ISP uplink, at
# VCLOUD level. This is the SINGLE NAT = the ISP hand-off. It egresses via vcloud's
# own uplink (a libvirt NAT network masquerades out ITS host's default route), which
# is exactly the per-DC egress that D-123 Model B severed by nesting vr1-dc0-wan
# inside vvr1-dc0 (whose only routed leg is the East-West-only transit, SEC-010
# FORWARD-dropped). vvr1-dc0 takes an IP-less uplink NIC on this network (below); the
# inner vr1-dc0-wan is a BRIDGE (modules/wan-bridge) onto vvr1-dc0's br-vr1-dc0-wan,
# so the inner OPNsense WAN lands directly on THIS /24 and there is one NAT, here.
# The /24 is the SAME simulated-ISP WAN D-115 already assigned vr1-dc0 (172.30.2.0/24,
# role edge) -- bridge-in is single-NAT, so there is exactly ONE /24 and its IPAM
# identity is UNCHANGED from Model A; only the libvirt realization moved to vcloud.
# So this is a RULED literal, NOT a NetBox-assigned HELD value. OPNsense WAN stays .2
# (D-113 edge config, unchanged). Same site-wan NAT module as office1-wan -- 1500 MTU,
# no DHCP (edge WAN is static).
# DEPLOY-TIME GATE (D-125): that vcloud's virbr actually NATs OPNsense's foreign
# MAC across the doubly-nested bridge is unprovable pre-apply -- the bridge-in
# equivalent of the depth-4 boot gate. Isolation-test first (bootstrap prints it);
# fallback = double-NAT. See D-125.
module "vr1_dc0_uplink" {
source = "./modules/site-wan"
network_name = "vr1-dc0-uplink"
cidr = "172.30.2.0/24" # D-115 vr1-dc0 sim-ISP edge WAN; realized at vcloud under D-125 (ruled, not HELD)
# mtu defaults to 1500 (the ISP-uplink domain; never jumbo).
}
# =====================================================================
# D-123 MODEL B / D-124 AMENDMENT: vvr1-dc0 -- the vr1-dc0 SITE CONTAINMENT VM. Under
# Model B it is BOTH the MAAS rack controller (enrolled to the Office1 region) AND the
# nested libvirt HOST for the DC's entire node fleet (9 nodes + 6 planes + wan + edge,
# all created by the inner root). Sized to hold that fleet (~416 GiB, derived) with
# expose_nested_virt=true. ONE outer leg: the office1<->dc0 transit (region, D-124
# Scheme A); metal-admin + the other planes are INNER bridges. It PROXIES OS images from
# the region (D-107). Site-down = a single `virsh destroy vvr1-dc0` (the D-122 intent).
# NOT YET APPLIED. The gated post-boot BOOTSTRAP (site-headend-install.sh node-host mode:
# rack enroll + nested libvirtd + inner pool + kvm nested=1 + base image) precedes the
# inner-root apply.
# =====================================================================
module "vvr1_dc0" {
source = "./modules/cloudinit-vm"
vm_name = "vvr1-dc0"
vcpu = var.vvr1_dc0_vcpu # D-123 Model B / D-124 AMENDMENT: hosts the DC's node fleet
memory_mib = var.vvr1_dc0_memory_mib # derived 416 GiB (384 node + 32 overhead)
disk_size_bytes = var.vvr1_dc0_disk_bytes # derived ~3000 GiB (holds the inner libvirt pool)
pool_name = module.vr1_dc0_storage.pool_name
base_volume_path = module.ubuntu_noble_base.path
expose_nested_virt = true # D-123 Model B: LOAD-BEARING -- inner node VMs need svm passthrough or no inner KVM / nova boots
# D-123 Model B: metal-admin is now an INNER bridge (created by the inner root's
# dc-planes INSIDE vvr1-dc0; the rack serves PXE/DHCP there). TWO outer legs:
# NIC1 (enp1s0 -> mgmt) = office1<->dc0 transit, region-facing (D-124 Scheme A).
# SEC-010/--transit-if key on THIS interface -- keep it first.
# NIC2 (enp2s0 -> uplink) = D-125 bridge-in: an IP-LESS port enslaved into the netplan
# bridge br-vr1-dc0-wan below, attached to the vcloud ISP NAT
# (vr1-dc0-uplink). The inner OPNsense WAN rides this bridge to
# the single vcloud NAT -- restoring per-DC egress (OBS-3).
network_names = [
module.mesh_vr1_dc0_office1.network_name, # NIC1 enp1s0 -> mgmt (transit; SEC-010 keys here)
module.vr1_dc0_uplink.network_name, # NIC2 enp2s0 -> uplink (IP-less port of br-vr1-dc0-wan)
]
user_data = <<-EOT
#cloud-config
hostname: vvr1-dc0
fqdn: vvr1-dc0.${var.domain_suffix}
manage_etc_hosts: true
users:
- name: jessea123
groups: [adm, sudo]
shell: /bin/bash
sudo: "ALL=(ALL) NOPASSWD:ALL"
ssh_authorized_keys:
- ${trimspace(file(var.office1_ssh_pubkey_path))}
package_update: true
packages:
- qemu-guest-agent
runcmd:
- [systemctl, enable, --now, qemu-guest-agent]
EOT
meta_data = <<-EOT
instance-id: vvr1-dc0-d123
local-hostname: vvr1-dc0
EOT
# STATIC on both legs (planes + mesh have NO DHCP). IPs are NetBox-assigned
# (D-124), supplied via tfvars -- no literal here. INTERFACE-NAMING TRAP: the
# guest NIC names (enp1s0/enp2s0 vs ens3/...) depend on machine type and are NOT
# known until first boot -- CONFIRM on boot and adjust `match` before relying on
# this (documented trap; modules/opnsense-edge + the runbook). metal-admin gives
# the rack no default route; the region route rides the transit toward Office1
# (10.10.0.0/22, as-built D-115).
# OUTER legs = transit (NIC1 enp1s0 -> mgmt) + D-125 uplink (NIC2 enp2s0 -> uplink).
# INTERFACE-NAMING TRAP applies to BOTH (enp1s0/enp2s0 vs ensX depend on machine type,
# unknown until first boot -- CONFIRM on boot and adjust each `match` before relying on
# this). metal-admin is configured on the INNER bridge by the BOOTSTRAP step
# (site-headend-install.sh node-host mode) using var.vr1_dc0_rack_metal_admin_ip -- that
# var is consumed by the bootstrap script, NOT by this outer root nor the inner OpenTofu
# root (it remains a required tfvar for the L1-07 rack-addressing gate). The region route
# rides the transit toward Office1 (10.10.0.0/22, as-built D-115).
#
# D-125 bridge-in (closes OBS-3): NIC2 `uplink` is an IP-LESS bridge port; the netplan
# bridge `br-vr1-dc0-wan` (no IP on the vvr1-dc0 side -- only the inner OPNsense WAN gets a
# vcloud-ISP /24 address across it) is declared HERE so it exists before the inner apply.
# STP off (single port). The bootstrap only VERIFIES the bridge + grants libvirt/AppArmor
# access + isolation-tests egress; it does not create the bridge. br_netfilter CONSTRAINT
# (D-125): SEC-010's FORWARD-drop MUST stay scoped to `mgmt` -- a GLOBAL forward drop would
# silently kill bridged WAN frames if br_netfilter is loaded.
network_config = <<-EOT
version: 2
ethernets:
mgmt:
match:
name: "enp1s0"
set-name: mgmt
addresses: ["${var.vr1_dc0_rack_transit_ip}/${var.vr1_dc0_rack_transit_prefix}"]
routes:
- to: "10.10.0.0/22"
via: "${var.vr1_dc0_rack_transit_peer_ip}"
uplink:
match:
name: "enp2s0"
set-name: uplink
dhcp4: false
dhcp6: false
bridges:
br-vr1-dc0-wan:
interfaces: [uplink]
dhcp4: false
dhcp6: false
parameters:
stp: false
forward-delay: 0
EOT
}
