# =====================================================================
# D-123 MODEL B (2026-07-16) -- vr1-dc0 INNER SUBSTRATE, created inside vvr1-dc0.
# Run this root AFTER: (1) the outer opentofu/ root has booted + sized vvr1-dc0
# (expose_nested_virt=true), and (2) the bootstrap gate (site-headend-install.sh
# node-host mode) has installed libvirtd + the inner pool + kvm nested=1 + the
# OPNsense base image on vvr1-dc0. The provider dials vvr1-dc0 over the D-124 transit
# (R-5: remote qemu+ssh from Office1). All module bodies are UNCHANGED from the outer
# root -- only the provider they run against moved (vcloud -> vvr1-dc0). See
# docs/model-a-fallback-plan.md for the Model A revert.
# =====================================================================

provider "libvirt" {
  # qemu+ssh to vvr1-dc0 over the transit. Key auth + known_hosts verified out of band
  # (NO no_verify). The IP is MEASURED after the outer apply (hard rule 2).
  # keyfile + sshauth are REQUIRED URI params (measured 2026-07-20, first real dial):
  # the provider's Go ssh transport ignores ~/.ssh/config and default identities
  # ("attempted methods [none]"). known_hosts: ALL key types must be present --
  # a single accept-new ED25519 entry fails the handshake with "knownhosts: key
  # mismatch" (seed via ssh-keyscan, fingerprint cross-checked against the
  # accept-new-trusted entry).
  uri = "qemu+ssh://${var.vvr1_dc0_ssh_user}@${var.vvr1_dc0_transit_ip}/system?keyfile=${var.vvr1_dc0_ssh_keyfile}&sshauth=privkey"
}

# Inner storage pool (backs the node + edge disks) -- created inside vvr1-dc0 at the
# path the bootstrap step provisioned.
module "inner_storage" {
  source      = "../modules/dc-storage-pool"
  dc_name     = "vr1-dc0-inner"
  target_path = var.inner_pool_path
}

# The six vr1-dc0 planes (D-101 layout, unchanged) -- now isolated-L2 networks INSIDE
# vvr1-dc0. metal-admin is where the rack serves PXE/DHCP to the inner nodes.
module "vr1_dc0_planes" {
  source        = "../modules/dc-planes"
  dc_name       = "vr1-dc0"
  domain_suffix = var.domain_suffix
  mtu           = var.underlay_mtu
  planes        = var.vr1_dc0_planes
}

# D-125 (Model B bridge-in, closes OBS-3): the DC WAN segment is a BRIDGE, not a NAT.
# Under D-123 Model B this network is created INSIDE vvr1-dc0, whose only routed leg is
# the East-West-only transit (SEC-010 FORWARD-dropped), so a NAT here would have no egress
# (that was OBS-3). Instead it bridges onto vvr1-dc0's br-vr1-dc0-wan (an IP-less uplink
# NIC on the vcloud ISP NAT `vr1-dc0-uplink` = 172.30.2.0/24, declared in the outer root's
# vvr1-dc0 netplan). The OPNsense WAN keeps its static .2 on that SAME /24 D-115 already
# assigned vr1-dc0 (bridge-in is single-NAT; the /24 is UNCHANGED from Model A, only the
# libvirt host moved) -- set in the edge config (D-113), no re-address, no new NetBox object.
# There is ONE NAT, at vcloud. host_bridge MUST match the netplan bridge name in opentofu/main.tf.
module "vr1_dc0_wan" {
  source       = "../modules/wan-bridge"
  network_name = "vr1-dc0-wan"
  host_bridge  = "br-vr1-dc0-wan"
  # mtu defaults to 1500 (ISP-uplink domain; NOT the jumbo planes/mesh).
}

# D-122: the DC edge -- 2-NIC (WAN + LAN), Office1 pattern (2048/2/nano). LAN =
# provider-public (external gateway, D-100); WAN = the dedicated uplink above.
module "vr1_dc0_opnsense" {
  source           = "../modules/opnsense-edge"
  vm_name          = "vr1-dc0-opnsense"
  autostart        = true # D-127: the DC edge comes up with its containment VM (routing for the site)
  memory_mib       = 2048
  vcpu             = 2
  pool_name        = module.inner_storage.pool_name
  base_volume_path = var.opnsense_base_path
  lan_network_name = module.vr1_dc0_planes.network_names["provider-public"]
  wan_network_name = module.vr1_dc0_wan.network_name
  # D-129: NEW DC edges boot WITH the qga channel so they never need the retrofit
  # (operator-ruled 2026-07-20). The office1 edge stays untouched at the default OFF.
  expose_qga_channel = true
}

# D-121 Option C layout (R-3 2026-07-16): 3 control + 2 compute + 4 storage = 9 nodes/DC.
# Six NICs each (one per plane); metal-admin FIRST = PXE/boot plane (D-052 default binding).
locals {
  vr1_dc0_node_nics = [
    module.vr1_dc0_planes.network_names["metal-admin"], # PXE / boot fabric first
    module.vr1_dc0_planes.network_names["provider-public"],
    module.vr1_dc0_planes.network_names["metal-internal"],
    module.vr1_dc0_planes.network_names["data-tenant"],
    module.vr1_dc0_planes.network_names["storage"],
    module.vr1_dc0_planes.network_names["replication"],
  ]

  # macs: MEASURED 2026-07-21 (`virsh domiflist` on vvr1-dc0, all 9 domains;
  # session changelog 2026-07-21 delivery). Order = vr1_dc0_node_nics order
  # (metal-admin, provider-public, metal-internal, data-tenant, storage,
  # replication). PINNED because an in-place apply regenerated all 9 unpinned
  # boot MACs on 2026-07-20 and stranded the fleet in MAAS (adjudication:
  # docs/audit/commissioning-diag-20260721.txt). The metal-admin entries were
  # cross-checked against MAAS boot-interface records 9/9 the same day. These
  # are identity values, not tunables -- change one and MAAS no longer knows
  # the node.
  vr1_dc0_nodes = {
    "vr1-dc0-control-01" = { vcpu = 16, mem = 65536, disk_gib = 150, macs = [
      "52:54:00:be:69:c5", "52:54:00:29:e5:2b", "52:54:00:05:98:c4",
      "52:54:00:96:e8:36", "52:54:00:5a:dc:91", "52:54:00:35:cc:01",
    ] }
    "vr1-dc0-control-02" = { vcpu = 16, mem = 65536, disk_gib = 150, macs = [
      "52:54:00:02:ff:57", "52:54:00:b0:17:2a", "52:54:00:58:17:23",
      "52:54:00:1c:ab:44", "52:54:00:a6:f8:0b", "52:54:00:2e:09:d8",
    ] }
    "vr1-dc0-control-03" = { vcpu = 16, mem = 65536, disk_gib = 150, macs = [
      "52:54:00:4f:de:a9", "52:54:00:0b:c2:1b", "52:54:00:c0:d3:e6",
      "52:54:00:10:0f:ea", "52:54:00:53:19:ef", "52:54:00:86:78:ed",
    ] }
    "vr1-dc0-compute-01" = { vcpu = 12, mem = 49152, disk_gib = 100, macs = [
      "52:54:00:1b:19:e6", "52:54:00:8c:2a:8c", "52:54:00:9c:7f:7a",
      "52:54:00:33:92:4e", "52:54:00:40:62:bb", "52:54:00:b7:d1:a2",
    ] }
    "vr1-dc0-compute-02" = { vcpu = 12, mem = 49152, disk_gib = 100, macs = [
      "52:54:00:18:ab:b4", "52:54:00:50:48:88", "52:54:00:78:fb:c5",
      "52:54:00:cc:84:61", "52:54:00:e4:ab:df", "52:54:00:5b:93:c4",
    ] }
    "vr1-dc0-storage-01" = { vcpu = 8, mem = 24576, disk_gib = 550, osd_gib = 500, macs = [
      "52:54:00:5f:8d:42", "52:54:00:4c:69:7b", "52:54:00:ac:2c:4d",
      "52:54:00:28:91:d2", "52:54:00:fa:7c:53", "52:54:00:0c:7a:ab",
    ] }
    "vr1-dc0-storage-02" = { vcpu = 8, mem = 24576, disk_gib = 550, osd_gib = 500, macs = [
      "52:54:00:48:86:2c", "52:54:00:2e:8b:55", "52:54:00:d7:4e:38",
      "52:54:00:05:60:af", "52:54:00:c6:02:bd", "52:54:00:2c:f8:42",
    ] }
    "vr1-dc0-storage-03" = { vcpu = 8, mem = 24576, disk_gib = 550, osd_gib = 500, macs = [
      "52:54:00:b1:94:d1", "52:54:00:bd:d1:24", "52:54:00:80:67:87",
      "52:54:00:0f:a6:35", "52:54:00:e0:73:ed", "52:54:00:62:65:77",
    ] }
    "vr1-dc0-storage-04" = { vcpu = 8, mem = 24576, disk_gib = 550, osd_gib = 500, macs = [ # R-3: 4th OSD host
      "52:54:00:2b:ed:ab", "52:54:00:90:ca:d0", "52:54:00:2a:ac:14",
      "52:54:00:af:34:fd", "52:54:00:28:b1:6d", "52:54:00:ab:5c:50",
    ] }
    # ---- D-104 AMENDMENT (RULED 2026-07-24): the DEDICATED per-DC Juju controller.
    # NOT a D-121 role node -- the 9 OpenStack nodes above are unchanged, so Option C's
    # 3/2/4 shape, the R-3 Ceph rebuild headroom and the control-quorum spread all stand.
    # `for_each` is keyed, so this plans 1 add / 0 change / 0 destroy.
    #
    # SIZING: 8 GiB / 4 vCPU are the values the capacity GATE actually modelled
    # (`scripts/dc-dc-whole-host-budget.py`, 10-node x 2-DC -> RAM 854/1024 = 83 pct, FIT,
    # 170 GiB headroom; capture docs/audit/stage5-controller-capacity-20260724.txt).
    # Disk is the authoring value the amendment left open; 100 GiB matches the smallest
    # existing node class.
    #
    # MACs DELIBERATELY EMPTY. modules/node-vm documents that as correct ONLY before a
    # node is enlisted anywhere -- which is this VM's state. PIN THEM FROM MEASUREMENT
    # after its first apply, exactly as the nine role nodes were (2026-07-21); leaving
    # them unpinned past enlistment is the MAC-regeneration trap that stranded the fleet.
    #
    # Its MAAS tag is `juju-controller-vr1-dc0` and it carries NO role tag, so
    # `juju bootstrap --constraints tags=juju-controller-vr1-dc0` targets it
    # deterministically and no OpenStack role constraint can cross-grab it.
    # MACs PINNED 2026-07-29 from measurement (`virsh domiflist`) after the VM's
    # first apply, exactly as the nine role nodes were on 2026-07-21. It is now
    # ENLISTED in MAAS, which is the boundary modules/node-vm warns about: past
    # enlistment an in-place apply can regenerate unpinned MACs and strand the node.
    # NOTE these are libvirt-generated `52:54:00:` values, NOT dc1's schematic
    # `52:54:01:d1:NN:PP` scheme -- a direct consequence of the deliberate `macs = []`
    # at first apply. Recorded so the difference is not later read as a defect.
    "vr1-dc0-juju-01" = { vcpu = 4, mem = 8192, disk_gib = 100, macs = [
      "52:54:00:48:e7:1e", "52:54:00:ed:cd:74", "52:54:00:0f:ee:3a",
    "52:54:00:b0:1e:dd", "52:54:00:24:26:0a", "52:54:00:00:a1:37"] }

    # ---- D-132 AMENDMENT ADDENDUM (RULED 2026-07-30): the DEDICATED per-DC MAAS REGION.
    # Operator utterance: "Dedicated region VM per DC at utility .6 (Recommended)".
    # NOT a D-121 role node and NOT the Juju controller -- the 9 OpenStack nodes and the
    # D-104 controller above are unchanged. `for_each` is keyed, so this plans 1 add /
    # 0 change / 0 destroy.
    #
    # WHY IT EXISTS: `jujud` runs ON the in-DC Juju controller and dials the MAAS region
    # API for its whole life. With the region at Office1 that dependency had to cross the
    # fiber, which SEC-010's transit FORWARD-drop forbids -- MEASURED as
    # `ERROR creating MAAS environ: ... 10.10.0.20:5240: connect: connection refused` on
    # the 2026-07-30 bootstrap. A DC-local region REMOVES the requirement instead of
    # excepting it, which is why SEC-010 stays UNAMENDED.
    #
    # SIZING: 4 vCPU / 8 GiB are the values the capacity gate actually modelled
    # (`scripts/dc-dc-whole-host-budget.py --containment-overhead-mem-gib 32
    # --containment-overhead-vcpu 12` -> RAM 870/1024 = 85 pct, FIT, 154 GiB headroom).
    # Disk 150 GiB, not the controller's 100: this VM stores the region's PostgreSQL AND
    # its boot-image set (jammy + noble are both synced today), so it is the one utility
    # VM whose disk grows with the image catalogue.
    #
    # MACs DELIBERATELY EMPTY -- correct ONLY before the VM is enlisted anywhere, which is
    # its state at first apply. PIN THEM FROM MEASUREMENT (`virsh domiflist`) immediately
    # after that apply, exactly as the nine role nodes were on 2026-07-21 and the Juju
    # controller was on 2026-07-29. Leaving them unpinned past enlistment is the
    # MAC-regeneration trap that stranded the whole fleet once already.
    #
    # ADDRESSING: utility-band `.6` on every plane it is attached to (D-134's octet map,
    # extended by the same 2026-07-30 ruling: .4 artifact service / .5 Juju controller /
    # .6 MAAS region -- a STANDING cross-DC standard, so divergence is a DEFECT).
    # CARVE BOTH metal-admin (10.12.8.6) AND provider-public (10.12.4.6, gw 10.12.4.1) at
    # enlistment. The region needs egress for image sync, and carving metal-admin ALONE is
    # exactly the under-carve that cost three bootstrap attempts on the controller VM the
    # same day -- it has no default route and cannot reach the internet.
    # MACs PINNED 2026-07-30 from measurement (`virsh domiflist vr1-dc0-maas-01` on
    # vvr1-dc0) in the SAME session as the first apply, before enlistment -- the boundary
    # modules/node-vm warns about. Order = vr1_dc0_node_nics order (metal-admin,
    # provider-public, metal-internal, data-tenant, storage, replication).
    # Libvirt-generated `52:54:00:` values, as expected from the deliberate `macs = []`
    # at first apply -- NOT dc1's schematic scheme. These are identity values, not
    # tunables: change one and MAAS no longer knows the node.
    "vr1-dc0-maas-01" = { vcpu = 4, mem = 8192, disk_gib = 150, macs = [
      "52:54:00:dc:5c:0b", "52:54:00:81:15:a7", "52:54:00:a9:0b:d6",
    "52:54:00:a6:8a:94", "52:54:00:db:7f:65", "52:54:00:0c:22:a3"] }

  }
}

module "vr1_dc0_node" {
  for_each = local.vr1_dc0_nodes

  source          = "../modules/node-vm"
  vm_name         = each.key
  autostart       = false # D-127: node VMs are MAAS-power-controlled -- MANUAL, never on host boot
  vcpu            = each.value.vcpu
  memory_mib      = each.value.mem
  disk_size_bytes = each.value.disk_gib * 1024 * 1024 * 1024
  # R1 (D-121 amendment): opt-in OSD data device, storage nodes only.
  # `try(..., 0)` keeps every other node's plan byte-identical.
  osd_disk_size_bytes = try(each.value.osd_gib, 0) * 1024 * 1024 * 1024
  pool_name           = module.inner_storage.pool_name
  network_names       = local.vr1_dc0_node_nics
  interface_macs      = each.value.macs # pinned -- see the vr1_dc0_nodes comment
}
