# opnsense-edge: one OPNsense edge VM (D-100's per-site independent OPNsense
# edge with a simulated ISP uplink; D-103: OpenTofu owns creating it).
#
# NOT the cloud-init pattern (modules/cloudinit-vm) -- confirmed unreliable
# on OPNsense/FreeBSD this session (see opentofu/README.md's "OPNsense
# deployment research" section, fully sourced). This module instead uses
# OPNsense's own Configuration Importer: a plain ISO9660 volume containing
# /conf/config.xml, attached as a secondary cdrom disk -- mechanically
# IDENTICAL to cloudinit-vm's seed-volume shape (a libvirt_volume with
# create.content.url, attached as device="cdrom"), just a different payload
# and no libvirt_cloudinit_disk resource (that resource is NoCloud-specific;
# wrong format here). Both the base image and the config ISO are prepared
# OUTSIDE OpenTofu by scripts/opnsense-prep-image.sh and
# scripts/opnsense-build-config-iso.sh respectively -- see each variable's
# description for why.
#
# UNVERIFIED, flagged plainly (see opentofu/README.md for the full account,
# including a 2026-07-09 audit pass that corrected the LAN/WAN claim below):
# (a) whether the Configuration Importer's ISO9660 support actually behaves
# as described once booted for real -- the research is well-sourced but not
# independently tested this session (no real OPNsense boot, no
# genisoimage/xorriso available).
# (b) `devices.interfaces` list ORDER and OPNsense's LAN/WAN ROLE are TWO
# SEPARATE THINGS, not one -- corrected during the audit pass (the original
# version of this comment conflated them). List order plausibly determines
# which `vtnetN` device number FreeBSD assigns each NIC (libvirt's own docs:
# PCI addresses auto-assigned in XML order "usually match" for a simple
# topology like this one -- reasonably well-supported, not a hard guarantee).
# But OPNsense's LAN/WAN designation is a SEPARATE, explicit mapping made
# inside config.xml itself (or the interactive/API assignment step) from
# device name to role -- "vtnet0 = WAN" is a convention some guides choose,
# not a FreeBSD/OPNsense-enforced rule. So: `var.lan_network_name` at index 0
# only expresses INTENT for which network should end up as LAN; the real
# config.xml (still undesigned, see variables.tf) MUST independently
# confirm, on a real boot, which vtnetN corresponds to which network, and
# map <wan>/<lan> to the CORRECT device name accordingly -- do not assume
# "first in this list" automatically becomes OPNsense's LAN role.
#
# No PXE boot-order attribute is used here (unlike modules/node-vm): this VM
# boots from its own pre-installed disk, not over the network, so the
# UNVERIFIED `boot`-order shape noted in node-vm/main.tf does not need
# touching for this module at all.

resource "libvirt_volume" "disk" {
  name = "${var.vm_name}-disk.qcow2"
  pool = var.pool_name

  # Direct per-VM COPY of the prepared nano image, NOT a backing/overlay. Matches the
  # documented `virt-install --import <nano.qcow2>` workflow; OPNsense nano is designed
  # to be booted + auto-expanded in place.
  #
  # CORRECTION 2026-07-12 (DOCFIX-189): the original version of this comment claimed the
  # COW backing "made boot2 fault ... the disk shape did [help]". That is FALSE -- the
  # fault persisted identically after this change. The real cause was the memory unit
  # (see the domain below). This shape is RETAINED because it is the documented import
  # workflow, not because it fixed anything.
  #
  # DISK SIZING IS NOT OWNED HERE. The copy inherits the prepped image's capacity, so the
  # size lever is scripts/opnsense-prep-image.sh's GROW arg (default +8G -> an 11 GiB
  # nano). There is deliberately no `capacity`/`disk_size_bytes` input: setting one here
  # would be silently ignored, which is exactly the trap DOCFIX-189 removed. Consequence
  # to know: every VM built from the same prepped base gets the same disk size.
  create = {
    content = {
      url = var.base_volume_path
    }
  }

  target = {
    format = {
      type = "qcow2"
    }
  }
}

resource "libvirt_volume" "config_seed" {
  name = "${var.vm_name}-config.iso"
  pool = var.pool_name

  create = {
    content = {
      url = var.config_iso_path
    }
  }
}

resource "libvirt_domain" "vm" {
  name = var.vm_name
  # memory_unit is REQUIRED. On dmacvicar/libvirt >=0.9, `memory` is a raw libvirt
  # value, NOT MiB (that was the 0.8-era meaning): with no unit libvirt defaults to
  # KiB, so `memory = 2048` renders `-m size=2048k` = 2 MiB. That was the ACTUAL root
  # cause of the 2026-07-12 boot triple-fault (measured: `virsh dominfo` Max memory
  # 2048 KiB) -- boot2 fits in 2 MiB and echoes /boot.config, then dies handing off to
  # /boot/loader, which does not. See docs/incident-20260712-opnsense-edge-boot-triplefault.md.
  memory      = var.memory_mib
  memory_unit = "MiB"
  vcpu        = var.vcpu
  type        = "kvm"
  running     = true

  # ACPI IS REQUIRED. With no `features` block, libvirt renders the machine with
  # `acpi=off`, and FreeBSD/OPNsense then panics on the FIRST interrupt setup:
  #
  #   panic: running without device atpic requires a local APIC
  #   apic_init() at ... mi_startup()
  #
  # FreeBSD discovers the local APIC from ACPI's MADT table, and the OPNsense kernel
  # carries no `atpic` fallback -- so no ACPI means no usable interrupt controller. The
  # guest lands in the ddb debugger (`db>`), which presents as a domain that is "running"
  # while burning 100% of a core and emitting nothing further on the console.
  #
  # This was the SECOND boot bug (2026-07-12), found only after the memory-unit fix let
  # the kernel get far enough to reach interrupt init. See the incident report.
  features = {
    acpi = true
    apic = {}
  }

  # host-passthrough with the AMD `svm` (nested-virt) feature disabled. NOTE: this was
  # tried as a triple-fault fix on 2026-07-12 and did NOT resolve it (the real cause was
  # the memory unit above). Retained because it is a legitimate nested-virt hardening for
  # a guest that has no business seeing svm, not because it fixed anything.
  cpu = {
    mode = "host-passthrough"
    features = [
      {
        name   = "svm"
        policy = "disable"
      }
    ]
  }

  os = {
    type      = "hvm"
    type_arch = "x86_64"
    # i440fx (pc) rather than q35. CORRECTION 2026-07-12 (DOCFIX-189): the original
    # comment here claimed q35 "triple-faults ... confirmed via the serial log". That is
    # FALSE -- the serial log showed the SAME 262-byte fault before AND after this change;
    # the real cause was the memory unit (see above). i440fx is RETAINED only because it
    # is the conventional, widely-exercised machine type for FreeBSD/OPNsense guests --
    # NOT because it fixed this incident. q35 was never actually shown to be a problem.
    type_machine = "pc"
  }

  devices = {
    disks = [
      {
        source = {
          volume = {
            pool   = libvirt_volume.disk.pool
            volume = libvirt_volume.disk.name
          }
        }
        target = {
          dev = "vda"
          bus = "virtio"
        }
        driver = {
          type = "qcow2"
        }
      },
      {
        device = "cdrom"
        source = {
          volume = {
            pool   = libvirt_volume.config_seed.pool
            volume = libvirt_volume.config_seed.name
          }
        }
        target = {
          dev = "sda"
          bus = "sata"
        }
      }
    ]

    # index 0 = the network intended for LAN, index 1 = the network intended
    # for WAN -- this ORDER plausibly controls which vtnetN device number
    # each gets (see the file header's corrected note), but does NOT by
    # itself make OPNsense treat either as LAN/WAN -- that role assignment
    # is config.xml's job. Confirm actual vtnetN<->network mapping on a real
    # boot before writing config.xml's <wan>/<lan> interface names.
    interfaces = [
      {
        model = {
          type = "virtio"
        }
        source = {
          network = {
            network = var.lan_network_name
          }
        }
      },
      {
        model = {
          type = "virtio"
        }
        source = {
          network = {
            network = var.wan_network_name
          }
        }
      }
    ]

    # OPNsense nano images are SERIAL-console only -- the image ships /boot.config with
    # `-S115200 -h`, so its loader expects a serial device.
    #
    # `unix` socket + `log`, NOT a plain `file` source. The two roles are different and we
    # need BOTH:
    #   source.unix -> an INTERACTIVE, BIDIRECTIONAL console. Required by D-112(c): the
    #     bootstrap is a one-time console login to enable SSH. A `file` source is WRITE-ONLY
    #     capture -- it cannot be typed into, so it cannot bootstrap anything. A unix socket
    #     is driven non-interactively with `socat - UNIX-CONNECT:<path>`, unlike a `pty`,
    #     which needs a controlling TTY and is therefore awkward to script.
    #   log         -> keeps the full boot capture we had before. Output emitted BEFORE any
    #     socket client connects would otherwise be lost; `log` records everything from
    #     power-on regardless. This capture is what made the 2026-07-12 boot bugs legible at
    #     all -- do not drop it.
    #
    # (DOCFIX-189 correction, for the record: adding a console in DOCFIX-187 did not CHANGE
    # the original triple-fault, it only revealed it. The domain faulted with and without.)
    serials = [
      {
        source = {
          unix = {
            mode = "bind"
            path = "/var/lib/libvirt/vr1/staging/${var.vm_name}-console.sock"
          }
        }
        log    = { file = "/var/lib/libvirt/vr1/staging/${var.vm_name}-serial.log", append = "on" }
        target = { port = "0" }
      }
    ]
  }
}