diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 7fa575e39aa9..44aa06b22740 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -874,6 +874,7 @@ impl TopologyConfigInfo { HYPERVISOR_NAME_CH, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_FIRECRACKER, + HYPERVISOR_NAME_OPENVMM, HYPERVISOR_NAME_REMOTE, ]; let hypervisor_name = toml_config.runtime.hypervisor_name.as_str(); diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 5a580fdcfbaf..cf107ae026e1 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -612,7 +612,18 @@ ifeq ($(USE_OPENVMM),true) KERNELTYPE_OPENVMM = uncompressed KERNEL_NAME_OPENVMM = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_OPENVMM)) KERNELPATH_OPENVMM = $(KERNELDIR)/$(KERNEL_NAME_OPENVMM) - DEFSANDBOXCGROUPONLY_OPENVMM := true + # OpenVMM runs in-process inside the runtime-rs shim, so guest RAM is + # shmem owned by the shim PID. When sandbox_cgroup_only=true the shim + # lives in the pod cgroup, capped by container.limits.memory, and the + # kernel OOM-killer fires during VM construction for any non-trivial + # guest RAM size ("misc mshv: Failed to populate memory region: -12", + # surfaced as "ttrpc: closed" / FailedCreatePodSandBox). Default to + # false so the shim+VMM land in the unconstrained kata-overhead + # cgroup; container processes still get sized correctly because they + # are placed in the pod cgroup at the usual time. Operators that + # want the legacy single-cgroup behaviour can still flip this back + # in configuration-openvmm.toml. + DEFSANDBOXCGROUPONLY_OPENVMM := false DEFSTATICRESOURCEMGMT_OPENVMM := false DEFSHAREDFS_OPENVMM := virtio-fs RUNTIMENAME := virt_container diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index 9c0aa8f3e5db..5df53e9ca0fa 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -45,7 +45,7 @@ protocols = { workspace = true, features = ["async"] } persist = { workspace = true } ch-config = { workspace = true, optional = true } tests_utils = { workspace = true } -tracing-subscriber = { version = "0.3", optional = true } +tracing-subscriber = { version = "0.3", optional = true, features = ["env-filter"] } # Dependencies that only exist to support hypervisors which upstream only # build on x86_64 and aarch64 (dragonball, firecracker). Keeping them in a diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio_device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio_device/mod.rs index 0758e3d43d41..be06b051ce13 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio_device/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio_device/mod.rs @@ -6,7 +6,7 @@ mod core; mod device; -pub use core::{discover_vfio_group_device, VfioDevice}; +pub use core::{discover_vfio_group_device, DeviceAddress, VfioDevice}; pub use device::VfioDeviceBase; pub use device::VfioDeviceModern; pub use device::VfioDeviceModernHandle; diff --git a/src/runtime-rs/crates/hypervisor/src/device/pci_path.rs b/src/runtime-rs/crates/hypervisor/src/device/pci_path.rs index 1d9cb8dd2806..1195f5372869 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/pci_path.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/pci_path.rs @@ -17,25 +17,73 @@ use anyhow::{anyhow, Context, Result}; const PCI_SLOT_BITS: u32 = 5; const MAX_PCI_SLOTS: u32 = (1 << PCI_SLOT_BITS) - 1; -// A PciSlot describes where a PCI device sits on a single bus +// The PCI spec reserves 3 bits for the function number (0..7), allowing +// up to 8 logical functions per PCI device slot. Most devices kata +// talks to are single-function (function 0), and the on-the-wire format +// omits the function in that case to stay backward-compatible (see the +// Display impl below). +const PCI_FUNCTION_BITS: u32 = 3; +const MAX_PCI_FUNCTIONS: u32 = (1 << PCI_FUNCTION_BITS) - 1; + +// A PciSlot describes where one logical PCI function sits on a single +// bus: a device number (5 bits, 0..0x1f) plus a function number +// (3 bits, 0..7). // -// This encapsulates the PCI slot number a.k.a device number, which is -// limited to a 5 bit value [0x00..0x1f] by the PCI specification +// Wire format: a single-function slot serialises as "ss" (e.g. "1f") +// to keep the historical slot-only form intact; a multi-function slot +// serialises as "ss.f" (e.g. "03.5"). Kata-agent's +// `pci::Path::from_str` accepts both ("00" is parsed as function 0). // -// To support multifunction device's, It's needed to extend -// this to include the PCI 3-bit function number as well. -#[derive(Clone, Debug, Default, PartialEq)] -pub struct PciSlot(pub u8); +// Multi-function packing is needed by the OpenVMM backend, where the +// PCIe root complex packs up to 8 root ports per device slot +// (see `GenericPcieRootComplex::new` in microsoft/openvmm: +// `vm/devices/pci/pcie/src/root.rs`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PciSlot { + pub device: u8, + pub function: u8, +} impl PciSlot { - pub fn new(v: u8) -> PciSlot { - PciSlot(v) + /// Construct a single-function PciSlot at the given device number. + pub fn new(device: u8) -> PciSlot { + PciSlot { + device, + function: 0, + } + } + + /// Construct a multi-function PciSlot. Used by the openvmm backend + /// when packing many root ports into one device slot. + pub fn with_function(device: u8, function: u8) -> Result { + if device as u32 > MAX_PCI_SLOTS { + return Err(anyhow!( + "PCI device number {} exceeds MAX: {}", + device, + MAX_PCI_SLOTS + )); + } + if function as u32 > MAX_PCI_FUNCTIONS { + return Err(anyhow!( + "PCI function number {} exceeds MAX: {}", + function, + MAX_PCI_FUNCTIONS + )); + } + Ok(PciSlot { device, function }) } } impl std::fmt::Display for PciSlot { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:02x}", self.0) + // Function 0 is the historical default; omit it so the wire + // format stays compatible with single-function callers and with + // kata-agent's `SlotFn::from_str("xx")` shorthand. + if self.function == 0 { + write!(f, "{:02x}", self.device) + } else { + write!(f, "{:02x}.{:01x}", self.device, self.function) + } } } @@ -43,23 +91,56 @@ impl TryFrom<&str> for PciSlot { type Error = anyhow::Error; fn try_from(s: &str) -> Result { - if s.is_empty() || s.len() > 2 { + if s.is_empty() { return Err(anyhow!("string given is invalid.")); } + // Accept both "ss" (function defaults to 0) and "ss.f". + let mut tokens = s.splitn(2, '.'); + let dev_str = tokens.next().unwrap(); + let func_str = tokens.next(); + + if dev_str.is_empty() || dev_str.len() > 2 { + return Err(anyhow!("string given is invalid.")); + } + if let Some(f) = func_str { + if f.is_empty() || f.len() > 1 { + return Err(anyhow!("function token '{}' is invalid", f)); + } + } + let base = 16; - let n = u64::from_str_radix(s, base).context(format!( - "convert string to number with base {base:?} failed." + let device = u8::from_str_radix(dev_str, base).context(format!( + "convert device string '{}' to number with base {} failed.", + dev_str, base ))?; - if n >> PCI_SLOT_BITS > 0 { + if (device as u32) > MAX_PCI_SLOTS { return Err(anyhow!( - "number {:?} exceeds MAX:{:?}, failed.", - n, + "PCI device number {} exceeds MAX: {}", + device, MAX_PCI_SLOTS )); } - Ok(PciSlot(n as u8)) + let function = match func_str { + Some(f) => { + let n = u8::from_str_radix(f, base).context(format!( + "convert function string '{}' to number with base {} failed.", + f, base + ))?; + if (n as u32) > MAX_PCI_FUNCTIONS { + return Err(anyhow!( + "PCI function number {} exceeds MAX: {}", + n, + MAX_PCI_FUNCTIONS + )); + } + n + } + None => 0, + }; + + Ok(PciSlot { device, function }) } } @@ -71,7 +152,10 @@ impl TryFrom for PciSlot { return Err(anyhow!("value {:?} exceeds MAX: {:?}", v, MAX_PCI_SLOTS)); } - Ok(PciSlot(v as u8)) + Ok(PciSlot { + device: v as u8, + function: 0, + }) } } @@ -117,12 +201,14 @@ impl PciPath { impl std::fmt::Display for PciPath { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Delegate to PciSlot::Display so that multi-function slots + // serialise as "ss.f" while single-function slots stay "ss". write!( f, "{}", self.slots .iter() - .map(|pci_slot| format!("{:02x}", pci_slot.0)) + .map(|pci_slot| pci_slot.to_string()) .collect::>() .join("/") ) @@ -182,30 +268,70 @@ mod tests { // valid number let pci_slot_04 = PciSlot::try_from(1_u32); assert!(pci_slot_04.is_ok()); - assert_eq!(pci_slot_04.as_ref().unwrap().0, 1_u8); + assert_eq!(pci_slot_04.as_ref().unwrap().device, 1_u8); + assert_eq!(pci_slot_04.as_ref().unwrap().function, 0_u8); let pci_slot_str = pci_slot_04.as_ref().unwrap().to_string(); - assert_eq!(pci_slot_str, format!("{:02x}", pci_slot_04.unwrap().0)); + assert_eq!( + pci_slot_str, + format!("{:02x}", pci_slot_04.unwrap().device) + ); // max number let pci_slot_05 = PciSlot::try_from(31_u32); assert!(pci_slot_05.is_ok()); - assert_eq!(pci_slot_05.unwrap().0, 31_u8); + assert_eq!(pci_slot_05.unwrap().device, 31_u8); // exceed and error let pci_slot_06 = PciSlot::try_from(32_u32); assert!(pci_slot_06.is_err()); } + #[test] + fn test_pci_slot_multifunction() { + // Multi-function constructor accepts function 0..7. + let slot = PciSlot::with_function(0x03, 5).unwrap(); + assert_eq!(slot.device, 0x03); + assert_eq!(slot.function, 5); + + // Wire format: non-zero function emits "ss.f". + assert_eq!(slot.to_string(), "03.5"); + + // Wire format: function 0 stays in the historical short form. + assert_eq!(PciSlot::new(0x03).to_string(), "03"); + assert_eq!(PciSlot::with_function(0x03, 0).unwrap().to_string(), "03"); + + // Parser accepts both forms and round-trips them. + assert_eq!(PciSlot::try_from("03.5").unwrap(), slot); + assert_eq!(PciSlot::try_from("03").unwrap(), PciSlot::new(0x03)); + + // Function > 7 is rejected by both the constructor and the parser. + assert!(PciSlot::with_function(0x03, 8).is_err()); + assert!(PciSlot::try_from("03.8").is_err()); + + // Device > 0x1f is rejected by both the constructor and the parser. + assert!(PciSlot::with_function(0x20, 0).is_err()); + assert!(PciSlot::try_from("20.0").is_err()); + + // Malformed forms are rejected. + assert!(PciSlot::try_from("03.").is_err()); + assert!(PciSlot::try_from("03.55").is_err()); + assert!(PciSlot::try_from(".5").is_err()); + } + #[test] fn test_pci_patch() { let pci_path_0 = PciPath::try_from("01/0a/05"); assert!(pci_path_0.is_ok()); let pci_path_unwrap = pci_path_0.unwrap(); - assert_eq!(pci_path_unwrap.slots[0].0, 1); - assert_eq!(pci_path_unwrap.slots[1].0, 10); - assert_eq!(pci_path_unwrap.slots[2].0, 5); - - let pci_path_01 = PciPath::new(vec![PciSlot(1), PciSlot(10), PciSlot(5)]); + assert_eq!(pci_path_unwrap.slots[0].device, 1); + assert_eq!(pci_path_unwrap.slots[1].device, 10); + assert_eq!(pci_path_unwrap.slots[2].device, 5); + + let pci_path_01 = PciPath::new(vec![ + PciSlot::new(1), + PciSlot::new(10), + PciSlot::new(5), + ]); assert!(pci_path_01.is_some()); let pci_path = pci_path_01.unwrap(); let pci_path_02 = pci_path.to_string(); @@ -213,10 +339,24 @@ mod tests { let dev_slot = pci_path.get_device_slot(); assert!(dev_slot.is_some()); - assert_eq!(dev_slot.unwrap().0, 5); + assert_eq!(dev_slot.unwrap().device, 5); let root_slot = pci_path.get_root_slot(); assert!(root_slot.is_some()); - assert_eq!(root_slot.unwrap().0, 1); + assert_eq!(root_slot.unwrap().device, 1); + } + + #[test] + fn test_pci_path_multifunction_roundtrip() { + // PciPath with a multi-function root slot (e.g. OpenVMM root + // port packed at device 0x03, function 5) round-trips through + // Display and TryFrom<&str>. + let mf_root = PciSlot::with_function(0x03, 5).unwrap(); + let endpoint = PciSlot::new(0); + let path = PciPath::new(vec![mf_root, endpoint]).unwrap(); + assert_eq!(path.to_string(), "03.5/00"); + + let parsed = PciPath::try_from("03.5/00").unwrap(); + assert_eq!(parsed, path); } } diff --git a/src/runtime-rs/crates/hypervisor/src/device/topology.rs b/src/runtime-rs/crates/hypervisor/src/device/topology.rs index 21e4701f1ec8..fa3772fd6aaf 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/topology.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/topology.rs @@ -378,7 +378,7 @@ impl PCIeTopology { pub fn insert_device(&mut self, ep: &mut PCIeEndpoint) -> Option { let to_pcipath = |v: u32| -> PciPath { PciPath { - slots: vec![PciSlot(v as u8)], + slots: vec![PciSlot::new(v as u8)], } }; diff --git a/src/runtime-rs/crates/hypervisor/src/openvmm/inner.rs b/src/runtime-rs/crates/hypervisor/src/openvmm/inner.rs index 8a7083daf261..2950f2fc8f6a 100644 --- a/src/runtime-rs/crates/hypervisor/src/openvmm/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/openvmm/inner.rs @@ -17,10 +17,10 @@ use tokio::sync::mpsc; use super::vmm_instance::VmmInstance; use super::{ - OPENVMM_BLOCK_HOTPLUG_PORT_COUNT, OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, + openvmm_port_pci_path, OPENVMM_BLOCK_HOTPLUG_PORT_COUNT, OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, OPENVMM_STATIC_PCI_PORT_COUNT, }; -use crate::device::pci_path::{PciPath, PciSlot}; +use crate::device::pci_path::PciPath; #[derive(Clone, Debug)] pub(crate) struct OpenVmmHotplugPort { @@ -30,9 +30,13 @@ pub(crate) struct OpenVmmHotplugPort { impl OpenVmmHotplugPort { fn new(index: u8) -> Self { - let root_slot = OPENVMM_STATIC_PCI_PORT_COUNT + index; - let pci_path = PciPath::new(vec![PciSlot::new(root_slot), PciSlot::new(0)]) - .expect("openvmm hotplug port PCI path must be non-empty"); + // OpenVMM packs root ports into multi-function device slots: + // port index `STATIC + index` lands at `device = (..)/8, + // function = (..)%8` of bus 0. See + // `openvmm_port_pci_path` for the rationale. + let port_index = OPENVMM_STATIC_PCI_PORT_COUNT + index; + let pci_path = openvmm_port_pci_path(port_index) + .expect("openvmm hotplug port PCI path must be valid"); Self { name: format!("{}{}", OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, index), diff --git a/src/runtime-rs/crates/hypervisor/src/openvmm/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/openvmm/inner_hypervisor.rs index ad48b4aa3de9..f57a828b2cb3 100644 --- a/src/runtime-rs/crates/hypervisor/src/openvmm/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/openvmm/inner_hypervisor.rs @@ -14,10 +14,13 @@ use std::{ use super::inner::OpenVmmInner; use super::vmm_instance::DeferredNetworkDevice; use super::{ - OPENVMM_BLOCK_HOTPLUG_PORT_COUNT, OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, OPENVMM_CONSOLE_PCI_PORT, - OPENVMM_NET_PCI_PORT, OPENVMM_ROOTFS_PCI_PORT, OPENVMM_SHAREFS_PCI_PORT, - OPENVMM_VFIO_COLDPLUG_PORT_COUNT, OPENVMM_VFIO_COLDPLUG_PORT_PREFIX, + openvmm_port_pci_path, OPENVMM_BLOCK_HOTPLUG_PORT_COUNT, OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, + OPENVMM_CONSOLE_PCI_PORT, OPENVMM_NET_PCI_PORT, OPENVMM_ROOTFS_PCI_PORT, + OPENVMM_SHAREFS_PCI_PORT, OPENVMM_STATIC_PCI_PORT_COUNT, OPENVMM_VFIO_COLDPLUG_PORT_COUNT, + OPENVMM_VFIO_COLDPLUG_PORT_PREFIX, }; +use crate::device::driver::vfio_device::DeviceAddress; +use crate::device::pci_path::PciPath; use crate::kernel_param::KernelParams; use crate::utils::{get_jailer_root, get_sandbox_path}; use crate::{MemoryConfig, VcpuThreadIds, VmmState, VM_ROOTFS_DRIVER_BLK}; @@ -159,6 +162,27 @@ impl OpenVmmInner { .context("memory size overflow")?; // PCIe root complex: low/high MMIO windows for BAR allocation. + // + // low_mmio is the 32-bit non-prefetchable BAR pool. Each cold-plug + // root port consumes a bridge MMIO32 window sized for that device's + // 32-bit MMIO BARs (rounded up to the bridge-alignment granularity); + // 64-bit prefetchable BARs (e.g. H100 BAR1/2 = 128 GB, BAR3/4 = 32 MB) + // go in high_mmio and do NOT count here. + // + // Worst case on the bench (NVIDIA A100/H100 + NVSwitch): + // 8x GPU BAR0 (16 MB ea.) = 128 MB + // 6x NVSwitch BAR0 (32 MB ea.) = 192 MB + // static + 24 block-hotplug + 16 vfio-coldplug bridge headers + // (each pre-reserves an aligned MMIO32 window even when empty) + // and 1 MB-aligned padding between bridges + // ~ 150-200 MB + // Total ≥ 512 MB; 640 MB gives ~2x headroom and still fits + // between PCI hole start (0xC000_0000) and ECAM (0xE800_0000+). + // + // The original 320 MB window (0xc000_0000..0xd400_0000) overflows + // with as little as 8 GPUs + 6 NVSwitches and the openvmm worker + // aborts with: "low_mmio MMIO exhaustion: need 0x14500000 bytes, + // have 0x14000000". let pcie_root_complexes = vec![PcieRootComplexConfig { index: 0, name: "rc0".to_string(), @@ -166,7 +190,7 @@ impl OpenVmmInner { start_bus: 0, end_bus: 127, low_mmio: PcieMmioRangeConfig::Fixed(ovmm_memory_range::MemoryRange::new( - 0xc000_0000..0xd400_0000, + 0xc000_0000..0xe800_0000, )), high_mmio: PcieMmioRangeConfig::Fixed(ovmm_memory_range::MemoryRange::new( 0x0020_3d30_0000..0x200f_3d30_0000, @@ -178,31 +202,48 @@ impl OpenVmmInner { PcieRootPortConfig { name: OPENVMM_ROOTFS_PCI_PORT.to_string(), hotplug: false, - acs_capabilities_supported: None, + // ACS (Access Control Services) capability bits the + // emulated root port advertises. 0x5f = SV | TB | RR + // | CR | UF | DT (everything except EC, the standard + // PCIe ACS bitmask for downstream ports). + // + // Without ACS visible on the upstream root port, the + // in-guest NVIDIA driver concludes that peer-to-peer + // DMA between assigned PCI devices is not safely + // supported and silently disables P2P (which leaves + // NVLink fabric down on multi-GPU baseboards, since + // fabricmanager needs cross-device DMA to program + // the switches). OpenVMM's CLI default when the user + // passes --pcie-root-port is also 0x5f for the same + // reason — kata previously left this None, which + // OpenVMM treats as "do not synthesize the ACS + // capability at all" (see + // openvmm/vm/devices/pci/pcie/src/port.rs:353). + acs_capabilities_supported: Some(0x5f), cxl: false, }, PcieRootPortConfig { name: OPENVMM_SHAREFS_PCI_PORT.to_string(), hotplug: false, - acs_capabilities_supported: None, + acs_capabilities_supported: Some(0x5f), cxl: false, }, PcieRootPortConfig { name: OPENVMM_NET_PCI_PORT.to_string(), hotplug: false, - acs_capabilities_supported: None, + acs_capabilities_supported: Some(0x5f), cxl: false, }, PcieRootPortConfig { name: super::OPENVMM_VSOCK_PCI_PORT.to_string(), hotplug: false, - acs_capabilities_supported: None, + acs_capabilities_supported: Some(0x5f), cxl: false, }, PcieRootPortConfig { name: OPENVMM_CONSOLE_PCI_PORT.to_string(), hotplug: false, - acs_capabilities_supported: None, + acs_capabilities_supported: Some(0x5f), cxl: false, }, ]; @@ -211,7 +252,7 @@ impl OpenVmmInner { ports.push(PcieRootPortConfig { name: format!("{}{}", OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, index), hotplug: true, - acs_capabilities_supported: None, + acs_capabilities_supported: Some(0x5f), cxl: false, }); } @@ -224,7 +265,7 @@ impl OpenVmmInner { ports.push(PcieRootPortConfig { name: format!("{}{}", OPENVMM_VFIO_COLDPLUG_PORT_PREFIX, index), hotplug: false, - acs_capabilities_supported: None, + acs_capabilities_supported: Some(0x5f), cxl: false, }); } @@ -374,6 +415,142 @@ impl OpenVmmInner { deferred_block_devices.push(dev.clone()); } } + crate::DeviceType::VfioModern(vfio_handle) => { + // Modern-VFIO cold-plug. This is the variant that + // prepare_coldplug_cdi_devices() and + // prepare_coldplug_raw_vfio_devices() actually emit, so + // this arm is what fires for kubelet CDI grants and for + // `ctr --device /dev/vfio/N` standalone containers. + // + // For each PCI function in the IOMMU group we: + // 1. Reserve the next pre-allocated vfio root port. + // 2. Open the group fd and hand it to openvmm. + // 3. Compute the guest PciPath [root_slot, 0] for that + // port and write it back onto the device's + // `config.guest_pci_path`. + // + // Step 3 is critical: downstream `handler_devices` in + // resource::manager_inner reads that field to build the + // agent's `vfio-pci` `device_options` + // ("HOST_BDF=GUEST_PCI_PATH"). With it unset the + // container create call fails with: + // "VFIO device has no guest PCI path assigned" + // even though openvmm successfully attached the device. + // + // `vfio_handle` is the Arc> the + // VfioModern enum variant carries directly, so it locks + // exactly like ch/inner_device.rs does. + let mut vfio_device = vfio_handle.lock().await; + + let host_path = vfio_device.config.host_path.clone(); + let group_devices: Vec<_> = if !vfio_device.device.devices.is_empty() { + vfio_device.device.devices.clone() + } else { + vec![vfio_device.device.primary.clone()] + }; + info!( + sl!(), + "openvmm: cold-plug VFIO group {} ({} device(s))", + host_path, + group_devices.len() + ); + + let mut primary_pci_path: Option = None; + + for dev_info in &group_devices { + // openvmm needs the full segment-qualified BDF + // (e.g. "0001:00:00.0") so it can look up + // /sys/bus/pci/devices/. Modern VFIO stores + // that as DeviceAddress::Pci(BdfAddress), whose + // Display impl prints exactly that form. + let host_bdf = match &dev_info.addr { + DeviceAddress::Pci(bdf) => bdf.to_string(), + other => { + warn!( + sl!(), + "openvmm: skipping non-PCI VFIO device {} in group {}", + other, + host_path + ); + continue; + } + }; + + if next_vfio_port >= OPENVMM_VFIO_COLDPLUG_PORT_COUNT { + return Err(anyhow!( + "openvmm: too many VFIO devices (limit {}), cannot cold-plug BDF {}", + OPENVMM_VFIO_COLDPLUG_PORT_COUNT, + host_bdf + )); + } + + let group_fd = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&host_path) + .with_context(|| { + format!( + "openvmm: failed to open VFIO group {} for BDF {}", + host_path, host_bdf + ) + })?; + + let port_index = next_vfio_port; + let port_name = format!( + "{}{}", + OPENVMM_VFIO_COLDPLUG_PORT_PREFIX, port_index + ); + next_vfio_port += 1; + + // openvmm's PCIe root-bus port layout (kata side): + // [0 .. OPENVMM_STATIC_PCI_PORT_COUNT) static + // [STATIC .. STATIC + BLOCK_HOTPLUG_PORT_COUNT) hp + // [STATIC + BLOCK_HOTPLUG .. + VFIO_COLDPLUG) vfio + // + // OpenVMM packs root ports into multi-function + // device slots (device = i/8, function = i%8) so + // the total port count is bounded by the MMIO + // budget, not PCI's 5-bit device-number field. + // The VFIO endpoint sits behind the port at + // function 0 of its secondary bus, which kata + // models as the second slot in the PciPath. + let openvmm_port_index = OPENVMM_STATIC_PCI_PORT_COUNT + + OPENVMM_BLOCK_HOTPLUG_PORT_COUNT + + port_index; + let pci_path = openvmm_port_pci_path(openvmm_port_index).context( + "openvmm: failed to build guest PciPath for VFIO device", + )?; + + info!( + sl!(), + "openvmm: assigning VFIO BDF {} to port {} (guest pci_path={})", + host_bdf, + port_name, + pci_path + ); + + pcie_devices.push(PcieDeviceConfig { + port_name, + resource: vfio_assigned_device_resources::VfioDeviceHandle { + pci_id: host_bdf, + group: group_fd, + } + .into_resource(), + }); + + if primary_pci_path.is_none() { + primary_pci_path = Some(pci_path); + } + } + + // handler_devices() exposes a single device per + // VfioDeviceModern (the IOMMU-group primary), so we + // record the primary's guest path here. Matches what + // ch/inner_device.rs does in its own cold-plug branch. + if let Some(pp) = primary_pci_path { + vfio_device.config.guest_pci_path = Some(pp); + } + } crate::DeviceType::Vfio(vfio_dev) => { // Cold-plug VFIO PCI pass-through. Each HostDevice in the // IOMMU group becomes its own PcieDeviceConfig on a @@ -397,12 +574,28 @@ impl OpenVmmInner { ); continue; } + if hostdev.domain.is_empty() { + warn!( + sl!(), + "openvmm: skipping VFIO device with empty PCI domain (BDF {}) in group {}", + hostdev.bus_slot_func, + host_path + ); + continue; + } + // OpenVMM expects the full PCI BDF including the segment/ + // domain (e.g. "0001:00:00.0") to resolve + // /sys/bus/pci/devices/. HostDevice splits + // these into `domain` ("0001") and `bus_slot_func` + // ("00:00.0"); recombine them here. + let full_bdf = + format!("{}:{}", hostdev.domain, hostdev.bus_slot_func); if next_vfio_port >= OPENVMM_VFIO_COLDPLUG_PORT_COUNT { return Err(anyhow!( "openvmm: too many VFIO devices (limit {}), cannot cold-plug BDF {}", OPENVMM_VFIO_COLDPLUG_PORT_COUNT, - hostdev.bus_slot_func + full_bdf )); } @@ -413,7 +606,7 @@ impl OpenVmmInner { .with_context(|| { format!( "openvmm: failed to open VFIO group {} for BDF {}", - host_path, hostdev.bus_slot_func + host_path, full_bdf ) })?; @@ -425,15 +618,13 @@ impl OpenVmmInner { info!( sl!(), - "openvmm: assigning VFIO BDF {} to port {}", - hostdev.bus_slot_func, - port_name + "openvmm: assigning VFIO BDF {} to port {}", full_bdf, port_name ); pcie_devices.push(PcieDeviceConfig { port_name, resource: vfio_assigned_device_resources::VfioDeviceHandle { - pci_id: hostdev.bus_slot_func.clone(), + pci_id: full_bdf, group: group_fd, } .into_resource(), diff --git a/src/runtime-rs/crates/hypervisor/src/openvmm/mod.rs b/src/runtime-rs/crates/hypervisor/src/openvmm/mod.rs index 3c2c52230dbb..1d9136a95020 100644 --- a/src/runtime-rs/crates/hypervisor/src/openvmm/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/openvmm/mod.rs @@ -37,12 +37,52 @@ pub(crate) const OPENVMM_VSOCK_PCI_PORT: &str = "rp3"; pub(crate) const OPENVMM_CONSOLE_PCI_PORT: &str = "rp4"; pub(crate) const OPENVMM_STATIC_PCI_PORT_COUNT: u8 = 5; pub(crate) const OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX: &str = "hp"; +// OpenVMM packs root ports into multi-function device slots (see +// `GenericPcieRootComplex::new` in microsoft/openvmm: +// `vm/devices/pci/pcie/src/root.rs`): port index `i` lands at PCI +// `device = i / 8, function = i % 8` on bus 0 of the root complex +// (assuming `first_port_device_number = 0`, which the openvmm worker +// uses when no IOMMU is attached). That gives us up to 8 root ports +// per device slot and 32 device slots, i.e. 256 ports total — far +// beyond anything kata needs. +// +// Each cold-plug root port still consumes a bridge MMIO32 window for +// the device's 32-bit BARs, so the practical ceiling is the size of +// the low_mmio window allocated above, not the PCI device-number +// space. The numbers below cover any Azure SKU we know about (HGX +// A100 8-GPU baseboard = 14 VFIO devices, HGX H100 = 16, plus IB VFs). pub(crate) const OPENVMM_BLOCK_HOTPLUG_PORT_COUNT: u8 = 24; /// Number of pre-reserved PCIe root ports for cold-plug VFIO pass-through /// devices (e.g., GPUs, NVSwitches). These ports are created with /// `hotplug: false` so the guest sees the assigned devices at boot. pub(crate) const OPENVMM_VFIO_COLDPLUG_PORT_PREFIX: &str = "vfio"; -pub(crate) const OPENVMM_VFIO_COLDPLUG_PORT_COUNT: u8 = 16; +pub(crate) const OPENVMM_VFIO_COLDPLUG_PORT_COUNT: u8 = 32; + +/// Map an OpenVMM PCIe root-port index (0..256) to the corresponding +/// guest PciPath. OpenVMM packs root ports into multi-function device +/// slots, so port `i` is at `device = i / 8, function = i % 8` on bus 0 +/// of the root complex. The endpoint device sits at function 0 of the +/// root port's secondary bus, which kata represents as the second +/// PciSlot in the path. +/// +/// We need this because the original single-function layout +/// (`root_slot = STATIC + BLOCK_HOTPLUG + port_index`) overflowed PCI's +/// 5-bit slot field (32 slot numbers) as soon as the shim allocated +/// more than 3 VFIO ports on a build that also keeps the full 24-slot +/// block-hotplug pool. The multi-function layout matches what OpenVMM +/// itself programs into the guest's PCIe config space, so the agent's +/// QOM-path → pci_path parser also accepts what we send. +pub(crate) fn openvmm_port_pci_path( + port_index: u8, +) -> anyhow::Result { + use crate::device::pci_path::{PciPath, PciSlot}; + let device = port_index / 8; + let function = port_index % 8; + let root = PciSlot::with_function(device, function)?; + let endpoint = PciSlot::new(0); + PciPath::new(vec![root, endpoint]) + .ok_or_else(|| anyhow::anyhow!("openvmm: failed to build PciPath for port {}", port_index)) +} /// The OpenVMM hypervisor struct, wrapping inner state behind a lock. pub struct OpenVmm { diff --git a/src/runtime-rs/crates/hypervisor/src/openvmm/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/openvmm/vmm_instance.rs index ed5da2cb3c5c..91839eef33ad 100644 --- a/src/runtime-rs/crates/hypervisor/src/openvmm/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/openvmm/vmm_instance.rs @@ -13,6 +13,7 @@ use ovmm_mesh::rpc::RpcSend; use ovmm_mesh_worker::RegisteredWorkers; use ovmm_vmm_core_defs::HaltReason; use tokio::sync::mpsc; +use tokio::task::JoinHandle; use vm_resource::IntoResource; use super::OPENVMM_VSOCK_PCI_PORT; @@ -34,7 +35,12 @@ extern crate openvmm_resources as _; pub(crate) struct VmmInstance { worker_handle: Option, worker_rpc: Option>, - _notify_recv: Option>, + /// Background task that consumes guest-halt notifications from the + /// in-process VmWorker and signals `exit_notify` so that `wait_vm()` + /// (and therefore the containerd-shim main loop) returns when the + /// guest powers off on its own (e.g. agent-initiated shutdown, + /// kernel panic, triple fault, OOM-killed init, etc.). + halt_watcher: Option>, exit_notify: Option>, } @@ -52,7 +58,7 @@ impl VmmInstance { VmmInstance { worker_handle: None, worker_rpc: None, - _notify_recv: None, + halt_watcher: None, exit_notify: Some(exit_notify), } } @@ -87,15 +93,30 @@ impl VmmInstance { .spawn(move || { // Set up tracing for the VmWorker thread. // Write openvmm tracing output to a log file for debugging. + // + // We install the subscriber globally (rather than thread-local + // via `set_default`) because `DefaultPool::run_with` may + // execute spawned tasks on its own internal threads where a + // thread-local subscriber would not be visible. + // + // `try_init()` is a no-op if a global subscriber is already + // installed (e.g. by an earlier sandbox in the same shim + // process). Kata normally creates a fresh shim per sandbox, + // so the first launch wins and configures verbosity. + // + // Verbosity is controlled by the `RUST_LOG` environment + // variable; default is `info`. For VM-launch debugging use + // e.g. `RUST_LOG=info,openvmm=debug,virt_mshv=debug`. if let Some(ref dir) = log_dir { let log_file_path = format!("{}/openvmm-worker.log", dir); if let Ok(file) = std::fs::File::create(&log_file_path) { - let subscriber = tracing_subscriber::fmt() + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let _ = tracing_subscriber::fmt() .with_writer(std::sync::Mutex::new(file)) .with_ansi(false) - .finish(); - // Use set_default (thread-local) not set_global_default - let _guard = tracing::subscriber::set_default(subscriber); + .with_env_filter(filter) + .try_init(); } } @@ -284,7 +305,7 @@ impl VmmInstance { self.worker_handle = Some(worker); self.worker_rpc = Some(rpc_send); - self._notify_recv = Some(notify_recv); + self.halt_watcher = Some(spawn_halt_watcher(notify_recv, self.exit_notify.clone())); Ok(()) } @@ -333,6 +354,14 @@ impl VmmInstance { /// Stop and teardown the VM. pub(crate) async fn stop(&mut self) -> Result<()> { + // Stop the halt watcher first so it does not race with the + // explicit exit_notify send below. Aborting the task is safe + // because it only borrows owned values (the mesh Receiver and + // a clone of the exit_notify Sender). + if let Some(handle) = self.halt_watcher.take() { + handle.abort(); + } + if let Some(mut worker_handle) = self.worker_handle.take() { worker_handle.stop(); if let Err(err) = worker_handle.join().await { @@ -343,7 +372,6 @@ impl VmmInstance { } } self.worker_rpc = None; - self._notify_recv = None; if let Some(exit_notify) = &self.exit_notify { let _ = exit_notify.try_send(0); @@ -352,3 +380,52 @@ impl VmmInstance { Ok(()) } } + +/// Spawn a tokio task that watches the OpenVMM halt-notification +/// channel and signals `exit_notify` when the guest reaches a terminal +/// halt state (anything other than a guest-initiated reset, which the +/// VmWorker handles internally via `automatic_guest_reset = true`). +/// +/// Without this watcher, `OpenVmm::wait_vm()` would block forever after +/// the guest powers off, because the in-process VmWorker has no child +/// process for the shim to reap and no other code reads the halt +/// channel. That stall surfaces as the kata-shim hanging on container +/// teardown. +fn spawn_halt_watcher( + mut notify_recv: ovmm_mesh::Receiver, + exit_notify: Option>, +) -> JoinHandle<()> { + tokio::spawn(async move { + loop { + match notify_recv.recv().await { + Ok(HaltReason::Reset) => { + // The VmWorker is configured with + // `automatic_guest_reset = true`, so it should + // handle resets internally and not forward them + // here. Be defensive in case that ever changes. + info!(sl!(), "openvmm: guest-initiated reset (ignored)"); + continue; + } + Ok(reason) => { + info!( + sl!(), + "openvmm: guest halted, signaling shim exit: {:?}", reason + ); + if let Some(tx) = &exit_notify { + let _ = tx.try_send(0); + } + return; + } + Err(err) => { + // The sender side was dropped — typically because + // `stop()` tore down the worker. Nothing more to do. + info!( + sl!(), + "openvmm: halt notification channel closed: {:?}", err + ); + return; + } + } + } + }) +} diff --git a/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs b/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs index 7fbcdb2e066c..40b36a4230c7 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/resource_inner.rs @@ -397,8 +397,40 @@ impl CgroupsResourceInner { // to the sandbox cgroup, it results in those threads being under // the overhead cgroup, and allowing them to consume more resources // than we have allocated for the sandbox. + // + // This check only applies to out-of-process VMMs (clh, qemu, fc): + // their vCPU threads belong to a child process that can be moved + // into the sandbox cgroup without disturbing the runtime, and a + // missing vCPU thread ID list at this point indicates a bug. + // + // For in-process VMMs (openvmm, and dragonball when sandbox + // _cgroup_only=false is opted into), the runtime IS the VMM: + // vCPU threads are threads of the runtime process and cannot be + // moved to a different cgroup without dragging the runtime + // (including the in-process VMM that allocated guest RAM) with + // them. Those drivers therefore return an empty VcpuThreadIds + // from `get_thread_ids` by design (see e.g. + // openvmm/inner_hypervisor.rs::get_thread_ids), which makes + // `update_sandbox_cgroups` return Ok(false). Treating that as + // fatal would make sandbox_cgroup_only=false unusable on those + // hypervisors, even though that setting is required for + // non-trivial guest RAM (otherwise the pod's memcg OOM-kills + // the in-process VMM during VM construction). + // + // Distinguish the two cases by asking the hypervisor for its + // PID list: an in-process VMM reports only the runtime's own + // PID, while an out-of-process VMM reports a distinct child. if self.overhead_cgroup.is_some() && !updated { - return Err(anyhow!("hypervisor cannot be moved to sandbox cgroup")); + let hv_pids = hypervisor.get_pids().await.unwrap_or_default(); + let runtime_pid = process::id(); + let in_process_vmm = + hv_pids.is_empty() || hv_pids.iter().all(|p| *p == runtime_pid); + if !in_process_vmm { + return Err(anyhow!("hypervisor cannot be moved to sandbox cgroup")); + } + // In-process VMM: leave the runtime (and its vCPU threads) + // in the overhead cgroup. The sandbox cgroup will materialise + // once container processes are added to it. } Ok(()) diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs b/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs index e0df15dfe4ca..f4259c9ac4a8 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs @@ -200,7 +200,11 @@ impl SwapTask { if let DeviceType::Block(device) = device_info { let ret = if let Some(pci_path) = device.config.pci_path.clone() { self.agent.add_swap(agent::types::AddSwapRequest { - pci_path: pci_path.slots.iter().map(|slot| slot.0 as u32).collect(), + pci_path: pci_path + .slots + .iter() + .map(|slot| slot.device as u32) + .collect(), }) } else if !device.config.virt_path.is_empty() { self.agent.add_swap_path(agent::types::AddSwapPathRequest { diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 30cf00f76130..dda0602ce25a 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -30,6 +30,7 @@ use containerd_shim_protos::events::task::{TaskExit, TaskOOM}; ))] use hypervisor::ch::CloudHypervisor; use hypervisor::device::topology::PCIePort; +use hypervisor::device::util::{get_host_path, DEVICE_TYPE_CHAR}; use hypervisor::remote::Remote; use hypervisor::VfioDeviceBase; use hypervisor::VsockConfig; @@ -227,7 +228,29 @@ impl VirtSandbox { None }; - let vfio_devices = self.prepare_coldplug_cdi_devices(sandbox_config).await?; + // Cold-plug VFIO devices for hypervisors that build the PCIe + // topology at VM-creation time (notably openvmm). Two sources + // are unioned, in declaration order: + // + // 1. CDI via the kubelet Pod Resources API + // (`prepare_coldplug_cdi_devices`). Upstream pattern; used + // when the device-plugin registers with the Pod Resources + // socket and emits CDI device nodes. This is the K8s path. + // 2. Raw OCI `linux.devices` + // (`prepare_coldplug_raw_vfio_devices`). Standalone-container + // path (`ctr --device /dev/vfio/0`). Aligned with the + // upstream PR that adds raw VFIO cold-plug support to + // runtime-rs. + // + // Each source independently gates on the `cold_plug_vfio` + // hypervisor config. Hot-plug back-ends (qemu, dragonball, ...) + // still pick devices up via the regular `handler_devices` + // post-start path. + let mut vfio_devices = self.prepare_coldplug_cdi_devices(sandbox_config).await?; + vfio_devices.extend( + self.prepare_coldplug_raw_vfio_devices(sandbox_config) + .await?, + ); if !vfio_devices.is_empty() { info!( sl!(), @@ -352,6 +375,100 @@ impl VirtSandbox { .collect()) } + // Raw VFIO cold-plug fallback for standalone containers + // (e.g. `ctr --device /dev/vfio/0`). Reads the OCI spec from the + // bundle and cold-plugs any VFIO char devices found in + // `linux.devices` before VM boot, mirroring the Go runtime's + // `coldOrHotPlugVFIO()`. Returns empty when `cold_plug_vfio` is not + // configured, when the bundle path is unavailable, or when the OCI + // spec is missing/has no eligible devices. + async fn prepare_coldplug_raw_vfio_devices( + &self, + sandbox_config: &SandboxConfig, + ) -> Result> { + let hypervisor_config = self.hypervisor.hypervisor_config().await; + let cold_plug_vfio = &hypervisor_config.device_info.cold_plug_vfio; + if cold_plug_vfio.is_empty() || cold_plug_vfio == "no-port" { + return Ok(Vec::new()); + } + + let port = match cold_plug_vfio.as_str() { + "root-port" => PCIePort::RootPort, + other => { + return Err(anyhow!( + "unsupported cold_plug_vfio value {:?}; only \"root-port\" is supported", + other + )) + } + }; + + let bundle = &sandbox_config.state.bundle; + if bundle.is_empty() { + return Ok(Vec::new()); + } + + let spec_path = format!("{}/{}", bundle, spec::OCI_SPEC_CONFIG_FILE_NAME); + let oci_spec = match oci::Spec::load(&spec_path) { + Ok(s) => s, + Err(e) => { + info!( + sl!(), + "no OCI spec at {:?}: {:?}, skipping raw VFIO cold-plug", spec_path, e + ); + return Ok(Vec::new()); + } + }; + + let linux_devices = oci_spec + .linux() + .as_ref() + .and_then(|l| l.devices().as_ref()) + .cloned() + .unwrap_or_default(); + + let mut vfio_configs = Vec::new(); + for d in linux_devices.iter() { + if d.typ() != oci::LinuxDeviceType::C { + continue; + } + let host_path = match get_host_path(DEVICE_TYPE_CHAR, d.major(), d.minor()) { + Ok(p) => p, + Err(e) => { + warn!( + sl!(), + "failed to resolve host path for {:?}: {:?}", + d.path(), + e + ); + continue; + } + }; + // Only process VFIO passthrough devices under /dev/vfio/*. + // Skip non-VFIO devices and the legacy VFIO control node + // (/dev/vfio/vfio). + if !host_path.starts_with("/dev/vfio/") || host_path == "/dev/vfio/vfio" { + continue; + } + vfio_configs.push(VfioDeviceBase { + host_path: host_path.clone(), + iommu_group_devnode: PathBuf::from(&host_path), + dev_type: "c".to_string(), + port, + hostdev_prefix: "vfio_device".to_owned(), + ..Default::default() + }); + } + info!( + sl!(), + "raw VFIO cold-plug candidates: {:?}", vfio_configs + ); + + Ok(vfio_configs + .into_iter() + .map(ResourceConfig::VfioDeviceModern) + .collect()) + } + async fn prepare_network_resource( &self, network_env: &SandboxNetworkEnv, diff --git a/src/runtime/cmd/kata-runtime/kata-check_amd64.go b/src/runtime/cmd/kata-runtime/kata-check_amd64.go index 8083b01be16c..283749850237 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_amd64.go +++ b/src/runtime/cmd/kata-runtime/kata-check_amd64.go @@ -86,6 +86,8 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { fallthrough case vc.DragonballHypervisor: fallthrough + case vc.OpenvmmHypervisor: + fallthrough case vc.QemuHypervisor: archRequiredCPUFlags = map[string]string{ cpuFlagVMX: "Virtualization support", @@ -198,6 +200,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { fallthrough case vc.StratovirtHypervisor: fallthrough + case vc.OpenvmmHypervisor: + fallthrough case vc.FirecrackerHypervisor: return kvmIsUsable() case vc.RemoteHypervisor: diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 397415c95531..b3e1d5863020 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -53,6 +53,7 @@ const ( dragonballHypervisorTableType = "dragonball" stratovirtHypervisorTableType = "stratovirt" remoteHypervisorTableType = "remote" + openvmmHypervisorTableType = "openvmm" // the maximum amount of PCI bridges that can be cold plugged in a VM maxPCIBridges uint32 = 5 @@ -182,31 +183,31 @@ type hypervisor struct { } type runtime struct { - InterNetworkModel string `toml:"internetworking_model"` - JaegerEndpoint string `toml:"jaeger_endpoint"` - JaegerUser string `toml:"jaeger_user"` - JaegerPassword string `toml:"jaeger_password"` - VfioMode string `toml:"vfio_mode"` - GuestSeLinuxLabel string `toml:"guest_selinux_label"` - SandboxBindMounts []string `toml:"sandbox_bind_mounts"` - Experimental []string `toml:"experimental"` - Tracing bool `toml:"enable_tracing"` - DisableNewNetNs bool `toml:"disable_new_netns"` - DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` - EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"` - Debug bool `toml:"enable_debug"` - SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` - StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` - StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"` + InterNetworkModel string `toml:"internetworking_model"` + JaegerEndpoint string `toml:"jaeger_endpoint"` + JaegerUser string `toml:"jaeger_user"` + JaegerPassword string `toml:"jaeger_password"` + VfioMode string `toml:"vfio_mode"` + GuestSeLinuxLabel string `toml:"guest_selinux_label"` + SandboxBindMounts []string `toml:"sandbox_bind_mounts"` + Experimental []string `toml:"experimental"` + Tracing bool `toml:"enable_tracing"` + DisableNewNetNs bool `toml:"disable_new_netns"` + DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` + EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"` + Debug bool `toml:"enable_debug"` + SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` + StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` + StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"` StaticSandboxWorkloadDefaultVcpus float32 `toml:"static_sandbox_default_workload_vcpus"` - EnablePprof bool `toml:"enable_pprof"` - DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` - EmptyDirMode string `toml:"emptydir_mode"` - CreateContainerTimeout uint64 `toml:"create_container_timeout"` - DanConf string `toml:"dan_conf"` - ForceGuestPull bool `toml:"experimental_force_guest_pull"` - PodResourceAPISock string `toml:"pod_resource_api_sock"` - KubeletRootDir string `toml:"kubelet_root_dir"` + EnablePprof bool `toml:"enable_pprof"` + DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` + EmptyDirMode string `toml:"emptydir_mode"` + CreateContainerTimeout uint64 `toml:"create_container_timeout"` + DanConf string `toml:"dan_conf"` + ForceGuestPull bool `toml:"experimental_force_guest_pull"` + PodResourceAPISock string `toml:"pod_resource_api_sock"` + KubeletRootDir string `toml:"kubelet_root_dir"` } // emptyDirMode returns a valid emptydir_mode value, defaulting to shared-fs @@ -1281,6 +1282,54 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +// newOpenvmmHypervisorConfig parses an [hypervisor.openvmm] section. +// +// OpenVMM is an in-process Rust VMM driven by the runtime-rs shim; the legacy +// Go runtime never launches it. This parser exists only so the Go kata-runtime +// CLI can read the config for diagnostic, read-only commands (list, env, +// check). It deliberately mirrors newDragonballHypervisorConfig (the other +// runtime-rs-only backend) rather than newClhHypervisorConfig: it skips the +// h.path() hypervisor-binary existence check (OpenVMM has no external binary) +// and the Cloud-Hypervisor-specific shared-FS validation (which would emit a +// misleading "Cloud Hypervisor does not support ..." error for an OpenVMM +// config). Note this does NOT make the Go runtime able to start an OpenVMM VM +// or to 'kata-runtime exec' into a runtime-rs sandbox – those are owned by the +// Rust shim and its separate state store. +func newOpenvmmHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err + } + + image, err := h.image() + if err != nil { + return vc.HypervisorConfig{}, err + } + + rootfsType, err := h.rootfsType() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernelParams := h.kernelParams() + + return vc.HypervisorConfig{ + KernelPath: kernel, + ImagePath: image, + RootfsType: rootfsType, + KernelParams: vc.DeserializeParams(vc.KernelParamFields(kernelParams)), + KernelVerityParams: h.kernelVerityParams(), + NumVCPUsF: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + EntropySource: h.GetEntropySource(), + ColdPlugVFIO: h.coldPlugVFIO(), + HotPlugVFIO: h.hotPlugVFIO(), + Debug: h.Debug, + }, nil +} + func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { hypervisor, err := h.path() if err != nil { @@ -1445,6 +1494,14 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case clhHypervisorTableType: config.HypervisorType = vc.ClhHypervisor hConfig, err = newClhHypervisorConfig(hypervisor) + case openvmmHypervisorTableType: + // OpenVMM is a runtime-rs-only (Rust) VMM, like Dragonball: the + // Go runtime never launches it (NewHypervisor maps it to the mock + // hypervisor) and only parses the config for read-only CLI + // commands. Report it as its own type rather than masquerading + // as clh. + config.HypervisorType = vc.OpenvmmHypervisor + hConfig, err = newOpenvmmHypervisorConfig(hypervisor) case dragonballHypervisorTableType: config.HypervisorType = vc.DragonballHypervisor hConfig, err = newDragonballHypervisorConfig(hypervisor) diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 7c109c1aefb1..a902072a53ab 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -1845,6 +1845,7 @@ func TestUpdateRuntimeConfigHypervisor(t *testing.T) { var entries = []tableTypeEntry{ {clhHypervisorTableType, true}, {dragonballHypervisorTableType, true}, + {openvmmHypervisorTableType, true}, {firecrackerHypervisorTableType, true}, {qemuHypervisorTableType, true}, {"foo", false}, diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 71b187e069c4..fe066ba399e2 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -56,6 +56,16 @@ const ( // DragonballHypervisor is the Dragonball hypervisor. DragonballHypervisor HypervisorType = "dragonball" + // OpenvmmHypervisor is the OpenVMM hypervisor. + // + // Like Dragonball, OpenVMM is only driven by the runtime-rs (Rust) shim; + // the legacy Go runtime never launches it. The type exists so the Go + // kata-runtime CLI can parse [hypervisor.openvmm] sections and report the + // backend honestly for read-only commands (list, env, check). It maps to + // the mock hypervisor in NewHypervisor() because the Go runtime has no + // OpenVMM VM driver of its own. + OpenvmmHypervisor HypervisorType = "openvmm" + // VirtFrameworkHypervisor is the Darwin Virtualization.framework hypervisor VirtframeworkHypervisor HypervisorType = "virtframework" @@ -388,6 +398,9 @@ func (hType *HypervisorType) Set(value string) error { case "dragonball": *hType = DragonballHypervisor return nil + case "openvmm": + *hType = OpenvmmHypervisor + return nil case "virtframework": *hType = VirtframeworkHypervisor return nil @@ -413,6 +426,8 @@ func (hType *HypervisorType) String() string { return string(ClhHypervisor) case StratovirtHypervisor: return string(StratovirtHypervisor) + case OpenvmmHypervisor: + return string(OpenvmmHypervisor) case RemoteHypervisor: return string(RemoteHypervisor) case MockHypervisor: diff --git a/src/runtime/virtcontainers/hypervisor_linux.go b/src/runtime/virtcontainers/hypervisor_linux.go index b1ec0cf4aac7..67aeb099247d 100644 --- a/src/runtime/virtcontainers/hypervisor_linux.go +++ b/src/runtime/virtcontainers/hypervisor_linux.go @@ -38,6 +38,11 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) { return &stratovirt{}, nil case DragonballHypervisor: return &mockHypervisor{}, nil + case OpenvmmHypervisor: + // OpenVMM is a runtime-rs-only (Rust) VMM; the legacy Go runtime + // never launches it, so – like Dragonball – it resolves to the mock + // hypervisor. The Go CLI only parses/reports the config. + return &mockHypervisor{}, nil case RemoteHypervisor: return &remoteHypervisor{}, nil case MockHypervisor: diff --git a/tests/hypervisor_helpers.sh b/tests/hypervisor_helpers.sh index 24575a55fa41..c8bc0c4326bb 100644 --- a/tests/hypervisor_helpers.sh +++ b/tests/hypervisor_helpers.sh @@ -17,6 +17,7 @@ ALL_HYPERVISORS=( "clh" "clh-runtime-rs" "dragonball" + "openvmm" "qemu" "qemu-runtime-rs" "qemu-nvidia-gpu" diff --git a/tools/packaging/kata-deploy/binary/src/runtime/containerd.rs b/tools/packaging/kata-deploy/binary/src/runtime/containerd.rs index 9bfa3ff1c70a..6f786083fe9f 100644 --- a/tools/packaging/kata-deploy/binary/src/runtime/containerd.rs +++ b/tools/packaging/kata-deploy/binary/src/runtime/containerd.rs @@ -112,6 +112,34 @@ fn is_containerd_v3_config(pluginid: &str) -> bool { pluginid == CONTAINERD_V3_RUNTIME_PLUGIN_ID } +/// Resolve the CRI runtime plugin id to use when registering a Kata runtime. +/// +/// When writing into a *drop-in* file, the plugin id must match the schema +/// containerd actually interprets the drop-in at — NOT the `version` declared +/// in the legacy main `/etc/containerd/config.toml`. +/// +/// Drop-in files are version-less, and containerd interprets them at its +/// native config schema. `use_drop_in` is only ever true for containerd 2.0 +/// or newer (see `is_containerd_capable_of_using_drop_in_files`), whose +/// native schema is v3 — even when the on-disk main config still declares +/// `version = 2`. Picking the v2 plugin id (`io.containerd.grpc.v1.cri`) from +/// that stale `version = 2` would make containerd's v3 loader treat the +/// imported runtime as an unknown plugin and silently drop it, so the runtime +/// never registers and never appears in `containerd config dump`. +/// +/// For non-drop-in installs (containerd 1.x) the main config IS the runtime +/// registration and is interpreted at its declared version, so fall back to +/// reading that version from the file. +fn resolve_runtime_plugin_id<'a>(paths: &'a ContainerdPaths, runtime: &str) -> Result<&'a str> { + if let Some(plugin_id) = paths.plugin_id.as_deref() { + return Ok(plugin_id); + } + if paths.use_drop_in { + return Ok(CONTAINERD_V3_RUNTIME_PLUGIN_ID); + } + get_containerd_pluginid(&paths.config_file, runtime) +} + /// Maps the runtime plugin ID (from `get_containerd_pluginid` / K3s `paths.plugin_id`) to the table where /// disable_snapshot_annotations lives. In v3 that's the *images* plugin; in v2 the CRI .containerd subtable. pub(crate) fn pluginid_for_snapshotter_annotations( @@ -232,10 +260,7 @@ pub async fn configure_containerd_runtime( let paths = config.get_containerd_paths(runtime).await?; let configuration_file = get_containerd_output_path(&paths); - let pluginid = match paths.plugin_id.as_deref() { - Some(plugin_id) => plugin_id, - None => get_containerd_pluginid(&paths.config_file, runtime)?, - }; + let pluginid = resolve_runtime_plugin_id(&paths, runtime)?; log::info!( "configure_containerd_runtime: Writing to {:?}, pluginid={}", @@ -312,10 +337,7 @@ pub async fn configure_custom_containerd_runtime( let paths = config.get_containerd_paths(runtime).await?; let configuration_file = get_containerd_output_path(&paths); - let pluginid = match paths.plugin_id.as_deref() { - Some(plugin_id) => plugin_id, - None => get_containerd_pluginid(&paths.config_file, runtime)?, - }; + let pluginid = resolve_runtime_plugin_id(&paths, runtime)?; log::info!( "configure_custom_containerd_runtime: Writing to {:?}, pluginid={}", @@ -715,6 +737,59 @@ mod tests { ); } + fn make_paths( + config_file: &str, + use_drop_in: bool, + plugin_id: Option<&str>, + ) -> ContainerdPaths { + ContainerdPaths { + config_file: config_file.to_string(), + backup_file: format!("{config_file}.bak"), + imports_file: Some(config_file.to_string()), + drop_in_file: "/opt/kata/containerd/config.d/kata-deploy.toml".to_string(), + use_drop_in, + plugin_id: plugin_id.map(|s| s.to_string()), + } + } + + /// A drop-in is version-less and interpreted by containerd at its native + /// (v3) schema, even when the legacy main config still declares + /// `version = 2`. The runtime must therefore be registered under the v3 + /// CRI plugin id, otherwise containerd silently drops it. + #[test] + fn test_resolve_runtime_plugin_id_drop_in_forces_v3_despite_version_2_main() { + let f = NamedTempFile::new().unwrap(); + std::fs::write(f.path(), "version = 2\n").unwrap(); + let paths = make_paths(f.path().to_str().unwrap(), true, None); + assert_eq!( + resolve_runtime_plugin_id(&paths, "containerd").unwrap(), + CONTAINERD_V3_RUNTIME_PLUGIN_ID + ); + } + + /// Without drop-in (containerd 1.x), the main config IS the registration + /// and is interpreted at its declared version. + #[test] + fn test_resolve_runtime_plugin_id_no_drop_in_reads_main_version() { + let f = NamedTempFile::new().unwrap(); + std::fs::write(f.path(), "version = 2\n").unwrap(); + let paths = make_paths(f.path().to_str().unwrap(), false, None); + assert_eq!( + resolve_runtime_plugin_id(&paths, "containerd").unwrap(), + CONTAINERD_V2_CRI_PLUGIN_ID + ); + } + + /// An explicit plugin id (K3s/RKE2 templates) always wins. + #[test] + fn test_resolve_runtime_plugin_id_explicit_plugin_id_wins() { + let paths = make_paths("/unused", true, Some(CONTAINERD_V2_CRI_PLUGIN_ID)); + assert_eq!( + resolve_runtime_plugin_id(&paths, "k3s").unwrap(), + CONTAINERD_V2_CRI_PLUGIN_ID + ); + } + /// CRI images runtime_platforms snapshotter is set only for v3 config when a snapshotter is configured. #[rstest] #[case(CONTAINERD_V3_RUNTIME_PLUGIN_ID, Some("\"nydus\""), "kata-qemu", true)]