pci_core/pcie: allow advertising support for PASID (TLP prefixing) in ports/switches

Guide/src/reference/openvmm/management/cli.md

@@ -303,6 +303,8 @@ name:
   root port. The value can be decimal or hexadecimal. Default is `0x005f`.
   Use `acs=0` to disable ACS for a root port.
 - `cxl`: marks the root port as CXL-capable.
+- `pasid`: advertises support for TLP prefixing (such as for guest PASID
+  behind a virtual IOMMU)
 
 `--pcie-switch` accepts optional comma-separated options as well:
 
@@ -315,6 +317,8 @@ name:
 - `acs=<mask>`: ACS capability mask requested for downstream switch ports.
   The upstream switch port does not expose ACS. Default is `0x005f`.
   Use `acs=0` to disable ACS for switch downstream ports.
+- `pasid`: advertises support for TLP prefixing (such as for guest PASID
+  behind a virtual IOMMU)
 
 ### Generic initiators
 

openvmm/openvmm_core/src/worker/dispatch.rs

@@ -2099,7 +2099,7 @@ impl InitializedVm {
                             let root_port_definitions = rc
                                 .ports
                                 .iter()
-                                .map(pcie_topology::build_root_port_definition)
+                                .map(pcie_topology::build_port_definition)
                                 .collect();
                             GenericPcieRootComplex::builder(
                                 &mut services.register_mmio(),
@@ -2227,7 +2227,7 @@ impl InitializedVm {
                     let downstream_ports = switch
                         .ports
                         .iter()
-                        .map(pcie_topology::build_root_port_definition)
+                        .map(pcie_topology::build_port_definition)
                         .collect();
                     let definition = pcie::switch::GenericPcieSwitchDefinition {
                         name: switch.name.clone().into(),

openvmm/openvmm_core/src/worker/dispatch/pcie_topology.rs

@@ -11,34 +11,39 @@ use cxl_spec::pci_registers::spec::flex_bus_port_dvsec::CxlFlexBusPortDvsecCapab
 use openvmm_defs::config::PciePortConfig;
 use openvmm_defs::config::PcieRootComplexConfig;
 use pci_core::spec::caps::acs::DEFAULT_ACS_CAP_MASK;
+use pci_core::spec::caps::pci_express::MaxEndEndTlpPrefixes;
 use pcie::GenericPciePortDefinition;
 use pcie::PciePortSettings;
 
-/// Builds root-port PCIe settings from manifest flags.
+/// Builds port PCIe settings from manifest flags.
 ///
 /// When CXL is enabled, emit a default Flex Bus capability advertising both
 /// cache and memory support.
-fn build_root_port_settings(rp_cfg: &PciePortConfig) -> PciePortSettings {
+///
+/// When PASID is enabled, advertise support for up to four TLP prefixes to
+/// work for both switch and root ports.
+fn build_port_settings(port_cfg: &PciePortConfig) -> PciePortSettings {
     PciePortSettings {
-        acs_capabilities_supported: rp_cfg
+        acs_capabilities_supported: port_cfg
             .acs_capabilities_supported
             .unwrap_or(DEFAULT_ACS_CAP_MASK),
-        cxl_flex_bus_port_capability: rp_cfg.cxl.then_some(
+        cxl_flex_bus_port_capability: port_cfg.cxl.then_some(
             CxlFlexBusPortDvsecCapability::new()
                 .with_cache_capable(true)
                 .with_mem_capable(true),
         ),
+        tlp_prefixing_supported: port_cfg.pasid.then_some(MaxEndEndTlpPrefixes::Four),
     }
 }
 
-/// Converts a manifest root-port entry into the runtime root-port definition.
-pub(super) fn build_root_port_definition(rp_cfg: &PciePortConfig) -> GenericPciePortDefinition {
-    let settings = build_root_port_settings(rp_cfg);
+/// Converts a manifest port entry into the runtime port definition.
+pub(super) fn build_port_definition(port_cfg: &PciePortConfig) -> GenericPciePortDefinition {
+    let settings = build_port_settings(port_cfg);
 
     GenericPciePortDefinition {
-        name: rp_cfg.name.as_str().into(),
-        devfn: rp_cfg.devfn,
-        hotplug: rp_cfg.hotplug,
+        name: port_cfg.name.as_str().into(),
+        devfn: port_cfg.devfn,
+        hotplug: port_cfg.hotplug,
         settings,
     }
 }

openvmm/openvmm_defs/src/config.rs

@@ -263,6 +263,8 @@ pub struct PciePortConfig {
     /// Runtime port construction derives required BAR/subregion layout from
     /// this flag (currently CXL component registers for BAR0).
     pub cxl: bool,
+    /// Enables PASID support for functions downstream of this port.
+    pub pasid: bool,
 }
 
 #[derive(Debug, MeshPayload)]

openvmm/openvmm_entry/src/cli_args.rs

@@ -1150,6 +1150,7 @@ Options:
     `hotplug`                      enable hotplug support for this root port
     `acs=<mask>`                   ACS capability bitmask (u16, decimal or 0x-prefixed hex)
     `cxl`                          configure this root port as CXL-capable
+    `pasid`                        configure this port to support PASID for downstream devices
 "#)]
     #[clap(long, conflicts_with("pcat"))]
     pub pcie_root_port: Vec<PcieRootPortCli>,
@@ -1173,6 +1174,9 @@ Examples:
     # Enable hotplug on all downstream switch ports of switch0
     --pcie-switch rp0:switch0,hotplug
 
+    # Enable PASID on all downstream switch ports of switch0
+    --pcie-switch rp0:switch0,pasid
+
 Syntax: <port_name>:<name>[,opt,opt=arg,...]
 
     port_name can be:
@@ -1183,6 +1187,7 @@ Options:
     `hotplug`                       enable hotplug support for all downstream switch ports
     `num_downstream_ports=<value>`  number of downstream ports, default 4
     `acs=<mask>`                    ACS capability bitmask for downstream switch ports
+    `pasid`                         configure this port to support PASID for downstream devices
 "#)]
     #[clap(long, conflicts_with("pcat"))]
     pub pcie_switch: Vec<GenericPcieSwitchCli>,
@@ -2881,6 +2886,7 @@ pub struct PcieRootPortCli {
     pub hotplug: bool,
     pub acs_capabilities_supported: Option<u16>,
     pub cxl: bool,
+    pub pasid: bool,
 }
 
 /// A colon-joined `parent:child` name pair used as the positional head of
@@ -2944,6 +2950,8 @@ struct RootPortArgs {
     acs: Option<AcsMask>,
     #[kv(flag)]
     cxl: bool,
+    #[kv(flag)]
+    pasid: bool,
 }
 
 impl FromStr for PcieRootPortCli {
@@ -2958,6 +2966,7 @@ impl FromStr for PcieRootPortCli {
             hotplug: args.hotplug,
             acs_capabilities_supported: args.acs.map(|a| a.0),
             cxl: args.cxl,
+            pasid: args.pasid,
         })
     }
 }
@@ -2999,6 +3008,7 @@ pub struct GenericPcieSwitchCli {
     pub num_downstream_ports: u8,
     pub hotplug: bool,
     pub acs_capabilities_supported: Option<u16>,
+    pub pasid: bool,
 }
 
 /// Raw `--pcie-switch` options, mapped into [`GenericPcieSwitchCli`].
@@ -3011,6 +3021,8 @@ struct SwitchArgs {
     #[kv(flag)]
     hotplug: bool,
     acs: Option<AcsMask>,
+    #[kv(flag)]
+    pasid: bool,
 }
 
 impl FromStr for GenericPcieSwitchCli {
@@ -3024,6 +3036,7 @@ impl FromStr for GenericPcieSwitchCli {
             num_downstream_ports: args.num_downstream_ports,
             hotplug: args.hotplug,
             acs_capabilities_supported: args.acs.map(|a| a.0),
+            pasid: args.pasid,
         })
     }
 }
@@ -4302,6 +4315,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
             }
         );
 
@@ -4314,6 +4328,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
             }
         );
 
@@ -4327,6 +4342,7 @@ mod tests {
                 hotplug: true,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
             }
         );
 
@@ -4339,6 +4355,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: Some(0),
                 cxl: false,
+                pasid: false,
             }
         );
 
@@ -4351,6 +4368,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: Some(0x005f),
                 cxl: false,
+                pasid: false,
             }
         );
 
@@ -4363,6 +4381,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: None,
                 cxl: true,
+                pasid: false,
             }
         );
 
@@ -4376,6 +4395,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
             }
         );
         assert_eq!(
@@ -4387,6 +4407,7 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
             }
         );
         assert_eq!(
@@ -4398,6 +4419,20 @@ mod tests {
                 hotplug: false,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
+            }
+        );
+
+        assert_eq!(
+            PcieRootPortCli::from_str("my_rc:port8,pasid").unwrap(),
+            PcieRootPortCli {
+                root_complex_name: "my_rc".to_string(),
+                name: "port8".to_string(),
+                devfn: None,
+                hotplug: false,
+                acs_capabilities_supported: None,
+                cxl: false,
+                pasid: true,
             }
         );
 
@@ -4412,6 +4447,7 @@ mod tests {
         assert!(PcieRootPortCli::from_str("rc0:rp0,addr=0.8").is_err());
         assert!(PcieRootPortCli::from_str("rc0:rp0,addr=1.2.3").is_err());
         assert!(PcieRootPortCli::from_str("rc0:rp0,addr").is_err());
+        assert!(PcieRootPortCli::from_str("rc0:rp0,pasid=foo").is_err());
     }
 
     #[test]
@@ -4453,6 +4489,7 @@ mod tests {
                 num_downstream_ports: 4,
                 hotplug: false,
                 acs_capabilities_supported: None,
+                pasid: false,
             }
         );
 
@@ -4464,6 +4501,7 @@ mod tests {
                 num_downstream_ports: 4,
                 hotplug: false,
                 acs_capabilities_supported: None,
+                pasid: false,
             }
         );
 
@@ -4475,6 +4513,7 @@ mod tests {
                 num_downstream_ports: 8,
                 hotplug: false,
                 acs_capabilities_supported: None,
+                pasid: false,
             }
         );
 
@@ -4487,6 +4526,7 @@ mod tests {
                 num_downstream_ports: 4,
                 hotplug: false,
                 acs_capabilities_supported: None,
+                pasid: false,
             }
         );
 
@@ -4499,6 +4539,7 @@ mod tests {
                 num_downstream_ports: 4,
                 hotplug: true,
                 acs_capabilities_supported: None,
+                pasid: false,
             }
         );
 
@@ -4511,6 +4552,7 @@ mod tests {
                 num_downstream_ports: 8,
                 hotplug: true,
                 acs_capabilities_supported: None,
+                pasid: false,
             }
         );
 
@@ -4522,6 +4564,7 @@ mod tests {
                 num_downstream_ports: 4,
                 hotplug: false,
                 acs_capabilities_supported: Some(0),
+                pasid: false,
             }
         );
 
@@ -4533,6 +4576,19 @@ mod tests {
                 num_downstream_ports: 4,
                 hotplug: false,
                 acs_capabilities_supported: Some(95),
+                pasid: false,
+            }
+        );
+
+        assert_eq!(
+            GenericPcieSwitchCli::from_str("rp0:switch0,pasid").unwrap(),
+            GenericPcieSwitchCli {
+                port_name: "rp0".to_string(),
+                name: "switch0".to_string(),
+                num_downstream_ports: 4,
+                hotplug: false,
+                acs_capabilities_supported: None,
+                pasid: true,
             }
         );
 
@@ -4544,6 +4600,7 @@ mod tests {
         assert!(GenericPcieSwitchCli::from_str("rp0:switch0,num_downstream_ports=bad").is_err());
         assert!(GenericPcieSwitchCli::from_str("rp0:switch0,num_downstream_ports=").is_err());
         assert!(GenericPcieSwitchCli::from_str("rp0:switch0,invalid_flag").is_err());
+        assert!(GenericPcieSwitchCli::from_str("rp0:switch0,pasid=bar").is_err());
     }
 
     #[test]

openvmm/openvmm_entry/src/lib.rs

@@ -220,6 +220,7 @@ fn build_switch_list(all_switches: &[cli_args::GenericPcieSwitchCli]) -> Vec<Pci
                     hotplug: switch_cli.hotplug,
                     acs_capabilities_supported: switch_cli.acs_capabilities_supported,
                     cxl: false,
+                    pasid: switch_cli.pasid,
                 })
                 .collect(),
         })
@@ -891,6 +892,7 @@ async fn vm_config_from_command_line(
                 hotplug: port_cli.hotplug,
                 acs_capabilities_supported: port_cli.acs_capabilities_supported,
                 cxl: port_cli.cxl,
+                pasid: port_cli.pasid,
             })
             .collect();
 

openvmm/openvmm_entry/src/ttrpc/mod.rs

@@ -1531,6 +1531,7 @@ async fn build_pcie_topology(
                 hotplug,
                 acs_capabilities_supported: None,
                 cxl: false,
+                pasid: false,
             });
             if let Some(attached) = attached {
                 walk_pcie_attachment(port_name, attached, &mut switches, &mut pending_devices)?;
@@ -1600,6 +1601,7 @@ fn walk_pcie_attachment(
                     hotplug,
                     acs_capabilities_supported: None,
                     cxl: false,
+                    pasid: false,
                 });
                 if let Some(attached) = attached {
                     children.push((downstream_name, attached));

petri/src/vm/openvmm/modify.rs

@@ -441,6 +441,7 @@ impl PetriVmConfigOpenVmm {
                         hotplug: true,
                         acs_capabilities_supported: Some(0),
                         cxl: false,
+                        pasid: false,
                     })
                     .collect();
 
@@ -486,6 +487,7 @@ impl PetriVmConfigOpenVmm {
                     hotplug,
                     acs_capabilities_supported: Some(0),
                     cxl: false,
+                    pasid: false,
                 })
                 .collect(),
         });