Emulated eXtensible Host Controller Interface (xHCI) host controller

bin/propolis-server/src/lib/initializer.rs

@@ -42,6 +42,7 @@ use propolis::hw::qemu::{
     ramfb,
 };
 use propolis::hw::uart::LpcUart;
+use propolis::hw::usb::xhci;
 use propolis::hw::{nvme, virtio};
 use propolis::intr_pins;
 use propolis::vmm::{self, Builder, Machine};
@@ -98,6 +99,9 @@ pub enum MachineInitError {
     #[error("failed to specialize CPUID for vcpu {0}")]
     CpuidSpecializationFailed(i32, #[source] propolis::cpuid::SpecializeError),
 
+    #[error("xHC USB root hub port number invalid: {0}")]
+    UsbRootHubPortNumberInvalid(String),
+
     #[cfg(feature = "falcon")]
     #[error("softnpu p9 device missing")]
     SoftNpuP9Missing,
@@ -814,6 +818,45 @@ impl MachineInitializer<'_> {
         Ok(())
     }
 
+    /// Initialize xHCI controllers, connect any USB devices given in the spec,
+    /// add them to the device map, and attach them to the chipset.
+    pub fn initialize_xhc_usb(
+        &mut self,
+        chipset: &RegisteredChipset,
+    ) -> Result<(), MachineInitError> {
+        for (xhc_id, xhc_spec) in &self.spec.xhcs {
+            info!(
+                self.log,
+                "Creating xHCI controller";
+                "pci_path" => %xhc_spec.pci_path,
+            );
+
+            let log = self.log.new(slog::o!("dev" => "xhci"));
+            let bdf: pci::Bdf = xhc_spec.pci_path.into();
+            let xhc = xhci::PciXhci::create(log);
+
+            for (usb_id, usb) in &self.spec.usbdevs {
+                if *xhc_id == usb.xhc_device {
+                    info!(
+                        self.log,
+                        "Attaching USB device";
+                        "usb_id" => %usb_id,
+                        "xhc_pci_path" => %xhc_spec.pci_path,
+                        "usb_port" => %usb.root_hub_port_num,
+                    );
+                    xhc.add_usb_device(usb.root_hub_port_num).map_err(
+                        MachineInitError::UsbRootHubPortNumberInvalid,
+                    )?;
+                }
+            }
+
+            self.devices.insert(xhc_id.clone(), xhc.clone());
+            chipset.pci_attach(bdf, xhc);
+        }
+
+        Ok(())
+    }
+
     #[cfg(feature = "failure-injection")]
     pub fn initialize_test_devices(&mut self) {
         use propolis::hw::testdev::{

bin/propolis-server/src/lib/spec/api_spec_v0.rs

@@ -5,7 +5,7 @@
 //! Conversions from version-0 instance specs in the [`propolis_api_types`]
 //! crate to the internal [`super::Spec`] representation.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 use propolis_api_types::instance_spec::{
     components::{
@@ -44,6 +44,9 @@ pub(crate) enum ApiSpecError {
     #[error("network backend {backend} not found for device {device}")]
     NetworkBackendNotFound { backend: SpecKey, device: SpecKey },
 
+    #[error("USB host controller {xhc} not found for device {device}")]
+    HostControllerNotFound { xhc: SpecKey, device: SpecKey },
+
     #[allow(dead_code)]
     #[error("support for component {component} compiled out via {feature}")]
     FeatureCompiledOut { component: SpecKey, feature: &'static str },
@@ -61,6 +64,8 @@ impl From<Spec> for InstanceSpecV0 {
             cpuid,
             disks,
             nics,
+            xhcs,
+            usbdevs,
             boot_settings,
             serial,
             pci_pci_bridges,
@@ -121,6 +126,14 @@ impl From<Spec> for InstanceSpecV0 {
             );
         }
 
+        for (id, xhc) in xhcs {
+            insert_component(&mut spec, id, ComponentV0::Xhci(xhc));
+        }
+
+        for (id, usb) in usbdevs {
+            insert_component(&mut spec, id, ComponentV0::UsbPlaceholder(usb));
+        }
+
         for (name, desc) in serial {
             if desc.device == SerialPortDevice::Uart {
                 insert_component(
@@ -230,6 +243,7 @@ impl TryFrom<InstanceSpecV0> for Spec {
             BTreeMap::new();
         let mut dlpi_backends: BTreeMap<SpecKey, DlpiNetworkBackend> =
             BTreeMap::new();
+        let mut xhci_controllers: BTreeSet<SpecKey> = BTreeSet::new();
 
         for (id, component) in value.components.into_iter() {
             match component {
@@ -249,6 +263,10 @@ impl TryFrom<InstanceSpecV0> for Spec {
                 ComponentV0::DlpiNetworkBackend(dlpi) => {
                     dlpi_backends.insert(id, dlpi);
                 }
+                ComponentV0::Xhci(xhc) => {
+                    xhci_controllers.insert(id.to_owned());
+                    builder.add_xhci_controller(id, xhc)?;
+                }
                 device => {
                     devices.push((id, device));
                 }
@@ -365,6 +383,19 @@ impl TryFrom<InstanceSpecV0> for Spec {
                 ComponentV0::P9fs(p9fs) => {
                     builder.set_p9fs(p9fs)?;
                 }
+                ComponentV0::Xhci(xhci) => {
+                    builder.add_xhci_controller(device_id, xhci)?;
+                }
+                ComponentV0::UsbPlaceholder(usbdev) => {
+                    if xhci_controllers.contains(&usbdev.xhc_device) {
+                        builder.add_usb_device(device_id, usbdev)?;
+                    } else {
+                        return Err(ApiSpecError::HostControllerNotFound {
+                            xhc: usbdev.xhc_device.to_owned(),
+                            device: device_id,
+                        });
+                    }
+                }
                 ComponentV0::CrucibleStorageBackend(_)
                 | ComponentV0::FileStorageBackend(_)
                 | ComponentV0::BlobStorageBackend(_)

bin/propolis-server/src/lib/spec/builder.rs

@@ -9,7 +9,7 @@ use std::collections::{BTreeSet, HashSet};
 use propolis_api_types::instance_spec::{
     components::{
         board::Board as InstanceSpecBoard,
-        devices::{PciPciBridge, SerialPortNumber},
+        devices::{PciPciBridge, SerialPortNumber, UsbDevice, XhciController},
     },
     PciPath, SpecKey,
 };
@@ -65,13 +65,17 @@ pub(crate) enum SpecBuilderError {
 
     #[error("failed to read default CPUID settings from the host")]
     DefaultCpuidReadFailed(#[from] cpuid_utils::host::GetHostCpuidError),
+
+    #[error("a USB device is already attached to xHC {0} root hub port {1}")]
+    UsbPortInUse(SpecKey, u8),
 }
 
 #[derive(Debug, Default)]
 pub(crate) struct SpecBuilder {
     spec: super::Spec,
     pci_paths: BTreeSet<PciPath>,
     serial_ports: HashSet<SerialPortNumber>,
+    xhc_usb_ports: BTreeSet<(SpecKey, u8)>,
     component_names: BTreeSet<SpecKey>,
 }
 
@@ -154,6 +158,23 @@ impl SpecBuilder {
         }
     }
 
+    fn register_usb_device(
+        &mut self,
+        usbdev: &UsbDevice,
+    ) -> Result<(), SpecBuilderError> {
+        // slightly awkward: we have to take a ref of an owned tuple for
+        // .contains() below, and in either case we need an owned SpecKey,
+        // so we'll just clone it once here
+        let xhc_and_port =
+            (usbdev.xhc_device.to_owned(), usbdev.root_hub_port_num);
+        if self.xhc_usb_ports.contains(&xhc_and_port) {
+            Err(SpecBuilderError::UsbPortInUse(xhc_and_port.0, xhc_and_port.1))
+        } else {
+            self.xhc_usb_ports.insert(xhc_and_port);
+            Ok(())
+        }
+    }
+
     /// Adds a storage device with an associated backend.
     pub(super) fn add_storage_device(
         &mut self,
@@ -355,6 +376,26 @@ impl SpecBuilder {
         Ok(self)
     }
 
+    pub fn add_xhci_controller(
+        &mut self,
+        device_id: SpecKey,
+        xhc: XhciController,
+    ) -> Result<&Self, SpecBuilderError> {
+        self.register_pci_device(xhc.pci_path)?;
+        self.spec.xhcs.insert(device_id, xhc);
+        Ok(self)
+    }
+
+    pub fn add_usb_device(
+        &mut self,
+        device_id: SpecKey,
+        usbdev: UsbDevice,
+    ) -> Result<&Self, SpecBuilderError> {
+        self.register_usb_device(&usbdev)?;
+        self.spec.usbdevs.insert(device_id, usbdev);
+        Ok(self)
+    }
+
     /// Yields the completed spec, consuming the builder.
     pub fn finish(self) -> super::Spec {
         self.spec

bin/propolis-server/src/lib/spec/mod.rs

@@ -26,7 +26,7 @@ use propolis_api_types::instance_spec::{
         board::{Chipset, GuestHypervisorInterface, I440Fx},
         devices::{
             NvmeDisk, PciPciBridge, QemuPvpanic as QemuPvpanicDesc,
-            SerialPortNumber, VirtioDisk, VirtioNic,
+            SerialPortNumber, UsbDevice, VirtioDisk, VirtioNic, XhciController,
         },
     },
     v0::ComponentV0,
@@ -66,6 +66,8 @@ pub(crate) struct Spec {
     pub cpuid: CpuidSet,
     pub disks: BTreeMap<SpecKey, Disk>,
     pub nics: BTreeMap<SpecKey, Nic>,
+    pub xhcs: BTreeMap<SpecKey, XhciController>,
+    pub usbdevs: BTreeMap<SpecKey, UsbDevice>,
     pub boot_settings: Option<BootSettings>,
 
     pub serial: BTreeMap<SpecKey, SerialPort>,

bin/propolis-server/src/lib/vm/ensure.rs

@@ -563,6 +563,7 @@ async fn initialize_vm_objects(
         &properties,
     ))?;
     init.initialize_network_devices(&chipset).await?;
+    init.initialize_xhc_usb(&chipset)?;
 
     #[cfg(feature = "failure-injection")]
     init.initialize_test_devices();

bin/propolis-standalone/src/main.rs

@@ -1248,6 +1248,13 @@ fn setup_instance(
                     block::attach(nvme.clone(), backend).unwrap();
                     chipset_pci_attach(bdf, nvme);
                 }
+                "pci-xhci" => {
+                    let log = log.new(slog::o!("dev" => "xhci"));
+                    let bdf = bdf.unwrap();
+                    let xhci = hw::usb::xhci::PciXhci::create(log);
+                    guard.inventory.register_instance(&xhci, &bdf.to_string());
+                    chipset_pci_attach(bdf, xhci);
+                }
                 qemu::pvpanic::DEVICE_NAME => {
                     let enable_isa = dev
                         .options

crates/propolis-api-types/src/instance_spec/components/devices.rs

@@ -190,6 +190,33 @@ pub struct P9fs {
     pub pci_path: PciPath,
 }
 
+/// Describes a PCI device implementing the eXtensible Host Controller Interface
+/// for the purpose of attaching USB devices.
+///
+/// (Note that at present no functional USB devices have yet been implemented.)
+#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema)]
+#[serde(deny_unknown_fields)]
+pub struct XhciController {
+    /// The PCI path at which to attach the guest to this xHC.
+    pub pci_path: PciPath,
+}
+
+/// Describes a USB device, requires the presence of an XhciController.
+///
+/// (Note that at present no USB devices have yet been implemented
+/// outside of a null device for testing purposes.)
+#[derive(Clone, Deserialize, Serialize, Debug, JsonSchema)]
+#[serde(deny_unknown_fields)]
+pub struct UsbDevice {
+    /// The name of the xHC to which this USB device shall be attached.
+    pub xhc_device: SpecKey,
+    /// The root hub port number to which this USB device shall be attached.
+    /// For USB 2.0 devices, valid values are 1-4, inclusive.
+    /// For USB 3.0 devices, valid values are 5-8, inclusive.
+    pub root_hub_port_num: u8,
+    // TODO(lif): a field for device type (e.g. HID tablet, mass storage...)
+}
+
 /// Describes a synthetic device that registers for VM lifecycle notifications
 /// and returns errors during attempts to migrate.
 ///

crates/propolis-api-types/src/instance_spec/v0.rs

@@ -27,6 +27,8 @@ pub enum ComponentV0 {
     SoftNpuPort(components::devices::SoftNpuPort),
     SoftNpuP9(components::devices::SoftNpuP9),
     P9fs(components::devices::P9fs),
+    Xhci(components::devices::XhciController),
+    UsbPlaceholder(components::devices::UsbDevice),
     MigrationFailureInjector(components::devices::MigrationFailureInjector),
     CrucibleStorageBackend(components::backends::CrucibleStorageBackend),
     FileStorageBackend(components::backends::FileStorageBackend),

crates/propolis-config-toml/src/lib.rs

@@ -97,6 +97,10 @@ impl Device {
     pub fn get<T: FromStr, S: AsRef<str>>(&self, key: S) -> Option<T> {
         self.get_string(key)?.parse().ok()
     }
+
+    pub fn get_integer<S: AsRef<str>>(&self, key: S) -> Option<i64> {
+        self.options.get(key.as_ref())?.as_integer()
+    }
 }
 
 #[derive(Debug, Deserialize, Serialize, PartialEq)]

crates/propolis-config-toml/src/spec.rs

@@ -13,8 +13,8 @@ use propolis_client::{
     instance_spec::{
         ComponentV0, DlpiNetworkBackend, FileStorageBackend,
         MigrationFailureInjector, NvmeDisk, P9fs, PciPath, PciPciBridge,
-        SoftNpuP9, SoftNpuPciPort, SoftNpuPort, SpecKey, VirtioDisk,
-        VirtioNetworkBackend, VirtioNic,
+        SoftNpuP9, SoftNpuPciPort, SoftNpuPort, SpecKey, UsbDevice, VirtioDisk,
+        VirtioNetworkBackend, VirtioNic, XhciController,
     },
     support::nvme_serial_from_str,
 };
@@ -33,6 +33,12 @@ pub enum TomlToSpecError {
     #[error("failed to get PCI path for device {0:?}")]
     InvalidPciPath(String),
 
+    #[error("failed to get USB root hub port for device {0:?}")]
+    InvalidUsbPort(String),
+
+    #[error("no xHC name for USB device {0:?}")]
+    NoHostControllerNameForUsbDevice(String),
+
     #[error("failed to parse PCI path string {0:?}")]
     PciPathParseFailed(String, #[source] std::io::Error),
 
@@ -249,6 +255,44 @@ impl TryFrom<&super::Config> for SpecConfig {
                         )?),
                     )?;
                 }
+                "pci-xhci" => {
+                    let pci_path: PciPath =
+                        device.get("pci-path").ok_or_else(|| {
+                            TomlToSpecError::InvalidPciPath(
+                                device_name.to_owned(),
+                            )
+                        })?;
+
+                    spec.components.insert(
+                        device_id,
+                        ComponentV0::Xhci(XhciController { pci_path }),
+                    );
+                }
+                "usb-dummy" => {
+                    let root_hub_port_num = device
+                        .get_integer("root-hub-port")
+                        .filter(|x| (1..=8).contains(x))
+                        .ok_or_else(|| {
+                            TomlToSpecError::InvalidUsbPort(
+                                device_name.to_owned(),
+                            )
+                        })? as u8;
+
+                    let xhc_device: SpecKey =
+                        device.get("xhc-device").ok_or_else(|| {
+                            TomlToSpecError::NoHostControllerNameForUsbDevice(
+                                device_name.to_owned(),
+                            )
+                        })?;
+
+                    spec.components.insert(
+                        device_id,
+                        ComponentV0::UsbPlaceholder(UsbDevice {
+                            root_hub_port_num,
+                            xhc_device,
+                        }),
+                    );
+                }
                 _ => {
                     return Err(TomlToSpecError::UnrecognizedDeviceType(
                         driver.to_owned(),