diff --git a/.gitignore b/.gitignore index 9c8a952..940d543 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,8 @@ **/*.rs.bk # log files -**/*.log \ No newline at end of file +**/*.log + +.idea* + +.vscode* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 6c6c7a3..bafdcd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,7 +280,7 @@ dependencies = [ [[package]] name = "dbs-address-space" version = "0.3.0" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "arc-swap", "lazy_static", @@ -294,7 +294,7 @@ dependencies = [ [[package]] name = "dbs-allocator" version = "0.1.1" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "thiserror", ] @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "dbs-arch" version = "0.2.3" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -316,7 +316,7 @@ dependencies = [ [[package]] name = "dbs-boot" version = "0.4.0" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "dbs-arch", "kvm-bindings", @@ -357,7 +357,7 @@ dependencies = [ [[package]] name = "dbs-device" version = "0.2.0" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "thiserror", ] @@ -365,7 +365,7 @@ dependencies = [ [[package]] name = "dbs-interrupt" version = "0.2.2" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "dbs-arch", "dbs-device", @@ -378,7 +378,7 @@ dependencies = [ [[package]] name = "dbs-legacy-devices" version = "0.1.1" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "dbs-device", "dbs-utils", @@ -389,10 +389,31 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "dbs-pci" +version = "0.1.0" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" +dependencies = [ + "byteorder", + "dbs-allocator", + "dbs-boot", + "dbs-device", + "dbs-interrupt", + "downcast-rs", + "kvm-bindings", + "kvm-ioctls", + "libc", + "log", + "thiserror", + "vfio-bindings", + "vfio-ioctls", + "vm-memory", +] + [[package]] name = "dbs-upcall" version = "0.3.0" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "anyhow", "dbs-utils", @@ -405,7 +426,7 @@ dependencies = [ [[package]] name = "dbs-utils" version = "0.2.1" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "anyhow", "event-manager", @@ -420,7 +441,7 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" version = "0.3.1" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "byteorder", "caps", @@ -446,6 +467,7 @@ dependencies = [ "serde_json", "thiserror", "threadpool", + "timerfd", "vhost", "virtio-bindings", "virtio-queue", @@ -496,10 +518,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + [[package]] name = "dragonball" version = "0.1.0" -source = "git+https://github.com/kata-containers/kata-containers?branch=main#38eb4077a6d0964834c9ea76d8609b01918198f3" +source = "git+https://github.com/kata-containers/kata-containers?branch=main#37a4049d0fb6c72b2839431bbbaf0d32eda6837a" dependencies = [ "anyhow", "arc-swap", @@ -512,6 +540,7 @@ dependencies = [ "dbs-device", "dbs-interrupt", "dbs-legacy-devices", + "dbs-pci", "dbs-upcall", "dbs-utils", "dbs-virtio-devices", @@ -534,6 +563,8 @@ dependencies = [ "slog-scope", "thiserror", "tracing", + "vfio-bindings", + "vfio-ioctls", "virtio-queue", "vm-memory", "vmm-sys-util", @@ -558,6 +589,12 @@ dependencies = [ "libc", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -749,9 +786,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -768,9 +805,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -923,11 +960,11 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ - "autocfg", + "equivalent", "hashbrown", ] @@ -2167,6 +2204,29 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vfio-bindings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43449b404c488f70507dca193debd4bea361fe8089869b947adc19720e464bce" + +[[package]] +name = "vfio-ioctls" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068bac78842164a8ecc1d1a84a8d8a9168ab29fa3c96942689e286a30ae22ac4" +dependencies = [ + "byteorder", + "kvm-bindings", + "kvm-ioctls", + "libc", + "log", + "thiserror", + "vfio-bindings", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "vhost" version = "0.6.1" diff --git a/Cargo.toml b/Cargo.toml index 3fefe7b..a6860cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ dragonball = { git = "https://github.com/kata-containers/kata-containers", branc "hotplug", "dbs-upcall", "vhost-user-net", + "host-device" ] } clap = { version = "4.0.27", features = ["derive"] } serde = "1.0.27" diff --git a/README.md b/README.md index 3b18b02..6241e44 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,30 @@ The supported network devices include: } ``` +### PCI Device + +You can choose to attach a pci device during the boot time of Dragonball. +Please note that hostdev_id and bus_slot_func are the must parameters for attaching pci device. +Please make sure that pci device is binded to `vfio-pci` driver. + +``` +./dbs-cli create --kernel-path $KERNEL_PATH --rootfs $ROOTFS_PATH --boot-args "console=ttyS0 tty0 reboot=k debug panic=1 root=/dev/vda1" --hostdev-id $HOST_DEVICE_ID --bus-slot-func $BUS_SLOT_FUNC +``` + +#### How to get hostdev_id and bus_slot_func? + +hostdev_id: This is an id you pick for each pci device attaching into VM. So name it whatever number you want. + +bus_slot_func: take a network device as the example, you could use `lspci | grep "network device"` and you could get something like +``` +[root@xxx ~]# lspci | grep "network device" +5d:00.0 Ethernet controller: Red Hat, Inc. Virtio network device +5e:00.0 Ethernet controller: Red Hat, Inc. Virtio network device +``` +`5d:00.0` is the bus_slot_func. + +As an alternative way to insert a host device, you can use upcall to hotplug / hot-unplug a pci device into Dragonball while Dragonball is running, for more details please go to advanced usage part of this document. + ## Advanced Usage ### Create API Server and Update VM @@ -153,6 +177,23 @@ sudo ./dbs-cli \ --hotplug-virblks '[{"drive_id":"testblk","device_type":"RawBlock","path_on_host":"/path/to/test.img","is_root_device":false,"is_read_only":false,"is_direct":false,"no_drop":false,"num_queues":1,"queue_size":1024}]' \ ``` +Hotplug a pci device into Dragonball +``` +./dbs-cli --api-sock-path $API_SOCK_PATH update --bus-slot-func $BUS_SLOT_FUNC --hostdev-id $HOST_DEVICE_ID +``` + +Prepare hot-unplug a pci device into Dragonball (must do before hotunplug) + +``` +./dbs-cli --api-sock-path ./sock update --prepare-remove-host-device $HOST_DEVICE_ID +``` + +Hot-unplug a pci device into Dragonball + +``` +./dbs-cli --api-sock-path ./sock update --remove-host-device $HOST_DEVICE_ID +``` + TODO : add document for hot-plug virtio-fs ### Exit VM diff --git a/src/api_client.rs b/src/api_client.rs index bfac8f5..b9a1e6e 100644 --- a/src/api_client.rs +++ b/src/api_client.rs @@ -7,25 +7,45 @@ use std::os::unix::net::UnixStream; use anyhow::{Context, Result}; use serde_json::{json, Value}; -use crate::parser::args::UpdateArgs; +use crate::parser::args::{HostDeviceArgs, UpdateArgs}; pub fn run_api_client(args: UpdateArgs, api_sock_path: &str) -> Result<()> { if let Some(vcpu_resize_num) = args.vcpu_resize { let request = request_cpu_resize(vcpu_resize_num); send_request(request, api_sock_path)?; } + if let Some(config) = args.virnets { let request = request_virtio_net(&config); send_request(request, api_sock_path)?; } + if let Some(config) = args.virblks { let request = request_virtio_blk(&config); send_request(request, api_sock_path)?; } + if let Some(config) = args.patch_fs { let request = request_patch_fs(&config); send_request(request, api_sock_path)?; } + + if let Some(host_device_args) = args.insert_host_device { + if host_device_args.bus_slot_func.is_some() { + let request = request_insert_host_device(host_device_args.clone()); + send_request(request, api_sock_path)?; + } + } + + if let Some(host_device_id) = args.prepare_remove_host_device { + let request = request_prepare_remove_host_device(host_device_id.clone()); + send_request(request, api_sock_path)?; + } + + if let Some(host_device_id) = args.remove_host_device { + let request = request_remove_host_device(host_device_id.clone()); + send_request(request, api_sock_path)?; + } Ok(()) } @@ -59,6 +79,28 @@ fn request_patch_fs(patch_fs_config: &str) -> Value { }) } +fn request_insert_host_device(host_device_args: HostDeviceArgs) -> Value { + json!({ + "action": "insert_host_device", + "hostdev-id": host_device_args.hostdev_id.expect("host device arg should be provided to insert host device."), + "bus-slot-func": host_device_args.bus_slot_func.expect("bus_slot_func should be provided to insert host device."), + }) +} + +fn request_prepare_remove_host_device(host_device_id: String) -> Value { + json!({ + "action": "prepare_remove_host_device", + "hostdev-id": host_device_id.clone(), + }) +} + +fn request_remove_host_device(host_device_id: String) -> Value { + json!({ + "action": "remove_host_device", + "hostdev-id": host_device_id.clone(), + }) +} + fn send_request(request: Value, api_sock_path: &str) -> Result<()> { let mut unix_stream = UnixStream::connect(api_sock_path).context("Could not create stream")?; diff --git a/src/api_server.rs b/src/api_server.rs index 8003f29..b70fade 100644 --- a/src/api_server.rs +++ b/src/api_server.rs @@ -12,6 +12,7 @@ use crossbeam_channel::{Receiver, Sender}; use dragonball::api::v1::{NetworkInterfaceConfig, VmmRequest, VmmResponse}; use dragonball::device_manager::blk_dev_mgr::BlockDeviceConfigInfo; use dragonball::device_manager::fs_dev_mgr::FsMountConfigInfo; +use dragonball::device_manager::vfio_dev_mgr::{HostDeviceConfig, VfioPciDeviceConfig}; use dragonball::vcpu::VcpuResizeInfo; use serde_json::Value; use vmm_sys_util::eventfd::EventFd; @@ -80,6 +81,31 @@ impl ApiServer { }; return self.resize_vcpu(resize_vcpu_cfg); } + Some("insert_host_device") => { + // TODO: add customize support for sysfs_path, vendor_device_id, guest_dev_id and clique_id. + // ignore them now since they are not the must parameters for hotplugging a host device. + // issue: #31 + let host_device_config = HostDeviceConfig { + hostdev_id: v["hostdev-id"].as_str().unwrap().to_owned(), + sysfs_path: "".to_string(), + dev_config: VfioPciDeviceConfig { + bus_slot_func: v["bus-slot-func"].as_str().unwrap().to_owned(), + vendor_device_id: 0, + guest_dev_id: None, + clique_id: None, + }, + }; + self.insert_host_device(host_device_config) + .expect("Failed to insert a host device"); + } + Some("prepare_remove_host_device") => { + self.prepare_remove_host_device(v["hostdev-id"].as_str().unwrap().to_owned()) + .expect("Failed to insert a host device"); + } + Some("remove_host_device") => { + self.remove_host_device(v["hostdev-id"].as_str().unwrap().to_owned()) + .expect("Failed to insert a host device"); + } Some("insert_virnets") => { let config_json = match v["config"].as_str() { Some(config_json) => config_json, diff --git a/src/cli_instance.rs b/src/cli_instance.rs index 343a5b2..262b5f0 100644 --- a/src/cli_instance.rs +++ b/src/cli_instance.rs @@ -20,7 +20,10 @@ use dragonball::{ BlockDeviceConfigInfo, BootSourceConfig, InstanceInfo, NetworkInterfaceConfig, VmmRequest, VmmResponse, VsockDeviceConfigInfo, }, - device_manager::fs_dev_mgr::FsDeviceConfigInfo, + device_manager::{ + fs_dev_mgr::FsDeviceConfigInfo, + vfio_dev_mgr::{HostDeviceConfig, VfioPciDeviceConfig}, + }, vm::{CpuTopology, VmConfigInfo}, }; @@ -97,6 +100,7 @@ impl CliInstance { // as in crate `dragonball` serial_path will be assigned with a default value, // we need a special token to enable the stdio console. serial_path: serial_path.clone(), + pci_hotplug_enabled: args.host_device.pci_hotplug_enabled, }; if let Some(com1_sock_path) = serial_path { @@ -153,6 +157,27 @@ impl CliInstance { .expect("failed to set vsock socket path"); } + // users should at least provide hostdev_id and bus_slot_func to insert a host device + if args.host_device.hostdev_id.is_some() && args.host_device.bus_slot_func.is_some() { + let host_device_config = HostDeviceConfig { + hostdev_id: args + .host_device + .hostdev_id + .expect("There has to be hostdev_id if you want to add host device."), + sysfs_path: args.host_device.sysfs_path.unwrap_or_default(), + dev_config: VfioPciDeviceConfig { + bus_slot_func: args + .host_device + .bus_slot_func + .expect("There has to be bus_slot_func if you want to add host device."), + vendor_device_id: args.host_device.vendor_device_id.unwrap_or_default(), + guest_dev_id: args.host_device.guest_dev_id, + clique_id: args.host_device.clique_id, + }, + }; + self.insert_host_device(host_device_config) + .expect("Failed to insert a host device"); + } // Virtio devices if !args.virnets.is_empty() { let configs: Vec = serde_json::from_str(&args.virnets) diff --git a/src/parser/args.rs b/src/parser/args.rs index b97cb5a..9da68e6 100644 --- a/src/parser/args.rs +++ b/src/parser/args.rs @@ -120,6 +120,10 @@ pub struct CreateArgs { #[clap(flatten)] pub mem: MemArgs, + /// features of host devices + #[clap(flatten)] + pub host_device: HostDeviceArgs, + // The serial path used to communicate with VM #[clap( short, @@ -282,6 +286,30 @@ pub struct MemArgs { pub mem_size: usize, } +#[derive(Args, Debug, Serialize, Deserialize, Clone)] +pub struct HostDeviceArgs { + #[clap( + long, + value_parser, + help = "whether pci hotplug ability is enabled or not", + default_value_t = false, + display_order = 2 + )] + pub pci_hotplug_enabled: bool, + #[clap(long, value_parser, help = "host dev id", display_order = 2)] + pub hostdev_id: Option, + #[clap(long, value_parser, help = "sys fs path", display_order = 2)] + pub sysfs_path: Option, + #[clap(long, value_parser, help = "bus slot function", display_order = 2)] + pub bus_slot_func: Option, + #[clap(long, value_parser, help = "vendor_device_id", display_order = 2)] + pub vendor_device_id: Option, + #[clap(long, value_parser, help = "guest_dev_id", display_order = 2)] + pub guest_dev_id: Option, + #[clap(long, value_parser, help = "clique_id", display_order = 2)] + pub clique_id: Option, +} + #[derive(Args, Debug, Serialize, Deserialize, Clone)] pub struct UpdateArgs { #[clap( @@ -319,4 +347,23 @@ The type of it is an array of BlockDeviceConfigInfo, e.g. display_order = 2 )] pub patch_fs: Option, + + #[clap(flatten)] + pub insert_host_device: Option, + + #[clap( + long, + value_parser, + help = r#"prepare remove host device with hostdev_id."#, + display_order = 2 + )] + pub prepare_remove_host_device: Option, + + #[clap( + long, + value_parser, + help = r#"remove host device with hostdev_id."#, + display_order = 2 + )] + pub remove_host_device: Option, } diff --git a/src/vmm_comm_trait.rs b/src/vmm_comm_trait.rs index 55d889a..94faccd 100644 --- a/src/vmm_comm_trait.rs +++ b/src/vmm_comm_trait.rs @@ -7,6 +7,7 @@ use dragonball::api::v1::{ VmmData, VmmRequest, VmmResponse, VsockDeviceConfigInfo, }; use dragonball::device_manager::fs_dev_mgr::{FsDeviceConfigInfo, FsMountConfigInfo}; +use dragonball::device_manager::vfio_dev_mgr::HostDeviceConfig; use dragonball::vcpu::VcpuResizeInfo; use dragonball::vm::VmConfigInfo; use vmm_sys_util::eventfd::EventFd; @@ -168,4 +169,31 @@ pub trait VMMComm { })?; Ok(()) } + + fn insert_host_device(&self, host_device_cfg: HostDeviceConfig) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::InsertHostDevice(host_device_cfg.clone()))) + .with_context(|| { + format!( + "Failed to insert host device hostdev_id {:?}, sysfs_path {:?}, host_device_cfg {:?}", + host_device_cfg.hostdev_id, host_device_cfg.sysfs_path, host_device_cfg.dev_config + ) + })?; + Ok(()) + } + + fn prepare_remove_host_device(&self, hostdev_id: String) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::PrepareRemoveHostDevice( + hostdev_id.clone(), + ))) + .with_context(|| format!("Failed to prepare remove host device {:?}", hostdev_id))?; + Ok(()) + } + + fn remove_host_device(&self, hostdev_id: String) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::RemoveHostDevice( + hostdev_id.clone(), + ))) + .with_context(|| format!("Failed to remove host device {:?}", hostdev_id))?; + Ok(()) + } }