pci: add pci device support in dbs-cli

1. attach pci device during the boot time of Dragonball. 2. hot plug pci device when the Dragonball is running. 3. prepare to hot unplug pci device when the Dragonball is running. 4. hot unplug pci device when the Dragonball is running. fixes: #29 Signed-off-by: Chao Wu <[email protected]>
openanolis · Jan 16, 2024 · 9378ba9 · 9378ba9
1 parent d720702
commit 9378ba9
Show file tree

Hide file tree

Showing 11 changed files with 182 additions and 38 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,4 +10,8 @@
 **/*.rs.bk
 
 # log files
-**/*.log
+**/*.log
+
+.idea*
+
+.vscode*
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-dragonball = { path = "/root/kata-containers/src/dragonball", features = [
+dragonball = { git = "https://github.com/kata-containers/kata-containers", branch = "main", features = [
     "virtio-blk",
     "virtio-fs",
     "virtio-vsock",
@@ -15,7 +15,7 @@ dragonball = { path = "/root/kata-containers/src/dragonball", features = [
     "hotplug",
     "dbs-upcall",
     "vhost-user-net",
-    "vhost-user-net"
+    "host-device"
 ] }
 clap = { version = "4.0.27", features = ["derive"] }
 serde = "1.0.27"

diff --git a/README.md b/README.md
@@ -117,6 +117,15 @@ The supported network devices include:
 }
 ```
 
+### PCI Device
+
+You can choose to attach a pci device during the boot time of Dragonball.
+```
+./dbs-cli create --log-file $LOG_FILE --log-level $LOG_LEVEL --kernel-path $KERNEL_PATH --rootfs $ROOTFS_PATH --boot-args "console=ttyS0 tty0 reboot=k debug panic=1 root=/dev/vda1" --hostdev-id $HOST_DEVICE_ID --sysfs-path $SYSFS_PATH --bus-slot-func $BUS_SLOT_FUNC --vendor-device-id $VENDOR_DEVICE_ID
+```
+
+Else, you can use upcall to hotplug / hot-unplug a pci device into Dragonball while Dragonball is running, for more details please go to advance usgae part of this document.
+
 ## Advanced Usage
 
 ### Create API Server and Update VM
@@ -153,6 +162,23 @@ sudo ./dbs-cli  \
   --hotplug-virblks '[{"drive_id":"testblk","device_type":"RawBlock","path_on_host":"/path/to/test.img","is_root_device":false,"is_read_only":false,"is_direct":false,"no_drop":false,"num_queues":1,"queue_size":1024}]' \
 ```
 
+Hotplug a pci device into Dragonball
+```
+./dbs-cli --api-sock-path $API_SOCK_PATH update --bus-slot-func $BUS_SLOT_FUNC --vendor-device-id $VENDOR_DEVICE_ID --hostdev-id $HOST_DEVICE_ID --sysfs-path $SYSFS_PATH
+```
+
+Prepare hot-unplug a pci device into Dragonball (must do before hotunplug)
+
+```
+./dbs-cli --api-sock-path ./sock update --prepare-remove-host-device $HOST_DEVICE_ID
+```
+
+Hot-unplug a pci device into Dragonball
+
+```
+./dbs-cli --api-sock-path ./sock update --remove-host-device $HOST_DEVICE_ID
+```
+
 TODO : add document for hot-plug virtio-fs
 
 ### Exit VM

diff --git a/src/api_client.rs b/src/api_client.rs
@@ -7,25 +7,45 @@ use std::os::unix::net::UnixStream;
 use anyhow::{Context, Result};
 use serde_json::{json, Value};
 
-use crate::parser::args::UpdateArgs;
+use crate::parser::args::{HostDeviceArgs, UpdateArgs};
 
 pub fn run_api_client(args: UpdateArgs, api_sock_path: &str) -> Result<()> {
     if let Some(vcpu_resize_num) = args.vcpu_resize {
         let request = request_cpu_resize(vcpu_resize_num);
         send_request(request, api_sock_path)?;
     }
+
     if let Some(config) = args.virnets {
         let request = request_virtio_net(&config);
         send_request(request, api_sock_path)?;
     }
+
     if let Some(config) = args.virblks {
         let request = request_virtio_blk(&config);
         send_request(request, api_sock_path)?;
     }
+
     if let Some(config) = args.patch_fs {
         let request = request_patch_fs(&config);
         send_request(request, api_sock_path)?;
     }
+
+    if let Some(host_device_args) = args.insert_host_device {
+        if host_device_args.bus_slot_func.is_some() {
+            let request = request_insert_host_device(host_device_args.clone());
+            send_request(request, api_sock_path)?;
+        }
+    }
+
+    if let Some(host_device_id) = args.prepare_remove_host_device {
+        let request = request_prepare_remove_host_device(host_device_id.clone());
+        send_request(request, api_sock_path)?;
+    }
+
+    if let Some(host_device_id) = args.remove_host_device {
+        let request = request_remove_host_device(host_device_id.clone());
+        send_request(request, api_sock_path)?;
+    }
     Ok(())
 }
 
@@ -59,6 +79,29 @@ fn request_patch_fs(patch_fs_config: &str) -> Value {
     })
 }
 
+fn request_insert_host_device(host_device_args: HostDeviceArgs) -> Value {
+    json!({
+        "action": "insert_host_device",
+        "hostdev-id": host_device_args.hostdev_id.expect("host device arg should be provided to insert host device."),
+        "bus-slot-func": host_device_args.bus_slot_func.expect("bus_slot_func should be provided to insert host device."),
+        "vendor-device-id": host_device_args.vendor_device_id.expect("vendor_device_id should be provided to insert host device."),
+    })
+}
+
+fn request_prepare_remove_host_device(host_device_id: String) -> Value {
+    json!({
+        "action": "prepare_remove_host_device",
+        "hostdev-id": host_device_id.clone(),
+    })
+}
+
+fn request_remove_host_device(host_device_id: String) -> Value {
+    json!({
+        "action": "remove_host_device",
+        "hostdev-id": host_device_id.clone(),
+    })
+}
+
 fn send_request(request: Value, api_sock_path: &str) -> Result<()> {
     let mut unix_stream = UnixStream::connect(api_sock_path).context("Could not create stream")?;
 

diff --git a/src/api_server.rs b/src/api_server.rs
@@ -83,11 +83,11 @@ impl ApiServer {
             }
             Some("insert_host_device") => {
                 let host_device_config = HostDeviceConfig {
-                    hostdev_id: v["hostdev_id"].as_str().unwrap().to_owned(),
-                    sysfs_path: v["sysfs_path"].as_str().unwrap().to_owned(),
+                    hostdev_id: v["hostdev-id"].as_str().unwrap().to_owned(),
+                    sysfs_path: "".to_string(),
                     dev_config: VfioPciDeviceConfig {
-                        bus_slot_func: v["bus_slot_func"].as_str().unwrap().to_owned(),
-                        vendor_device_id: v["vendor_device_id"]
+                        bus_slot_func: v["bus-slot-func"].as_str().unwrap().to_owned(),
+                        vendor_device_id: v["vendor-device-id"]
                             .as_u64()
                             .map(|vendor_device_id: u64| vendor_device_id as u32)
                             .unwrap(),
@@ -98,6 +98,14 @@ impl ApiServer {
                 self.insert_host_device(host_device_config)
                     .expect("Failed to insert a host device");
             }
+            Some("prepare_remove_host_device") => {
+                self.prepare_remove_host_device(v["hostdev-id"].as_str().unwrap().to_owned())
+                    .expect("Failed to insert a host device");
+            }
+            Some("remove_host_device") => {
+                self.remove_host_device(v["hostdev-id"].as_str().unwrap().to_owned())
+                    .expect("Failed to insert a host device");
+            }
             Some("insert_virnets") => {
                 let config_json = match v["config"].as_str() {
                     Some(config_json) => config_json,

diff --git a/src/main.rs b/src/main.rs
@@ -9,14 +9,6 @@ use parser::run_with_cli;
 
 use crate::parser::args::{Commands, DBSArgs};
 
-use dragonball::api::v1::NetworkInterfaceConfig;
-use slog::Drain;
-use slog::*;
-use slog_scope::set_global_logger;
-
-use std::str::FromStr;
-use std::sync::Mutex;
-
 mod api_client;
 mod api_server;
 mod cli_instance;
@@ -28,24 +20,6 @@ fn main() -> Result<()> {
     let args: DBSArgs = DBSArgs::parse();
     match args.command {
         Some(Commands::Create { create_args }) => {
-            // let log_file = &create_args.log_file;
-            // let log_level = Level::from_str(&create_args.log_level).unwrap();
-
-            // let file = std::fs::OpenOptions::new()
-            //     .truncate(true)
-            //     .read(true)
-            //     .create(true)
-            //     .write(true)
-            //     .open(log_file)
-            //     .expect("Cannot write to the log file.");
-
-            // let root = slog::Logger::root(
-            //     Mutex::new(slog_json::Json::default(file).filter_level(log_level)).map(slog::Fuse),
-            //     o!("version" => env!("CARGO_PKG_VERSION")),
-            // );
-
-            // let _guard = set_global_logger(root);
-            // slog_stdlog::init().unwrap();
             utils::setup_db_log(&create_args.log_file, &create_args.log_level);
             run_with_cli(create_args, &args.api_sock_path)?;
         }