diff --git a/Cargo.lock b/Cargo.lock index 3c28488..7e66a71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "beau_collector" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33a143066d3cbd3d32c15b51c39449e50e32a68293d31589fb941d6a9df0df4f" +dependencies = [ + "anyhow", + "itertools", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -70,6 +86,12 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "erased-serde" version = "0.3.13" @@ -79,6 +101,15 @@ dependencies = [ "serde 1.0.125", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.7" @@ -110,12 +141,27 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + [[package]] name = "linked-hash-map" version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + [[package]] name = "memchr" version = "2.3.4" @@ -181,6 +227,60 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "qapi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e679d7e201702340604a0afd896a1db3bf1488bb9a2d794523d1fd770bbd744" +dependencies = [ + "log", + "qapi-qmp", + "qapi-spec", + "serde 1.0.125", + "serde_json", +] + +[[package]] +name = "qapi-codegen" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b4abbe2a6fbc026bf9bb9021d181d8da4661c6a3cf9a221f6a36aca2617acd" +dependencies = [ + "qapi-parser", +] + +[[package]] +name = "qapi-parser" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ee46393e919bb03d7ef4c3d633dbbf7d7196fb02b9928ea9fccae7529db07c" +dependencies = [ + "serde 1.0.125", + "serde_json", +] + +[[package]] +name = "qapi-qmp" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ef8ca64a52853030d0eac261c34c87978b65060aee7fe54e01659fb0d238500" +dependencies = [ + "qapi-codegen", + "qapi-spec", + "serde 1.0.125", +] + +[[package]] +name = "qapi-spec" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b360919a24ea5fc02fa762cb01bd8f43b643fee51c585f763257773b4dc5a9e8" +dependencies = [ + "base64", + "serde 1.0.125", + "serde_json", +] + [[package]] name = "quote" version = "1.0.9" @@ -319,18 +419,18 @@ name = "vore-core" version = "0.1.0" dependencies = [ "anyhow", + "beau_collector", "config", "kiam", + "libc", "mlua", + "qapi", + "qapi-qmp", "serde 1.0.125", "serde_json", "toml", ] -[[package]] -name = "vore-lua" -version = "0.1.0" - [[package]] name = "vored" version = "0.1.0" diff --git a/config/example.toml b/config/example.toml index 210b19e..2734d1f 100644 --- a/config/example.toml +++ b/config/example.toml @@ -1,6 +1,7 @@ [machine] name = "win10" memory = "12G" +features = ["uefi", "spice", "scream", "looking-glass"] [cpu] amount = 12 @@ -13,26 +14,27 @@ path = "/dev/disk/by-id/wwn-0x500a0751f008e09d" preset = "ssd" path = "/dev/disk/by-id/wwn-0x5002538e4038852d" -[uefi] -enabled = true - -[[vfio]] -slot = "08:00.0" -graphics = true - -[[vfio]] -slot = "08:00.1" +[[disk]] +preset = "nvme" +path = "/dev/disk/by-id/nvme-eui.6479a74530201073" + +#[[vfio]] +#vendor = 0x10de +#device = 0x1b80 +#index = 1 +# +#graphics = true +# +#[[vfio]] +#vendor = 0x10de +#device = 0x10f0 +#index = 1 [[vfio]] -slot = "0a:00.3" - -[spice] -enabled = true - -[scream] -enabled = true +vendor = 0x1022 +device = 0x149c +addr = "0b:00.3" [looking-glass] -enabled = true width = 2560 height = 1080 \ No newline at end of file diff --git a/config/qemu.lua b/config/qemu.lua index 7b3dc63..1cc1a2d 100644 --- a/config/qemu.lua +++ b/config/qemu.lua @@ -45,13 +45,9 @@ end vore:set_build_command(function(instance, vm) vm:arg("-rtc", "driftfix=slew") - --vm:arg("-mon", "stdio") vm:arg("-no-hpet") vm:arg("-boot", "strict=on") - vm:arg("-chardev", "socket,id=charmonitor,path=/tmp/qemu.sock,server=on,wait=off") - vm:arg("-mon", "chardev=charmonitor,id=monitor,mode=readline") - if instance.kvm then vm:arg("-enable-kvm") end @@ -61,7 +57,7 @@ vore:set_build_command(function(instance, vm) end --this disables the QEMU GUI - --vm:arg("-display", "none") + vm:arg("-display", "none") vm:arg("-no-user-config") --vm:arg("-nodefaults") @@ -101,7 +97,7 @@ vore:set_build_command(function(instance, vm) end for _, vfio in ipairs(instance.vfio) do - local def = "vfio-pci,host=" .. vfio.slot + local def = "vfio-pci,host=" .. vfio.address if vfio.graphics then def = def .. ",x-vga=on" end @@ -146,6 +142,7 @@ end) ---@param type string ---@return fun(vm: VM, idx: number, disk: Disk): VM function scsi_disk_gen(type) + -- see https://blog.christophersmart.com/2019/12/18/kvm-guests-with-emulated-ssd-and-nvme-drives/ return function(vm, idx, disk) vm:arg( "-blockdev", @@ -184,4 +181,13 @@ function scsi_disk_gen(type) end vore:register_disk_preset("ssd", scsi_disk_gen("ssd")) -vore:register_disk_preset("hdd", scsi_disk_gen("hdd")) \ No newline at end of file +vore:register_disk_preset("hdd", scsi_disk_gen("hdd")) +vore:register_disk_preset("nvme", function(vm, _, disk) + local nvme_id = vm:get_counter("nvme", 1) + + -- see https://blog.christophersmart.com/2019/12/18/kvm-guests-with-emulated-ssd-and-nvme-drives/ + vm:arg("-drive", "file=" .. disk.path .. ",driver=raw,if=none,id=NVME" .. nvme_id) + vm:arg("-device", "nvme,drive=NVME" .. nvme_id .. ",serial=nvme-" .. nvme_id) + + return vm +end) \ No newline at end of file diff --git a/resources/vore.def.lua b/resources/vore.def.lua index d9c1bf7..3697f1e 100644 --- a/resources/vore.def.lua +++ b/resources/vore.def.lua @@ -72,7 +72,10 @@ end ---@field buffer_size number ---@class Vfio ----@field slot string +---@field device number|nil +---@field vendor number|nil +---@field index number|nil +---@field address string ---@field graphics boolean ---@field multifunction boolean diff --git a/vore-core/Cargo.toml b/vore-core/Cargo.toml index 077fa36..a3f9636 100644 --- a/vore-core/Cargo.toml +++ b/vore-core/Cargo.toml @@ -13,4 +13,8 @@ serde_json = "1.0.64" toml = "*" anyhow = "1.0.40" kiam = "0.1" -mlua = { version = "0.5.3", features = ["lua54", "serialize", "send"] } \ No newline at end of file +mlua = { version = "0.5.3", features = ["lua54", "serialize", "send"] } +beau_collector = "0.2.1" +qapi-qmp = "0.7.0" +qapi = { version = "0.7.0", features = ["qapi-qmp"] } +libc = "0.2.94" \ No newline at end of file diff --git a/vore-core/src/instance_config.rs b/vore-core/src/instance_config.rs index e2468b2..06c1f3a 100644 --- a/vore-core/src/instance_config.rs +++ b/vore-core/src/instance_config.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Error}; use config::{Config, File, FileFormat, Value}; -use serde::{Deserialize, Serialize}; +use serde::de::Visitor; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use std::collections::HashMap; +use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; #[derive(Deserialize, Serialize, Clone, Debug)] @@ -71,17 +73,27 @@ impl InstanceConfig { } } - if let Ok(looking_glass) = config.get_table("looking-glass") { - instance_config.looking_glass = - LookingGlassConfig::from_table(looking_glass, &instance_config.name)?; - } - - if let Ok(scream) = config.get_table("scream") { - instance_config.scream = ScreamConfig::from_table(scream, &instance_config.name)?; - } - - if let Ok(scream) = config.get_table("spice") { - instance_config.spice = SpiceConfig::from_table(scream)?; + instance_config.looking_glass = LookingGlassConfig::from_table( + config.get_table("looking-glass").unwrap_or_default(), + &instance_config.name, + )?; + instance_config.scream = ScreamConfig::from_table( + config.get_table("scream").unwrap_or_default(), + &instance_config.name, + )?; + instance_config.spice = + SpiceConfig::from_table(config.get_table("spice").unwrap_or_default())?; + + if let Ok(features) = config.get::>("machine.features") { + for feature in features { + match feature.as_str() { + "looking-glass" => instance_config.looking_glass.enabled = true, + "spice" => instance_config.spice.enabled = true, + "scream" => instance_config.scream.enabled = true, + "uefi" => instance_config.uefi.enabled = true, + _ => {} + } + } } Ok(instance_config) @@ -432,7 +444,7 @@ impl DiskConfig { path, }; - // TODO: Add blockdev details + // TODO: Add block dev details Ok(disk) } @@ -440,20 +452,141 @@ impl DiskConfig { #[derive(Deserialize, Serialize, Clone, Debug)] pub struct VfioConfig { - pub slot: String, + pub address: PCIAddress, + pub vendor: Option, + pub device: Option, + pub index: u32, pub graphics: bool, pub multifunction: bool, } +pub fn read_pci_ids(addr: &PCIAddress) -> Result<(u32, u32), anyhow::Error> { + let device = std::fs::read_to_string(format!("/sys/bus/pci/devices/{:#}/device", addr)) + .with_context(|| { + format!( + "Failed to read the device id of PCI device at {:#} ({})", + addr, + format!("/sys/bus/pci/devices/{:#}/device", addr) + ) + })?; + let found_device = u32::from_str_radix(device.trim_start_matches("0x").trim_end(), 16)?; + + let vendor = std::fs::read_to_string(format!("/sys/bus/pci/devices/{:#}/vendor", addr)) + .with_context(|| { + format!( + "Failed to read the vendor id of PCI device at {:#} ({})", + addr, + format!("/sys/bus/pci/devices/{:#}/vendor", addr) + ) + })?; + let found_vendor = u32::from_str_radix(vendor.trim_start_matches("0x").trim_end(), 16)?; + + Ok((found_vendor, found_device)) +} + impl VfioConfig { pub fn from_table(table: HashMap) -> Result { - let slot = table - .get("slot") + let mut address = table + .get("addr") + .or_else(|| table.get("address")) + .cloned() + .map(|x| PCIAddress::from_str(&x.into_str()?)) + .transpose()?; + + let vendor = table + .get("vendor") .cloned() - .ok_or_else(|| anyhow::anyhow!("vfio table needs a slot"))? - .into_str()?; + .map(|x| x.into_int().map(|x| x as u32)) + .transpose()?; + let device = table + .get("device") + .cloned() + .map(|x| x.into_int().map(|x| x as u32)) + .transpose()?; + let index = table + .get("index") + .cloned() + .map(|x| x.into_int().map(|x| x as u32)) + .transpose()? + .unwrap_or(0); + + let address = match (address, vendor, device) { + (Some(addr), vendor, device) => { + let (found_vendor, found_device) = read_pci_ids(&addr)?; + + if let Some(device) = device { + if device != found_device { + anyhow::bail!( + "VFIO expects a PCI device on address {} with the device id {:#04x} but found the id {:#04x} instead", + addr, + device, + found_device + ) + } + } + + if let Some(vendor) = vendor { + if vendor != found_vendor { + anyhow::bail!( + "VFIO expects a PCI device on address {} with the vendor id {:#04x} but found the id {:#04x} instead", + addr, + vendor, + found_vendor + ) + } + } + + addr + } + + (None, Some(vendor), Some(device)) => { + let mut counter = index; + let mut items: Vec<(PCIAddress, u32, u32)> = vec![]; + + for entry in std::fs::read_dir("/sys/bus/pci/devices")? { + let entry = entry?; + let file_name = entry.file_name(); + let addr_name = file_name + .to_str() + .ok_or_else(|| anyhow::anyhow!("Failed to parse PCI device name"))?; + let addr = PCIAddress::from_str(addr_name)?; + let (found_vendor, found_device) = read_pci_ids(&addr)?; + items.push((addr, found_vendor, found_device)); + } + + items.sort_by_key(|&(addr, _, _)| addr); + + for (addr, found_vendor, found_device) in items { + if found_vendor == vendor && found_device == device { + if counter == 0 { + address = Some(addr); + break; + } + + counter -= 1; + } + } + + if let Some(address) = address { + address + } else { + anyhow::bail!( + "Can't find {}th PCI device with vendor id {:#04x} and device id {:#04x}", + index + 1, + vendor, + device + ) + } + } + + _ => anyhow::bail!("VFIO element needs either vendor and device or address to be set"), + }; + let mut cfg = VfioConfig { - slot, + address, + vendor: None, + device: None, + index: 0, graphics: false, multifunction: false, }; @@ -490,3 +623,134 @@ impl SpiceConfig { Ok(cfg) } } + +#[derive(Default, Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] +pub struct PCIAddress { + domain: u32, + bus: u8, + slot: u8, + func: u8, +} + +impl<'de> Deserialize<'de> for PCIAddress { + fn deserialize(deserializer: D) -> Result>::Error> + where + D: Deserializer<'de>, + { + struct X; + impl Visitor<'_> for X { + type Value = String; + + fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str("Expecting a string") + } + + fn visit_string(self, v: String) -> Result { + Ok(v) + } + } + + let x = deserializer.deserialize_string(X)?; + Ok(PCIAddress::from_str(&x).map_err(|x| de::Error::custom(x))?) + } +} + +impl Serialize for PCIAddress { + fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> + where + S: Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl PCIAddress { + fn to_string(&self) -> String { + format!( + "{:04x}:{:02x}:{:02x}.{:x}", + self.domain, self.bus, self.slot, self.func + ) + } +} + +impl Debug for PCIAddress { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("PCIAddress(")?; + if f.alternate() && self.domain == 0 { + f.write_str(&format!("{:04x}:", self.domain))?; + } + + f.write_str(&format!( + "{:02x}:{:02x}.{:x}", + self.bus, self.slot, self.func + ))?; + f.write_str(")") + } +} + +impl Display for PCIAddress { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if f.alternate() && self.domain == 0 { + f.write_str(&format!("{:04x}:", self.domain))?; + } + + f.write_str(&format!( + "{:02x}:{:02x}.{:x}", + self.bus, self.slot, self.func + )) + } +} + +impl FromStr for PCIAddress { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let mut rev = s.rsplit(":"); + let mut addr = PCIAddress::default(); + + if let Some(slot_and_func) = rev.next() { + let mut splitter = slot_and_func.split("."); + + if let Some(slot) = splitter.next() { + addr.slot = u8::from_str_radix(slot, 16)?; + } + + if let Some(func) = splitter.next() { + addr.func = u8::from_str_radix(func, 16)?; + } + } + + if let Some(bus) = rev.next() { + addr.bus = u8::from_str_radix(bus, 16)?; + } + + if let Some(domain) = rev.next() { + addr.domain = u32::from_str_radix(domain, 16)?; + } + + Ok(addr) + } +} + +#[cfg(test)] +mod tests { + use crate::PCIAddress; + use std::str::FromStr; + + #[test] + fn test_input_and_output_are_same() { + assert_eq!( + PCIAddress::from_str("0000:00:00.1") + .expect("Failed to parse correct string") + .to_string(), + "0000:00:00.1" + ); + + assert_eq!( + PCIAddress::from_str("0000:00:01.0") + .expect("Failed to parse correct string") + .to_string(), + "0000:00:01.0" + ); + } +} diff --git a/vore-core/src/lib.rs b/vore-core/src/lib.rs index 0e92e21..19e45fd 100644 --- a/vore-core/src/lib.rs +++ b/vore-core/src/lib.rs @@ -6,3 +6,4 @@ mod virtual_machine; pub use global_config::*; pub use instance_config::*; pub use qemu::QemuCommandBuilder; +pub use virtual_machine::*; diff --git a/vore-core/src/qemu.rs b/vore-core/src/qemu.rs index 25a18d1..a816a27 100644 --- a/vore-core/src/qemu.rs +++ b/vore-core/src/qemu.rs @@ -259,6 +259,8 @@ impl QemuCommandBuilder { let item = VM::default(); let multi = MultiValue::from_vec(vec![self.lua.to_value(config)?, item.to_lua(&self.lua)?]); + let working_dir = { self.storage.0.lock().unwrap().working_dir.clone() }; + let build_command = if let Some(build_command) = &self .storage .0 @@ -277,9 +279,32 @@ impl QemuCommandBuilder { cmd.push("-name".to_string()); cmd.push(format!("guest={},debug-threads=on", config.name)); + // Don't start the machine cmd.push("-S".to_string()); + + // Set timestamps on log cmd.push("-msg".to_string()); cmd.push("timestamp=on".to_string()); + + // Drop privileges as soon as possible + cmd.push("-runas".to_string()); + cmd.push("nobody".to_string()); + + let working_dir = working_dir + .to_str() + .ok_or_else(|| anyhow::anyhow!("Can't change working directory into string"))?; + + // Control socket + cmd.push("-chardev".to_string()); + cmd.push(format!( + "socket,id=charmonitor,path={}/qemu.sock,server=on,wait=off", + working_dir + )); + + // Set mode to control so we use qapi/qmp instead of readline mode + cmd.push("-mon".to_string()); + cmd.push("chardev=charmonitor,id=monitor,mode=control".to_string()); + cmd.append(&mut vm_instance.args); self.lua.globals().raw_remove("vore")?; diff --git a/vore-core/src/virtual_machine.rs b/vore-core/src/virtual_machine.rs index b16de6f..5ec4af1 100644 --- a/vore-core/src/virtual_machine.rs +++ b/vore-core/src/virtual_machine.rs @@ -1,38 +1,285 @@ use crate::{GlobalConfig, InstanceConfig, QemuCommandBuilder}; +use anyhow::{Context, Error}; +use beau_collector::BeauCollector; +use qapi::qmp::QMP; +use qapi::Qmp; +use std::fmt; +use std::fmt::{Debug, Formatter}; +use std::fs::{read_link, OpenOptions}; +use std::io; +use std::io::{BufReader, ErrorKind, Read, Write}; use std::option::Option::Some; +use std::os::unix::net::UnixStream; use std::path::PathBuf; use std::process::{Child, Command}; +use std::result::Result::Ok; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; #[derive(Debug)] -struct VirtualMachine { +pub struct VirtualMachine { working_dir: PathBuf, config: InstanceConfig, + global_config: GlobalConfig, process: Option, + control_socket: Option, } +struct ControlSocket { + unix_stream: CloneableUnixStream, + qmp: Qmp, CloneableUnixStream>>, + info: QMP, +} + +impl Debug for ControlSocket { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_tuple("ControlSocket") + .field(&self.unix_stream) + .finish() + } +} + +const AUTO_UNBIND_BLACKLIST: &[&str] = &["nvidia"]; + impl VirtualMachine { - pub fn new(config: InstanceConfig, working_dir: PathBuf) -> VirtualMachine { + pub fn new( + config: InstanceConfig, + global_config: &GlobalConfig, + working_dir: PathBuf, + ) -> VirtualMachine { VirtualMachine { working_dir, config, + global_config: global_config.clone(), process: None, + control_socket: None, } } - pub fn start(&mut self, global_config: &GlobalConfig) -> Result<(), anyhow::Error> { + pub fn prepare(&mut self, execute_fixes: bool, force: bool) -> Result<(), anyhow::Error> { + let mut results = vec![]; + results.extend(self.prepare_disks()); + results.extend(self.prepare_vfio(execute_fixes, force)); + results + .into_iter() + .bcollect::<()>() + .with_context(|| format!("Failed to prepare VM {}", self.config.name))?; + Ok(()) + } + + /// + /// Doesn't really prepare them, but mostly checks if the user has permissions to read them + /// + pub fn prepare_disks(&self) -> Vec> { + self.config + .disks + .iter() + .map(|disk| { + OpenOptions::new() + .read(true) + .open(&disk.path) + .with_context(|| format!("Failed to open disk {}", disk.path))?; + + Ok(()) + }) + .collect::>() + } + + /// Prepare VFIO related shenanigans, + /// This includes if requested via [execute_fixes] unbinding the requested vfio pci devices + /// And binding them to vfio-pci + /// + /// With [execute_fixes] set to false, it will only check if everything is sane, and the correct driver is loaded + fn prepare_vfio(&mut self, execute_fixes: bool, force: bool) -> Vec> { + self.config.vfio.iter().map(|vfio| { + let pci_driver_path = format!("/sys/bus/pci/devices/{:#}/driver", vfio.address); + + let driver = match read_link(&pci_driver_path) { + Ok(driver_link) => { + let driver_path = driver_link.to_str().ok_or_else(|| { + anyhow::anyhow!( + "Path to device driver for PCI device at {} is not valid utf-8", + vfio.address + ) + })?; + let driver = driver_path.split("/").last().ok_or_else(|| { + anyhow::anyhow!( + "Path to device driver for PCI device at {} doesn't have a path to a driver", + vfio.address + ) + })?; + + driver.to_string() + } + + Err(err) if err.kind() == ErrorKind::NotFound => "".to_string(), + + Err(err) => return Err(err.into()), + }; + + let is_blacklisted = AUTO_UNBIND_BLACKLIST.contains(&driver.as_str()) && !force; + + if driver != "vfio-pci" && (!execute_fixes || is_blacklisted) { + if !driver.is_empty() && is_blacklisted { + anyhow::bail!("PCI device {} it's current driver is {}, but to be used with VFIO needs to be set to vfio-pci, this driver ({1}) has been blacklisted from automatic rebinding because it can't be cleanly unbound, please make sure this device is unbound before running vore", vfio.address, driver) + } else if !driver.is_empty() { + anyhow::bail!("PCI device {} it's current driver is {}, but to be used with VFIO needs to be set to vfio-pci", vfio.address, driver) + } else { + anyhow::bail!("PCI device at {} currently has no driver, but to be used with VFIO needs to be set to vfio-pci", vfio.address) + } + } + + if driver != "vfio-pci" && execute_fixes && !is_blacklisted { + let address = format!("{:#}\n", vfio.address).into_bytes(); + + if !driver.is_empty() { + let mut unbind = std::fs::OpenOptions::new().append(true).open(format!( + "/sys/bus/pci/devices/{:#}/driver/unbind", + vfio.address + ))?; + + unbind.write_all(&address)?; + } + + { + let mut driver_override = OpenOptions::new().append(true).open(format!( + "/sys/bus/pci/devices/{:#}/driver_override", + vfio.address + ))?; + + driver_override.write_all(b"vfio-pci\n")?; + } + + let mut probe = OpenOptions::new() + .append(true) + .open("/sys/bus/pci/drivers_probe")?; + probe.write_all(&address)?; + + let new_link = read_link(&pci_driver_path)?; + if !new_link.ends_with("vfio-pci") { + anyhow::bail!("Tried to bind {} to vfio-pci but failed to do so (see /sys/bus/pci/devices/{:#} for more info)", vfio.address, vfio.address) + } + } + + Ok(()) + }) + .collect::>() + } + + pub fn get_cmd_line(&self) -> Result, anyhow::Error> { + let builder = QemuCommandBuilder::new(&self.global_config, self.working_dir.clone())?; + builder.build(&self.config) + } + + pub fn pin_qemu_threads(&self) { + let pid = if let Some(child) = &self.process { + child.id() + } else { + return; + }; + } + + pub fn start(&mut self) -> Result<(), anyhow::Error> { if let Some(proc) = &mut self.process { if proc.try_wait()?.is_none() { return Ok(()); } } - let builder = QemuCommandBuilder::new(global_config, self.working_dir.clone())?; - let cmd = builder.build(&self.config)?; - let mut command = Command::new("qemu-system-x86_64"); - command.args(cmd); + command.args(self.get_cmd_line()?); self.process = Some(command.spawn()?); - Ok(()) + let mut res = || { + let qemu_control_socket = format!("{}/qemu.sock", self.working_dir.to_str().unwrap()); + + let mut unix_stream = UnixStream::connect(&qemu_control_socket); + let mut time = 30; + while let Err(err) = unix_stream { + if time < 0 { + Err(err).context(format!( + "After 30 seconds, QEMU Control socket ({}) didn't come up", + qemu_control_socket + ))?; + } + + std::thread::sleep(Duration::from_secs(1)); + unix_stream = UnixStream::connect(&qemu_control_socket); + time -= 1; + } + + let unix_stream = unix_stream.unwrap(); + let unix_stream = CloneableUnixStream(Arc::new(Mutex::new(unix_stream))); + let mut qmp = Qmp::from_stream(unix_stream.clone()); + + let handshake = qmp.handshake()?; + + let mut control_socket = ControlSocket { + unix_stream, + qmp, + info: handshake, + }; + + self.pin_qemu_threads(); + + control_socket + .qmp + .execute(&qapi_qmp::cont {}) + .context("Failed to send start command on qemu control socket")?; + + control_socket.qmp.nop()?; + + while let Some(event) = control_socket.qmp.events().next() { + println!("event: {:?}", event); + } + + self.control_socket = Some(control_socket); + + Ok(()) + }; + + let result_ = res(); + if result_.is_err() { + if let Some(mut qemu) = self.process.take() { + qemu.kill()?; + qemu.wait()?; + } + } + + result_ + } +} + +#[derive(Clone, Debug)] +struct CloneableUnixStream(Arc>); + +impl CloneableUnixStream { + pub fn lock(&self) -> Result, std::io::Error> { + self.0.lock().map_err(|_| { + io::Error::new( + ErrorKind::Other, + anyhow::anyhow!("Failed to lock UnixStream"), + ) + }) + } +} + +impl Read for CloneableUnixStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let res = self.lock()?.read(buf); + if let Ok(size) = res { + println!("READ: {}", String::from_utf8_lossy(&buf[..size])); + } + res + } +} + +impl Write for CloneableUnixStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.lock()?.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.lock()?.flush() } } diff --git a/vored/src/main.rs b/vored/src/main.rs index 2fd6a8f..d8724be 100644 --- a/vored/src/main.rs +++ b/vored/src/main.rs @@ -1,23 +1,10 @@ use std::path::PathBuf; -use std::process::Command; -use vore_core::{GlobalConfig, InstanceConfig, QemuCommandBuilder}; +use vore_core::{GlobalConfig, InstanceConfig, VirtualMachine}; fn main() { let cfg = InstanceConfig::from_toml(include_str!("../../config/example.toml")).unwrap(); - println!("CONFIG:\n{:#?}", cfg); let global = GlobalConfig::load(include_str!("../../config/global.toml")).unwrap(); - let builder = - QemuCommandBuilder::new(&global, PathBuf::from("/home/eater/.lib/vore/win10")).unwrap(); - let command = builder.build(&cfg).unwrap(); - // .iter() - // .map(|x| format!("'{}'", x)) - // .collect::>() - // .join(" "); - - Command::new("qemu-system-x86_64") - .args(command) - .spawn() - .unwrap() - .wait() - .unwrap(); + let mut vm = VirtualMachine::new(cfg, &global, PathBuf::from("/home/eater/.local/vore/win10")); + vm.prepare(true, false).unwrap(); + vm.start().unwrap(); }