diff --git a/Cargo.lock b/Cargo.lock index 7e66a71..286352e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,14 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "aho-corasick" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" -dependencies = [ - "memchr", -] - [[package]] name = "anyhow" version = "1.0.40" @@ -78,12 +69,8 @@ checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369" dependencies = [ "lazy_static", "nom", - "rust-ini", - "serde 1.0.125", - "serde-hjson", - "serde_json", + "serde", "toml", - "yaml-rust", ] [[package]] @@ -98,7 +85,7 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0465971a8cc1fa2455c8465aaa377131e1f1cf4983280f474a13e68793aa770c" dependencies = [ - "serde 1.0.125", + "serde", ] [[package]] @@ -130,9 +117,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lexical-core" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags", @@ -147,12 +134,6 @@ version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" -[[package]] -name = "linked-hash-map" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" - [[package]] name = "log" version = "0.4.14" @@ -170,17 +151,17 @@ checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "mlua" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2fc8e1085d53b72898c59ceee1980b5826b0c98ce99886b7518f0ead00e5cb" +checksum = "dd448d3e7018f2ff38dd732a374045f5b037eb4ee477d9241d9bb8c209528c1c" dependencies = [ "bstr", "cc", "erased-serde", "lazy_static", - "num-traits 0.2.14", + "num-traits", "pkg-config", - "serde 1.0.125", + "serde", ] [[package]] @@ -194,15 +175,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.14", -] - [[package]] name = "num-traits" version = "0.2.14" @@ -218,6 +190,19 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "polling" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" +dependencies = [ + "cfg-if", + "libc", + "log", + "wepoll-sys", + "winapi", +] + [[package]] name = "proc-macro2" version = "1.0.26" @@ -236,7 +221,7 @@ dependencies = [ "log", "qapi-qmp", "qapi-spec", - "serde 1.0.125", + "serde", "serde_json", ] @@ -255,7 +240,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2ee46393e919bb03d7ef4c3d633dbbf7d7196fb02b9928ea9fccae7529db07c" dependencies = [ - "serde 1.0.125", + "serde", "serde_json", ] @@ -267,7 +252,7 @@ checksum = "9ef8ca64a52853030d0eac261c34c87978b65060aee7fe54e01659fb0d238500" dependencies = [ "qapi-codegen", "qapi-spec", - "serde 1.0.125", + "serde", ] [[package]] @@ -277,7 +262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b360919a24ea5fc02fa762cb01bd8f43b643fee51c585f763257773b4dc5a9e8" dependencies = [ "base64", - "serde 1.0.125", + "serde", "serde_json", ] @@ -290,41 +275,12 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "regex" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" - -[[package]] -name = "rust-ini" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" - [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" -[[package]] -name = "serde" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" - [[package]] name = "serde" version = "1.0.125" @@ -334,18 +290,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-hjson" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" -dependencies = [ - "lazy_static", - "num-traits 0.1.43", - "regex", - "serde 0.8.23", -] - [[package]] name = "serde_derive" version = "1.0.125" @@ -365,7 +309,7 @@ checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "itoa", "ryu", - "serde 1.0.125", + "serde", ] [[package]] @@ -376,9 +320,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883" dependencies = [ "proc-macro2", "quote", @@ -391,7 +335,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" dependencies = [ - "serde 1.0.125", + "serde", ] [[package]] @@ -422,11 +366,12 @@ dependencies = [ "beau_collector", "config", "kiam", + "lazy_static", "libc", "mlua", "qapi", "qapi-qmp", - "serde 1.0.125", + "serde", "serde_json", "toml", ] @@ -436,14 +381,37 @@ name = "vored" version = "0.1.0" dependencies = [ "anyhow", + "polling", "vore-core", ] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "wepoll-sys" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" dependencies = [ - "linked-hash-map", + "cc", ] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/config/example.toml b/config/example.toml index 2734d1f..5a02a4c 100644 --- a/config/example.toml +++ b/config/example.toml @@ -1,11 +1,15 @@ [machine] name = "win10" memory = "12G" -features = ["uefi", "spice", "scream", "looking-glass"] +features = ["uefi", "spice", "scream"] [cpu] amount = 12 +[[disk]] +preset = "nvme" +path = "/dev/disk/by-id/nvme-eui.6479a74530201073" + [[disk]] preset = "ssd" path = "/dev/disk/by-id/wwn-0x500a0751f008e09d" @@ -14,10 +18,6 @@ path = "/dev/disk/by-id/wwn-0x500a0751f008e09d" preset = "ssd" path = "/dev/disk/by-id/wwn-0x5002538e4038852d" -[[disk]] -preset = "nvme" -path = "/dev/disk/by-id/nvme-eui.6479a74530201073" - #[[vfio]] #vendor = 0x10de #device = 0x1b80 @@ -29,12 +29,12 @@ path = "/dev/disk/by-id/nvme-eui.6479a74530201073" #vendor = 0x10de #device = 0x10f0 #index = 1 - -[[vfio]] -vendor = 0x1022 -device = 0x149c -addr = "0b:00.3" - -[looking-glass] -width = 2560 -height = 1080 \ No newline at end of file +# +#[[vfio]] +#vendor = 0x1022 +#device = 0x149c +#addr = "0b:00.3" +# +#[looking-glass] +#width = 2560 +#height = 1080 diff --git a/config/qemu.lua b/config/qemu.lua index 1cc1a2d..9ff54ae 100644 --- a/config/qemu.lua +++ b/config/qemu.lua @@ -86,7 +86,7 @@ vore:set_build_command(function(instance, vm) end for idx, disk in ipairs(instance.disks) do - vm = vore:add_disk(vm, idx, disk) + vm = vore:add_disk(vm, instance, idx, disk) end if instance.uefi.enabled then @@ -140,14 +140,20 @@ end) --- ---@param type string ----@return fun(vm: VM, idx: number, disk: Disk): VM -function scsi_disk_gen(type) +---@return fun(vm: VM, instance: Instance, idx: number, disk: Disk): VM +function virtio_scsi_disk_gen(type) -- see https://blog.christophersmart.com/2019/12/18/kvm-guests-with-emulated-ssd-and-nvme-drives/ - return function(vm, idx, disk) + return function(vm, _, idx, disk) + local scsi_pci = vm:get_device_id("virtio-scsi-pci") + if scsi_pci == nil then + scsi_pci = "scsi-pci" + vm:arg("-device", "virtio-scsi-pci,id=" .. scsi_pci) + end + vm:arg( "-blockdev", tojson({ - ["driver"] = "raw", + ["driver"] = disk.disk_type, ["file"] = { ["driver"] = "host_device", ["filename"] = disk.path, @@ -162,12 +168,6 @@ function scsi_disk_gen(type) }) ) - local scsi_pci = vm:get_device_id("virtio-scsi-pci") - if scsi_pci == nil then - scsi_pci = "scsi-pci" - vm:arg("-device", "virtio-scsi-pci,id=" .. scsi_pci) - end - local hd = "scsi-hd,drive=format-" .. idx .. ",bus=" .. scsi_pci .. ".0" if type == "ssd" then -- Having a rotation rate of 1 signals Windows it's an ssd @@ -180,13 +180,32 @@ function scsi_disk_gen(type) end end -vore:register_disk_preset("ssd", scsi_disk_gen("ssd")) -vore:register_disk_preset("hdd", scsi_disk_gen("hdd")) -vore:register_disk_preset("nvme", function(vm, _, disk) +--- +---@param name string +---@param device_type string +---@return fun(vm: VM, instance: Instance, idx: number, disk: Disk): VM +function ide_disk_gen(name, device_type) + return function(vm, _, _, disk) + local drive_id = name .. vm:get_counter(name, 1) + + vm:arg("-drive", "file=" .. disk.path .. ",driver=" .. disk.disk_type .. ",if=none,id=" .. drive_id) + vm:arg("-device", device_type .. ",drive=" .. drive_id .. ",bus=ide." .. vm:get_counter("ide", 0)) + + return vm + end +end + +vore:register_disk_preset("ssd", virtio_scsi_disk_gen("ssd")) +vore:register_disk_preset("hdd", virtio_scsi_disk_gen("hdd")) + +vore:register_disk_preset("iso", ide_disk_gen("iso", "ide-cd")) +vore:register_disk_preset("ide", ide_disk_gen("ide", "ide-hd")) + +vore:register_disk_preset("nvme", function(vm, _, _, disk) local nvme_id = vm:get_counter("nvme", 1) -- see https://blog.christophersmart.com/2019/12/18/kvm-guests-with-emulated-ssd-and-nvme-drives/ - vm:arg("-drive", "file=" .. disk.path .. ",driver=raw,if=none,id=NVME" .. nvme_id) + vm:arg("-drive", "file=" .. disk.path .. ",driver=" .. disk.disk_type .. ",if=none,id=NVME" .. nvme_id) vm:arg("-device", "nvme,drive=NVME" .. nvme_id .. ",serial=nvme-" .. nvme_id) return vm diff --git a/resources/vore.def.lua b/resources/vore.def.lua index 3697f1e..dfdf524 100644 --- a/resources/vore.def.lua +++ b/resources/vore.def.lua @@ -100,16 +100,17 @@ end ---- ---Add a disk definition to the argument list ---@param vm VM +---@param instance Instance ---@param index number ---@param disk Disk ---@return VM -function vore:add_disk(vm, index, disk) +function vore:add_disk(vm, instance, index, disk) end ---- ---Register a disk preset ---@param name string ----@param cb fun(vm: VM, idx: number, disk: Disk): VM +---@param cb fun(vm: VM, instance: Instance, idx: number, disk: Disk): VM function vore:register_disk_preset(name, cb) end diff --git a/vore-core/Cargo.toml b/vore-core/Cargo.toml index a3f9636..714e083 100644 --- a/vore-core/Cargo.toml +++ b/vore-core/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -config = "0.11.0" +config = { version = "0.11.0", default-features = false, features = ["toml"] } serde = { version = "1.0.125", features = ["serde_derive"] } serde_json = "1.0.64" toml = "*" @@ -17,4 +17,5 @@ mlua = { version = "0.5.3", features = ["lua54", "serialize", "send"] } beau_collector = "0.2.1" qapi-qmp = "0.7.0" qapi = { version = "0.7.0", features = ["qapi-qmp"] } -libc = "0.2.94" \ No newline at end of file +libc = "0.2.94" +lazy_static = "1.4.0" \ No newline at end of file diff --git a/vore-core/src/cpu_list.rs b/vore-core/src/cpu_list.rs new file mode 100644 index 0000000..9e871f7 --- /dev/null +++ b/vore-core/src/cpu_list.rs @@ -0,0 +1,118 @@ +use lazy_static::lazy_static; + +#[derive(Copy, Clone, Debug)] +pub struct Cpu { + pub id: usize, + pub package: usize, + pub die: usize, + pub core: usize, + pub layer_0: Option, + pub layer_1: Option, + pub layer_2: Option, + pub layer_3: Option, +} + +lazy_static! { + static ref CPUS: Box<[Cpu]> = get_cpus().into_boxed_slice(); + static ref CPU_LIST: CpuList = CpuList { list: &*CPUS }; +} + +pub fn get_cpus() -> Vec { + if cfg!(target_os = "linux") { + return crate::cpu_list::linux::get_cpus(); + } else { + unimplemented!(); + } +} + +#[derive(Copy, Clone, Debug)] +pub struct CpuList { + list: &'static [Cpu], +} + +impl CpuList { + pub fn _get() -> CpuList { + return *CPU_LIST; + } + + pub fn _amount() -> usize { + CPU_LIST.len() + } + + pub fn _load() -> CpuListOwned { + CpuListOwned { list: get_cpus() } + } + + pub fn adjacent(amount: usize) -> Option<&'static [Cpu]> { + CPU_LIST.get_adjacent(amount) + } + + pub fn len(&self) -> usize { + self.list.len() + } + + pub fn _as_slice(&self) -> &[Cpu] { + self.list + } + + pub fn get_adjacent(&self, amount: usize) -> Option<&[Cpu]> { + if self.len() < amount { + None + } else { + Some(&self.list[..amount]) + } + } +} + +#[derive(Clone, Debug)] +pub struct CpuListOwned { + list: Vec, +} + +impl CpuListOwned {} + +#[cfg(target_os = "linux")] +mod linux { + use crate::cpu_list::Cpu; + use std::fs::read_to_string; + use std::str::FromStr; + + pub fn get_cpus() -> Vec { + let cpu = std::fs::read_dir("/sys/devices/system/cpu") + .expect("Failed to read /sys/devices/system/cpu, no /sys mounted?"); + let mut cpus = vec![]; + for cpu_dir in cpu { + let cpu_dir = cpu_dir.unwrap(); + let file_name = cpu_dir.file_name(); + let cpu_name = file_name.to_str().unwrap(); + if cpu_name.starts_with("cpu") && cpu_name[3..].chars().all(|x| x.is_ascii_digit()) { + let cpu_id = usize::from_str(&cpu_name[3..]).unwrap(); + let topology = cpu_dir.path(); + let read_id = |name: &str| -> Option { + let mut path = topology.clone(); + path.push(name); + let id_str = read_to_string(&path).ok()?; + usize::from_str(id_str.trim_end()).ok() + }; + cpus.push(Cpu { + id: cpu_id, + package: read_id("topology/physical_package_id").unwrap(), + die: read_id("topology/die_id").unwrap(), + core: read_id("topology/core_id").unwrap(), + layer_0: read_id("cache/index0/id"), + layer_1: read_id("cache/index1/id"), + layer_2: read_id("cache/index2/id"), + layer_3: read_id("cache/index3/id"), + }) + } + } + + cpus.sort_by_key(|x| { + ( + x.package, x.die, x.layer_3, x.layer_2, x.layer_1, x.layer_0, x.core, x.id, + ) + }); + + cpus + } +} \ No newline at end of file diff --git a/vore-core/src/instance_config.rs b/vore-core/src/instance_config.rs index 06c1f3a..ecc52c3 100644 --- a/vore-core/src/instance_config.rs +++ b/vore-core/src/instance_config.rs @@ -381,7 +381,7 @@ impl LookingGlassConfig { if let Some(mem_path) = table.get("mem-path").cloned() { cfg.mem_path = mem_path.into_str()?; } else { - cfg.mem_path = format!("/dev/shm/{}-looking-glass", name); + cfg.mem_path = format!("/dev/shm/{}/looking-glass", name); } match (table.get("buffer-size").cloned(), table.get("width").cloned(), table.get("height").cloned()) { @@ -415,6 +415,7 @@ pub struct DiskConfig { pub disk_type: String, pub preset: String, pub path: String, + pub read_only: bool, } impl DiskConfig { @@ -430,22 +431,31 @@ impl DiskConfig { disk_type.into_str()? } else { (kiam::when! { - path.starts_with("/dev") => "raw", + path.starts_with("/dev") | path.ends_with(".iso") => "raw", path.ends_with(".qcow2") => "qcow2", _ => return Err(anyhow::Error::msg("Can't figure out from path what type of disk driver should be used")) }).to_string() }; - let preset = table.get("preset").cloned().context("gamer")?.into_str()?; + let preset = table.get("preset") + .cloned() + .context("Every disk should have a preset set")? + .into_str()?; + + let read_only = table.get("read-only") + .cloned() + .map(|x| x.into_bool()) + .transpose() + .context("Failed to read read-only as boolean from config")? + .unwrap_or(false); let disk = DiskConfig { disk_type, preset, path, + read_only, }; - // TODO: Add block dev details - Ok(disk) } } @@ -634,8 +644,8 @@ pub struct PCIAddress { impl<'de> Deserialize<'de> for PCIAddress { fn deserialize(deserializer: D) -> Result>::Error> - where - D: Deserializer<'de>, + where + D: Deserializer<'de>, { struct X; impl Visitor<'_> for X { @@ -657,8 +667,8 @@ impl<'de> Deserialize<'de> for PCIAddress { impl Serialize for PCIAddress { fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> - where - S: Serializer, + where + S: Serializer, { serializer.serialize_str(&self.to_string()) } diff --git a/vore-core/src/lib.rs b/vore-core/src/lib.rs index 19e45fd..b5f05c6 100644 --- a/vore-core/src/lib.rs +++ b/vore-core/src/lib.rs @@ -2,8 +2,11 @@ mod global_config; mod instance_config; mod qemu; mod virtual_machine; +mod cpu_list; +pub mod rpc; pub use global_config::*; pub use instance_config::*; pub use qemu::QemuCommandBuilder; pub use virtual_machine::*; + diff --git a/vore-core/src/qemu.rs b/vore-core/src/qemu.rs index a816a27..ac3a6ad 100644 --- a/vore-core/src/qemu.rs +++ b/vore-core/src/qemu.rs @@ -167,8 +167,8 @@ impl UserData for VoreLuaWeakStorage { methods.add_method( "add_disk", - |lua, weak, args: (VM, u64, mlua::Table)| -> Result { - let (arg_list, index, disk): (VM, u64, Table) = args; + |lua, weak, args: (VM, mlua::Table, u64, mlua::Table)| -> Result { + let (vm, instance, index, disk): (VM, mlua::Table, u64, Table) = args; let function = { let strong = weak .0 @@ -195,7 +195,7 @@ impl UserData for VoreLuaWeakStorage { lua.registry_value::(key)? }; - function.call((arg_list, index, disk)) + function.call((vm, instance, index, disk)) }, ) } diff --git a/vore-core/src/rpc.rs b/vore-core/src/rpc.rs new file mode 100644 index 0000000..e69de29 diff --git a/vore-core/src/virtual_machine.rs b/vore-core/src/virtual_machine.rs index 5ec4af1..f5f1ecf 100644 --- a/vore-core/src/virtual_machine.rs +++ b/vore-core/src/virtual_machine.rs @@ -1,11 +1,11 @@ use crate::{GlobalConfig, InstanceConfig, QemuCommandBuilder}; use anyhow::{Context, Error}; use beau_collector::BeauCollector; -use qapi::qmp::QMP; -use qapi::Qmp; -use std::fmt; +use qapi::qmp::{QMP, Event}; +use qapi::{Qmp, ExecuteError}; +use std::{fmt, mem}; use std::fmt::{Debug, Formatter}; -use std::fs::{read_link, OpenOptions}; +use std::fs::{read_link, OpenOptions, read_dir}; use std::io; use std::io::{BufReader, ErrorKind, Read, Write}; use std::option::Option::Some; @@ -14,11 +14,23 @@ use std::path::PathBuf; use std::process::{Child, Command}; use std::result::Result::Ok; use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; +use std::time::{Duration, Instant}; +use qapi_qmp::QmpCommand; +use std::str::FromStr; +use libc::{cpu_set_t, CPU_SET, sched_setaffinity}; +use crate::cpu_list::CpuList; + +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +pub enum VirtualMachineState { + Stopped, + Paused, + Running, +} #[derive(Debug)] pub struct VirtualMachine { working_dir: PathBuf, + state: VirtualMachineState, config: InstanceConfig, global_config: GlobalConfig, process: Option, @@ -28,7 +40,7 @@ pub struct VirtualMachine { struct ControlSocket { unix_stream: CloneableUnixStream, qmp: Qmp, CloneableUnixStream>>, - info: QMP, + _info: QMP, } impl Debug for ControlSocket { @@ -49,6 +61,7 @@ impl VirtualMachine { ) -> VirtualMachine { VirtualMachine { working_dir, + state: VirtualMachineState::Stopped, config, global_config: global_config.clone(), process: None, @@ -90,7 +103,27 @@ impl VirtualMachine { /// And binding them to vfio-pci /// /// With [execute_fixes] set to false, it will only check if everything is sane, and the correct driver is loaded + /// + /// [force] can be given to auto-bind PCI devices that are blacklisted anyway. this can result in vore indefinitely hanging. fn prepare_vfio(&mut self, execute_fixes: bool, force: bool) -> Vec> { + if self.config.vfio.is_empty() { + return vec![]; + } + + match Command::new("modprobe") + .arg("vfio-pci") + .spawn() + .and_then(|mut x| x.wait()) + { + Err(err) => return vec![Err(err.into())], + Ok(x) if !x.success() => { + return vec![Err(anyhow::anyhow!( + "Failed to load vfio-pci kernel module. can't use VFIO" + ))]; + } + Ok(_) => {} + } + self.config.vfio.iter().map(|vfio| { let pci_driver_path = format!("/sys/bus/pci/devices/{:#}/driver", vfio.address); @@ -133,6 +166,7 @@ impl VirtualMachine { let address = format!("{:#}\n", vfio.address).into_bytes(); if !driver.is_empty() { + // Unbind the PCI device from the current driver let mut unbind = std::fs::OpenOptions::new().append(true).open(format!( "/sys/bus/pci/devices/{:#}/driver/unbind", vfio.address @@ -142,6 +176,7 @@ impl VirtualMachine { } { + // Set a driver override let mut driver_override = OpenOptions::new().append(true).open(format!( "/sys/bus/pci/devices/{:#}/driver_override", vfio.address @@ -150,10 +185,13 @@ impl VirtualMachine { driver_override.write_all(b"vfio-pci\n")?; } - let mut probe = OpenOptions::new() - .append(true) - .open("/sys/bus/pci/drivers_probe")?; - probe.write_all(&address)?; + { + // Probe the PCI device so the driver override is picked up + let mut probe = OpenOptions::new() + .append(true) + .open("/sys/bus/pci/drivers_probe")?; + probe.write_all(&address)?; + } let new_link = read_link(&pci_driver_path)?; if !new_link.ends_with("vfio-pci") { @@ -171,12 +209,184 @@ impl VirtualMachine { builder.build(&self.config) } - pub fn pin_qemu_threads(&self) { + pub fn pin_qemu_threads(&self) -> Result<(), anyhow::Error> { let pid = if let Some(child) = &self.process { child.id() } else { - return; + return Ok(()); }; + + let list = CpuList::adjacent(self.config.cpu.amount as usize); + if list.is_none() { + // If we are over provisioning CPU's there's not much use to pinning + return Ok(()); + } + + let list = list.unwrap(); + + let mut kvm_threads = vec![]; + for item in read_dir(format!("/proc/{}/task", pid))? { + let entry = item?; + if !entry.file_type()?.is_dir() { + continue; + } + + let res = entry.file_name().to_str().ok_or_else(|| anyhow::anyhow!("")).and_then(|x| usize::from_str(x).map_err(From::from)); + if res.is_err() { + continue; + } + + let tid = res.unwrap(); + let name = entry.path().join("comm"); + let comm = std::fs::read_to_string(name)?; + if comm.starts_with("CPU ") { + let nr = comm.chars().skip(4).take_while(|x| x.is_ascii_digit()).collect::(); + let cpu_id = usize::from_str(&nr).unwrap(); + kvm_threads.push((tid, cpu_id)); + } + } + + for (tid, cpu_id) in kvm_threads { + if cpu_id >= list.len() { + // ??? + continue; + } + + let cpu = &list[cpu_id]; + unsafe { + let mut set = mem::zeroed::(); + CPU_SET(cpu.id, &mut set); + sched_setaffinity(tid as i32, mem::size_of::(), &set); + } + } + + Ok(()) + } + + fn process_qmp_events(&mut self) -> Result<(), anyhow::Error> { + let events = if let Some(qmp) = self.control_socket.as_mut() { + // While we could iter, we keep hold of the mutable reference, so it's easier to just collect the events + qmp.qmp.events().collect::>() + } else { + return Ok(()); + }; + + for event in events { + println!("Event: {:?}", event); + + match event { + Event::STOP { .. } => { + if self.state != VirtualMachineState::Stopped { + self.state = VirtualMachineState::Paused; + } + } + Event::RESUME { .. } => { + self.state = VirtualMachineState::Running; + } + Event::SHUTDOWN { .. } => { + self.state = VirtualMachineState::Stopped; + } + + _ => {} + } + } + + Ok(()) + } + + pub fn pause(&mut self) -> Result<(), anyhow::Error> { + if self.state != VirtualMachineState::Running { + return Ok(()); + } + + self.send_qmp_command(&qapi_qmp::stop {})?; + + Ok(()) + } + + fn send_qmp_command(&mut self, command: &C) -> Result { + let res = if let Some(qmp) = self.control_socket.as_mut() { + qmp.qmp.execute(command)? + } else { + anyhow::bail!("No control socket available") + }; + + self.process_qmp_events()?; + Ok(res) + } + + pub fn stop(&mut self) -> Result<(), anyhow::Error> { + if self.process.is_none() || self.control_socket.is_none() || self.state == VirtualMachineState::Stopped { + return Ok(()); + } + + self.send_qmp_command(&qapi_qmp::system_powerdown {})?; + Ok(()) + } + + pub fn stop_now(&mut self) -> Result<(), anyhow::Error> { + self.stop()?; + + if let Some(mut process) = self.process.take() { + self.wait(Some(Duration::from_secs(30)), VirtualMachineState::Stopped)?; + self.quit()?; + process.wait()?; + } + + Ok(()) + } + + pub fn wait_till_stopped(&mut self) -> Result<(), anyhow::Error> { + self.wait(None, VirtualMachineState::Stopped)?; + Ok(()) + } + + pub fn quit(&mut self) -> Result<(), anyhow::Error> { + if self.control_socket.is_none() { + return Ok(()); + } + + self.send_qmp_command(&qapi_qmp::quit {}) + .map(|_| ()) + .or_else(|x| + if let Some(ExecuteError::Io(err)) = x.downcast_ref::() { + if err.kind() == ErrorKind::UnexpectedEof { + Ok(()) + } else { + Err(x) + } + } else { + Err(x) + }) + .map_err(From::from) + } + + fn wait(&mut self, duration: Option, target_state: VirtualMachineState) -> Result { + let start = Instant::now(); + while duration.map_or(true, |dur| (Instant::now() - start) < dur) { + let has_socket = self.control_socket.as_mut() + .map(|x| x.qmp.nop()) + .transpose()? + .is_some(); + + if !has_socket { + return Ok(self.state == target_state); + } + + self.process_qmp_events()?; + + if self.state == target_state { + return Ok(true); + } + + if duration.is_some() { + std::thread::sleep(Duration::from_millis(500)); + } else { + std::thread::sleep(Duration::from_secs(5)); + } + } + + Ok(self.state == target_state) } pub fn start(&mut self) -> Result<(), anyhow::Error> { @@ -192,7 +402,6 @@ impl VirtualMachine { let mut res = || { let qemu_control_socket = format!("{}/qemu.sock", self.working_dir.to_str().unwrap()); - let mut unix_stream = UnixStream::connect(&qemu_control_socket); let mut time = 30; while let Err(err) = unix_stream { @@ -205,6 +414,13 @@ impl VirtualMachine { std::thread::sleep(Duration::from_secs(1)); unix_stream = UnixStream::connect(&qemu_control_socket); + + if let Some(proc) = self.process.as_mut() { + if let Some(_) = proc.try_wait()? { + anyhow::bail!("QEMU quit early") + } + } + time -= 1; } @@ -217,10 +433,10 @@ impl VirtualMachine { let mut control_socket = ControlSocket { unix_stream, qmp, - info: handshake, + _info: handshake, }; - self.pin_qemu_threads(); + // self.pin_qemu_threads()?; control_socket .qmp @@ -228,13 +444,10 @@ impl VirtualMachine { .context("Failed to send start command on qemu control socket")?; control_socket.qmp.nop()?; - - while let Some(event) = control_socket.qmp.events().next() { - println!("event: {:?}", event); - } - self.control_socket = Some(control_socket); + self.process_qmp_events()?; + Ok(()) }; diff --git a/vored/Cargo.toml b/vored/Cargo.toml index 04461ee..eec21fe 100644 --- a/vored/Cargo.toml +++ b/vored/Cargo.toml @@ -8,4 +8,5 @@ edition = "2018" [dependencies] anyhow = "1.0.40" -vore-core = { path = "../vore-core" } \ No newline at end of file +vore-core = { path = "../vore-core" } +polling = "2.0.3" \ No newline at end of file diff --git a/vored/src/daemon.rs b/vored/src/daemon.rs new file mode 100644 index 0000000..24d6d1a --- /dev/null +++ b/vored/src/daemon.rs @@ -0,0 +1,17 @@ +use std::collections::HashMap; +use vore_core::VirtualMachine; +use std::os::unix::net::UnixListener; + +struct RPCConnection {} + +struct DaemonState { + machines: HashMap, + connections: Vec, + rpc_listener: UnixListener, +} + +impl DaemonState { + pub fn wait() { + + } +} \ No newline at end of file diff --git a/vored/src/main.rs b/vored/src/main.rs index d8724be..b3f927a 100644 --- a/vored/src/main.rs +++ b/vored/src/main.rs @@ -1,3 +1,5 @@ +mod daemon; + use std::path::PathBuf; use vore_core::{GlobalConfig, InstanceConfig, VirtualMachine}; @@ -7,4 +9,6 @@ fn main() { let mut vm = VirtualMachine::new(cfg, &global, PathBuf::from("/home/eater/.local/vore/win10")); vm.prepare(true, false).unwrap(); vm.start().unwrap(); + vm.wait_till_stopped().unwrap(); + vm.stop_now().unwrap() }