@ -1,24 +1,27 @@
use std ::collections ::{ HashMap } ;
use vore_core ::{ VirtualMachine , InstanceConfig , GlobalConfig , GLOBAL_CONFIG_LOCATION } ;
use std ::os ::unix ::net ::{ UnixListener , SocketAddr , UnixStream } ;
use polling ::{ Poller , Event } ;
use std ::time ::Duration ;
use anyhow ::Context ;
use std ::{ mem , io } ;
use std ::io ::{ Read , Write } ;
use vore_core ::rpc ::{ CommandCenter , Response , Command , AllRequests , AllResponses } ;
use vore_core ::rpc ;
use signal_hook ::low_level ::{ signal_name } ;
use signal_hook ::consts ::{ SIGINT , SIGTERM , SIGHUP } ;
use std ::path ::PathBuf ;
use std ::str ::FromStr ;
use signal_hook ::iterator ::{ SignalsInfo , Signals , Handle } ;
use std ::os ::unix ::io ::AsRawFd ;
use polling ::{ Event , Poller } ;
use signal_hook ::consts ::{ SIGHUP , SIGINT , SIGTERM } ;
use signal_hook ::iterator ::{ Handle , Signals , SignalsInfo } ;
use signal_hook ::low_level ::signal_name ;
use std ::collections ::HashMap ;
use std ::ffi ::CStr ;
use std ::fs ;
use std ::fs ::{ read_dir , read_to_string , DirEntry } ;
use std ::io ::{ Read , Write } ;
use std ::mem ::size_of ;
use std ::os ::unix ::io ::AsRawFd ;
use std ::os ::unix ::net ::{ SocketAddr , UnixListener , UnixStream } ;
use std ::path ::{ Path , PathBuf } ;
use std ::str ::FromStr ;
use std ::time ::Duration ;
use std ::{ io , mem } ;
use vore_core ::consts ::{ VORE_CONFIG , VORE_DIRECTORY , VORE_SOCKET } ;
use vore_core ::rpc ::{ AllRequests , AllResponses , Command , CommandCenter , Response } ;
use vore_core ::{ rpc , VirtualMachineInfo } ;
use vore_core ::{ GlobalConfig , InstanceConfig , VirtualMachine } ;
#[ derive(Debug) ]
struct RPCConnection {
struct R pc Connection {
stream : UnixStream ,
address : SocketAddr ,
buffer : Vec < u8 > ,
@ -27,7 +30,7 @@ struct RPCConnection {
pid : i32 ,
}
impl Write for R PC Connection {
impl Write for R pc Connection {
fn write ( & mut self , buf : & [ u8 ] ) -> io ::Result < usize > {
self . stream . write ( buf )
}
@ -37,16 +40,20 @@ impl Write for RPCConnection {
}
}
impl Read for R PC Connection {
impl Read for R pc Connection {
fn read ( & mut self , buf : & mut [ u8 ] ) -> io ::Result < usize > {
self . stream . read ( buf )
}
}
#[ allow(clippy::char_lit_as_u8) ]
const NEWLINE : u8 = '\n' as u8 ;
impl RPCConnection {
pub fn handle_input ( & mut self , own_id : usize ) -> Result < ( bool , Vec < ( usize , Command ) > ) , anyhow ::Error > {
impl RpcConnection {
pub fn handle_input (
& mut self ,
own_id : usize ,
) -> Result < ( bool , Vec < ( usize , Command ) > ) , anyhow ::Error > {
let mut still_open = true ;
loop {
let mut buffer = vec! [ 0 u8 ; 4096 ] ;
@ -57,13 +64,21 @@ impl RPCConnection {
}
Ok ( amount ) = > self . buffer . extend_from_slice ( & buffer [ .. amount ] ) ,
Err ( err ) if err . kind ( ) = = io ::ErrorKind ::WouldBlock = > break ,
Err ( err ) = > return Err ( err . into ( ) )
Err ( err ) = > return Err ( err . into ( ) ) ,
} ;
}
let mut buffer = mem ::take ( & mut self . buffer ) ;
if still_open {
self . buffer = buffer . split_off ( buffer . iter ( ) . enumerate ( ) . rev ( ) . find ( | ( _ , x ) | * * x = = NEWLINE ) . map ( | ( idx , _ ) | idx + 1 ) . unwrap_or ( buffer . len ( ) ) ) ;
self . buffer = buffer . split_off (
buffer
. iter ( )
. enumerate ( )
. rev ( )
. find ( | ( _ , x ) | * * x = = NEWLINE )
. map ( | ( idx , _ ) | idx + 1 )
. unwrap_or ( buffer . len ( ) ) ,
) ;
}
let mut commands = vec! [ ] ;
@ -73,7 +88,6 @@ impl RPCConnection {
continue ;
}
let lossy = String ::from_utf8_lossy ( part ) ;
match CommandCenter ::read_command ( & lossy ) {
@ -94,9 +108,9 @@ impl RPCConnection {
#[ derive(Clone, Eq, PartialEq, Debug) ]
enum EventTarget {
R PC Listener,
R pc Listener,
Machine ( String ) ,
R PC Connection( usize ) ,
R pc Connection( usize ) ,
None ,
}
@ -105,7 +119,7 @@ pub struct Daemon {
event_key_storage : Vec < EventTarget > ,
global_config : GlobalConfig ,
machines : HashMap < String , VirtualMachine > ,
connections : Vec < Option < R PC Connection> > ,
connections : Vec < Option < R pc Connection> > ,
rpc_listener : UnixListener ,
socket_path : PathBuf ,
poller : Poller ,
@ -117,18 +131,22 @@ pub struct Daemon {
impl Daemon {
pub fn new ( ) -> Result < Daemon , anyhow ::Error > {
log ::debug ! ( "Loading global config ({})" , GLOBAL_CONFIG_LOCATION ) ;
let toml = std ::fs ::read_to_string ( GLOBAL_CONFIG_LOCATION ) ? ;
let global_config = GlobalConfig ::load ( & toml ) ? ;
log ::debug ! ( "Loading global config ({})" , VORE_CONFIG ) ;
let toml = std ::fs ::read_to_string ( VORE_CONFIG ) ? ;
let mut global_config = GlobalConfig ::load ( & toml ) ? ;
log ::debug ! ( "Creating vore daemon" ) ;
let signals = Signals ::new ( & [ SIGINT , SIGHUP ] ) ? ;
let handle = signals . handle ( ) ;
log ::debug ! ( "Bound signal handlers" ) ;
let poller = Poller ::new ( ) . context ( "Failed to make poller" ) ? ;
let socket_path = PathBuf ::from_str ( "/run/vore.sock" ) ? ;
let rpc_listener = UnixListener ::bind ( & socket_path ) . context ( "Failed to bind vore socket" ) ? ;
let socket_path = PathBuf ::from_str ( VORE_SOCKET ) ? ;
let rpc_listener =
UnixListener ::bind ( & socket_path ) . context ( "Failed to bind vore socket" ) ? ;
global_config . vore . chown ( socket_path . to_str ( ) . unwrap ( ) ) ? ;
rpc_listener . set_nonblocking ( true ) ? ;
log ::debug ! ( "Bound to /run/vore.sock" ) ;
log ::debug ! ( "Bound to {}", VORE_SOCKET ) ;
let mut daemon = Daemon {
event_key_storage : vec ! [ ] ,
@ -149,23 +167,106 @@ impl Daemon {
}
pub fn init ( & mut self ) -> Result < ( ) , anyhow ::Error > {
let new_key = self . add_target ( EventTarget ::RPCListener ) ;
self . poller . add ( & self . rpc_listener , Event ::readable ( new_key ) ) ? ;
let new_key = self . add_target ( EventTarget ::RpcListener ) ;
self . poller
. add ( & self . rpc_listener , Event ::readable ( new_key ) ) ? ;
Ok ( ( ) )
}
pub fn load_definitions ( & mut self ) -> Result < ( ) , anyhow ::Error > {
let vm_dir = PathBuf ::from ( format! ( "{}/definitions" , VORE_DIRECTORY ) ) ;
if ! vm_dir . is_dir ( ) {
return Ok ( ( ) ) ;
}
let dir_iter =
read_dir ( & vm_dir ) . with_context ( | | format! ( "Failed to list {:?} for vm's" , & vm_dir ) ) ? ;
let mut process = | entry : Result < DirEntry , io ::Error > | -> anyhow ::Result < ( ) > {
let entry = entry ? ;
let file_name = entry . path ( ) ;
let path = file_name . to_str ( ) . context ( "Entry has invalid UTF-8 path" ) ? ;
if ! path . ends_with ( ".toml" ) {
return Ok ( ( ) ) ;
}
let toml = read_to_string ( path )
. with_context ( | | format! ( "Failed to read VM definition {}" , path ) ) ? ;
self . load_virtual_machine ( & toml , None , false ) ? ;
Ok ( ( ) )
} ;
for entry in dir_iter {
if let Err ( err ) = process ( entry ) {
log ::error ! ( "Failed parsing entry in {:?}: {:?}" , vm_dir , err ) ;
}
}
Ok ( ( ) )
}
pub fn reserve_vfio_devices ( & mut self ) {
for machine in self . machines . values ( ) {
for vfio_device in machine . vfio_devices ( ) {
if ! vfio_device . reserve {
continue ;
}
if let Err ( err ) = VirtualMachine ::prepare_vfio_device ( true , true , & vfio_device ) {
log ::error ! (
"Failed to reserve PCI device {} for {}: {:?}" ,
vfio_device . address ,
machine . name ( ) ,
err
) ;
} else {
log ::info ! (
"Reserved PCI device {} for {}" ,
vfio_device . address ,
machine . name ( )
) ;
}
}
}
}
pub fn auto_start_machines ( & mut self ) {
for machine in self . machines . values_mut ( ) {
if ! machine . should_auto_start ( ) {
continue ;
}
if let Err ( err ) = machine . start ( ) {
log ::error ! ( "Failed to auto-start {}: {:?}" , machine . name ( ) , err ) ;
} else {
log ::info ! ( "Autostarted {}" , machine . name ( ) ) ;
}
}
}
pub fn run ( & mut self ) -> Result < ( ) , anyhow ::Error > {
self . load_definitions ( ) ? ;
self . reserve_vfio_devices ( ) ;
self . auto_start_machines ( ) ;
loop {
let res = self . wait ( ) . context ( "Got error while waiting for new notifications" ) ;
let res = self
. wait ( )
. context ( "Got error while waiting for new notifications" ) ;
match res {
// Interrupted is uh "always" when we get a signal
Err ( err ) if err . downcast_ref ::< io ::Error > ( ) . map ( | x | x . kind ( ) = = io ::ErrorKind ::Interrupted ) . unwrap_or ( false ) = > {
Err ( err )
if err
. downcast_ref ::< io ::Error > ( )
. map ( | x | x . kind ( ) = = io ::ErrorKind ::Interrupted )
. unwrap_or ( false ) = >
{
if ! self . handle_exit_code ( ) ? {
break ;
}
}
err = > err ?
err = > err ? ,
}
if ! self . handle_event_queue ( ) ? {
@ -196,37 +297,63 @@ impl Daemon {
Ok ( ( ) )
}
pub fn load_virtual_machine (
& mut self ,
toml : & str ,
working_directory : Option < String > ,
save : bool ,
) -> anyhow ::Result < VirtualMachineInfo > {
let config = InstanceConfig ::from_toml ( & toml ) ? ;
if save {
let save_file = format! ( "{}/definitions/{}.toml" , VORE_DIRECTORY , config . name ) ;
let file_path = Path ::new ( & save_file ) ;
if let Some ( parent_dir ) = file_path . parent ( ) {
if ! parent_dir . is_dir ( ) {
fs ::create_dir_all ( parent_dir ) ? ;
}
}
fs ::write ( & save_file , toml ) . with_context ( | | {
format! (
"Failed to save vm definition for {} to {}" ,
config . name , save_file
)
} ) ? ;
}
let working_dir = working_directory
. unwrap_or_else ( | | format! ( "{}/instance/{}" , VORE_DIRECTORY , config . name ) ) ;
let vm = VirtualMachine ::new ( config , & self . global_config , working_dir ) ;
let info = vm . info ( ) ;
self . mount_machine ( vm ) ;
Ok ( info )
}
pub fn handle_command ( & mut self , command : & Command ) -> Result < AllResponses , anyhow ::Error > {
let resp = match & command . data {
AllRequests ::Info ( _ ) = > {
rpc ::InfoResponse {
name : "vore" . to_string ( ) ,
version : format ! ( "{}.{}.{}{}" ,
env! ( "CARGO_PKG_VERSION_MAJOR" ) ,
env! ( "CARGO_PKG_VERSION_MINOR" ) ,
env! ( "CARGO_PKG_VERSION_PATCH" ) ,
option_env! ( "CARGO_PKG_VERSION_PRE" ) . unwrap_or ( "" ) ) ,
}
. into_enum ( )
AllRequests ::Info ( _ ) = > rpc ::InfoResponse {
name : "vore" . to_string ( ) ,
version : format ! (
"{}.{}.{}{}" ,
env! ( "CARGO_PKG_VERSION_MAJOR" ) ,
env! ( "CARGO_PKG_VERSION_MINOR" ) ,
env! ( "CARGO_PKG_VERSION_PATCH" ) ,
option_env! ( "CARGO_PKG_VERSION_PRE" ) . unwrap_or ( "" )
) ,
}
AllRequests ::List ( _ ) = > {
rpc ::ListResponse {
items : self . machines . values ( ) . map ( | x | x . info ( ) ) . collect ( )
}
. into_enum ( )
. into_enum ( ) ,
AllRequests ::List ( _ ) = > rpc ::ListResponse {
items : self . machines . values ( ) . map ( | x | x . info ( ) ) . collect ( ) ,
}
AllRequests ::Load ( val ) = > {
let config = InstanceConfig ::from_toml ( & val . toml ) ? ;
let working_dir = val . working_directory . as_ref ( ) . cloned ( ) . unwrap_or_else ( | | format! ( "/var/lib/vore/{}" , config . name ) ) ;
let vm = VirtualMachine ::new ( config , & self . global_config , working_dir ) ;
let info = vm . info ( ) ;
self . mount_machine ( vm ) ;
rpc ::LoadResponse {
info ,
}
. into_enum ( )
. into_enum ( ) ,
AllRequests ::Load ( val ) = > rpc ::LoadResponse {
info : self . load_virtual_machine (
& val . toml ,
val . working_directory . as_ref ( ) . cloned ( ) ,
val . save ,
) ? ,
}
. into_enum ( ) ,
AllRequests ::Prepare ( val ) = > {
if let Some ( machine ) = self . machines . get_mut ( & val . name ) {
machine . prepare ( true , false ) ? ;
@ -280,7 +407,11 @@ impl Daemon {
pub fn handle_exit_code ( & mut self ) -> Result < bool , anyhow ::Error > {
for signal in self . signals . pending ( ) {
log ::info ! ( "Received signal {} ({})" , signal_name ( signal ) . unwrap_or ( "<unknown>" ) , signal ) ;
log ::info ! (
"Received signal {} ({})" ,
signal_name ( signal ) . unwrap_or ( "<unknown>" ) ,
signal
) ;
match signal {
SIGINT | SIGTERM = > return Ok ( false ) ,
_ = > { }
@ -297,8 +428,9 @@ impl Daemon {
log ::debug ! ( "Handling {:?} from target {:?}" , event , item ) ;
match item {
EventTarget ::RPCListener = > {
self . poller . modify ( & self . rpc_listener , Event ::readable ( event . key ) ) ? ;
EventTarget ::RpcListener = > {
self . poller
. modify ( & self . rpc_listener , Event ::readable ( event . key ) ) ? ;
self . accept_rpc_connections ( ) ? ;
}
EventTarget ::Machine ( name ) if self . machines . contains_key ( & name ) = > {
@ -306,15 +438,27 @@ impl Daemon {
machine . boop ( ) ? ;
}
if let Some ( control_socket ) = self . machines . get ( & name ) . and_then ( | x | x . control_stream ( ) ) {
self . poller . modify ( control_socket , Event ::readable ( event . key ) ) ? ;
if let Some ( control_socket ) =
self . machines . get ( & name ) . and_then ( | x | x . control_stream ( ) )
{
self . poller
. modify ( control_socket , Event ::readable ( event . key ) ) ? ;
}
}
EventTarget ::RPCConnection ( rpc_connection_id ) if self . connections . get ( rpc_connection_id ) . map ( Option ::is_some ) . unwrap_or ( false ) = > {
let ( still_open , mut commands ) = if let Some ( rpc_connection ) = & mut self . connections [ rpc_connection_id ] {
EventTarget ::RpcConnection ( rpc_connection_id )
if self
. connections
. get ( rpc_connection_id )
. map ( Option ::is_some )
. unwrap_or ( false ) = >
{
let ( still_open , mut commands ) = if let Some ( rpc_connection ) =
& mut self . connections [ rpc_connection_id ]
{
let input_res = rpc_connection . handle_input ( rpc_connection_id ) ? ;
if input_res . 0 {
self . poller . modify ( & rpc_connection . stream , Event ::readable ( event . key ) ) ? ;
self . poller
. modify ( & rpc_connection . stream , Event ::readable ( event . key ) ) ? ;
}
input_res
@ -342,7 +486,7 @@ impl Daemon {
let ( stream , address ) = match self . rpc_listener . accept ( ) {
Ok ( value ) = > value ,
Err ( err ) if err . kind ( ) = = io ::ErrorKind ::WouldBlock = > return Ok ( ( ) ) ,
Err ( err ) = > return Err ( err )?
Err ( err ) = > return Err ( err .into ( ) ) ,
} ;
stream . set_nonblocking ( true ) ? ;
@ -351,16 +495,25 @@ impl Daemon {
let ucred = unsafe {
let mut ucred : libc ::ucred = mem ::zeroed ( ) ;
let mut length = size_of ::< libc ::ucred > ( ) as u32 ;
libc ::getsockopt ( stream . as_raw_fd ( ) , libc ::SOL_SOCKET , libc ::SO_PEERCRED , ( & mut ucred ) as * mut _ as _ , & mut length ) ;
libc ::getsockopt (
stream . as_raw_fd ( ) ,
libc ::SOL_SOCKET ,
libc ::SO_PEERCRED ,
( & mut ucred ) as * mut _ as _ ,
& mut length ,
) ;
let passwd = libc ::getpwuid ( ucred . uid ) ;
if ! passwd . is_null ( ) {
user = CStr ::from_ptr ( ( * passwd ) . pw_name ) . to_str ( ) . ok ( ) . map ( | x | x . to_string ( ) )
user = CStr ::from_ptr ( ( * passwd ) . pw_name )
. to_str ( )
. ok ( )
. map ( | x | x . to_string ( ) )
}
ucred
} ;
let conn = R PC Connection {
let conn = R pc Connection {
stream ,
address ,
buffer : vec ! [ ] ,
@ -371,24 +524,35 @@ impl Daemon {
log ::info ! (
"Got new RPC connection from {} (pid: {}, socket: {:?})" ,
conn . user . as_ref ( ) . map_or_else ( | | format! ( "uid:{}" , conn . uid ) , | x | format! ( "{} ({})" , x , conn . uid ) ) ,
conn . user . as_ref ( ) . map_or_else (
| | format! ( "uid:{}" , conn . uid ) ,
| x | format! ( "{} ({})" , x , conn . uid ) ,
) ,
conn . pid ,
conn . address ,
) ;
let id = self . add_rpc_connection ( conn ) ;
let event_target = self . add_target ( EventTarget ::RPCConnection ( id ) ) ;
self . poller . add ( & self . connections [ id ] . as_ref ( ) . unwrap ( ) . stream , Event ::readable ( event_target ) ) ? ;
let event_target = self . add_target ( EventTarget ::RpcConnection ( id ) ) ;
self . poller . add (
& self . connections [ id ] . as_ref ( ) . unwrap ( ) . stream ,
Event ::readable ( event_target ) ,
) ? ;
}
}
pub fn wait ( & mut self ) -> Result < ( ) , anyhow ::Error > {
self . poller . wait ( & mut self . queue , Some ( Duration ::from_secs ( 5 ) ) ) ? ;
self . poller
. wait ( & mut self . queue , Some ( Duration ::from_secs ( 5 ) ) ) ? ;
Ok ( ( ) )
}
fn add_target ( & mut self , event_target : EventTarget ) -> usize {
let id = self . event_key_storage . iter ( ) . enumerate ( ) . find ( | ( _ , target ) | target . eq ( & & EventTarget ::None ) ) ;
let id = self
. event_key_storage
. iter ( )
. enumerate ( )
. find ( | ( _ , target ) | target . eq ( & & EventTarget ::None ) ) ;
if let Some ( ( id , _ ) ) = id {
self . event_key_storage [ id ] = event_target ;
return id ;
@ -396,11 +560,15 @@ impl Daemon {
let new_id = self . event_key_storage . len ( ) ;
self . event_key_storage . push ( event_target ) ;
return new_id ;
new_id
}
fn add_rpc_connection ( & mut self , rpc_connection : RPCConnection ) -> usize {
let id = self . connections . iter ( ) . enumerate ( ) . find ( | ( _ , target ) | target . is_none ( ) ) ;
fn add_rpc_connection ( & mut self , rpc_connection : RpcConnection ) -> usize {
let id = self
. connections
. iter ( )
. enumerate ( )
. find ( | ( _ , target ) | target . is_none ( ) ) ;
if let Some ( ( id , _ ) ) = id {
self . connections [ id ] = Some ( rpc_connection ) ;
return id ;
@ -408,11 +576,12 @@ impl Daemon {
let new_id = self . connections . len ( ) ;
self . connections . push ( Some ( rpc_connection ) ) ;
return new_id ;
new_id
}
fn mount_machine ( & mut self , vm : VirtualMachine ) {
log ::info ! ( "Loaded {}" , vm . name ( ) ) ;
let name = vm . name ( ) . to_string ( ) ;
self . machines . insert ( name .clone ( ) , vm ) ;
self . machines . insert ( name vm ) ;
}
}
}
