forked from mirror/async-std
Friedel Ziegelmayer
5 years ago
committed by
GitHub
commit
fc4e472599
@ -0,0 +1,23 @@
|
|||||||
|
//! The runtime.
|
||||||
|
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
|
|
||||||
|
use crate::utils::abort_on_panic;
|
||||||
|
|
||||||
|
pub use reactor::{Reactor, Watcher};
|
||||||
|
pub use runtime::Runtime;
|
||||||
|
|
||||||
|
mod reactor;
|
||||||
|
mod runtime;
|
||||||
|
|
||||||
|
/// The global runtime.
|
||||||
|
pub static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
|
||||||
|
thread::Builder::new()
|
||||||
|
.name("async-std/runtime".to_string())
|
||||||
|
.spawn(|| abort_on_panic(|| RUNTIME.run()))
|
||||||
|
.expect("cannot start a runtime thread");
|
||||||
|
|
||||||
|
Runtime::new()
|
||||||
|
});
|
||||||
@ -0,0 +1,343 @@
|
|||||||
|
use std::cell::Cell;
|
||||||
|
use std::io;
|
||||||
|
use std::iter;
|
||||||
|
use std::sync::atomic::{self, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use crossbeam_deque::{Injector, Steal, Stealer, Worker};
|
||||||
|
use crossbeam_utils::thread::scope;
|
||||||
|
use once_cell::unsync::OnceCell;
|
||||||
|
|
||||||
|
use crate::rt::Reactor;
|
||||||
|
use crate::sync::Spinlock;
|
||||||
|
use crate::task::Runnable;
|
||||||
|
use crate::utils::{abort_on_panic, random};
|
||||||
|
|
||||||
|
thread_local! {
|
||||||
|
/// A reference to the current machine, if the current thread runs tasks.
|
||||||
|
static MACHINE: OnceCell<Arc<Machine>> = OnceCell::new();
|
||||||
|
|
||||||
|
/// This flag is set to true whenever `task::yield_now()` is invoked.
|
||||||
|
static YIELD_NOW: Cell<bool> = Cell::new(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Scheduler {
|
||||||
|
/// Set to `true` while a machine is polling the reactor.
|
||||||
|
polling: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An async runtime.
|
||||||
|
pub struct Runtime {
|
||||||
|
/// The reactor.
|
||||||
|
reactor: Reactor,
|
||||||
|
|
||||||
|
/// The global queue of tasks.
|
||||||
|
injector: Injector<Runnable>,
|
||||||
|
|
||||||
|
/// Handles to local queues for stealing work.
|
||||||
|
stealers: Vec<Stealer<Runnable>>,
|
||||||
|
|
||||||
|
/// Machines to start
|
||||||
|
machines: Vec<Arc<Machine>>,
|
||||||
|
|
||||||
|
/// The scheduler state.
|
||||||
|
sched: Mutex<Scheduler>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
/// Creates a new runtime.
|
||||||
|
pub fn new() -> Runtime {
|
||||||
|
let cpus = num_cpus::get().max(1);
|
||||||
|
let processors: Vec<_> = (0..cpus).map(|_| Processor::new()).collect();
|
||||||
|
|
||||||
|
let machines: Vec<_> = processors
|
||||||
|
.into_iter()
|
||||||
|
.map(|p| Arc::new(Machine::new(p)))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let stealers = machines
|
||||||
|
.iter()
|
||||||
|
.map(|m| m.processor.lock().worker.stealer())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Runtime {
|
||||||
|
reactor: Reactor::new().unwrap(),
|
||||||
|
injector: Injector::new(),
|
||||||
|
stealers,
|
||||||
|
machines,
|
||||||
|
sched: Mutex::new(Scheduler { polling: false }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a reference to the reactor.
|
||||||
|
pub fn reactor(&self) -> &Reactor {
|
||||||
|
&self.reactor
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flushes the task slot so that tasks get run more fairly.
|
||||||
|
pub fn yield_now(&self) {
|
||||||
|
YIELD_NOW.with(|flag| flag.set(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Schedules a task.
|
||||||
|
pub fn schedule(&self, task: Runnable) {
|
||||||
|
MACHINE.with(|machine| {
|
||||||
|
// If the current thread is a worker thread, schedule it onto the current machine.
|
||||||
|
// Otherwise, push it into the global task queue.
|
||||||
|
match machine.get() {
|
||||||
|
None => {
|
||||||
|
self.injector.push(task);
|
||||||
|
self.notify();
|
||||||
|
}
|
||||||
|
Some(m) => m.schedule(&self, task),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs the runtime on the current thread.
|
||||||
|
pub fn run(&self) {
|
||||||
|
scope(|s| {
|
||||||
|
for m in &self.machines {
|
||||||
|
s.builder()
|
||||||
|
.name("async-std/machine".to_string())
|
||||||
|
.spawn(move |_| {
|
||||||
|
abort_on_panic(|| {
|
||||||
|
let _ = MACHINE.with(|machine| machine.set(m.clone()));
|
||||||
|
m.run(self);
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.expect("cannot start a machine thread");
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unparks a thread polling the reactor.
|
||||||
|
fn notify(&self) {
|
||||||
|
atomic::fence(Ordering::SeqCst);
|
||||||
|
self.reactor.notify().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attempts to poll the reactor without blocking on it.
|
||||||
|
///
|
||||||
|
/// Returns `Ok(true)` if at least one new task was woken.
|
||||||
|
///
|
||||||
|
/// This function might not poll the reactor at all so do not rely on it doing anything. Only
|
||||||
|
/// use for optimization.
|
||||||
|
fn quick_poll(&self) -> io::Result<bool> {
|
||||||
|
if let Ok(sched) = self.sched.try_lock() {
|
||||||
|
if !sched.polling {
|
||||||
|
return self.reactor.poll(Some(Duration::from_secs(0)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A thread running a processor.
|
||||||
|
struct Machine {
|
||||||
|
/// Holds the processor until it gets stolen.
|
||||||
|
processor: Spinlock<Processor>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Machine {
|
||||||
|
/// Creates a new machine running a processor.
|
||||||
|
fn new(p: Processor) -> Machine {
|
||||||
|
Machine {
|
||||||
|
processor: Spinlock::new(p),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Schedules a task onto the machine.
|
||||||
|
fn schedule(&self, rt: &Runtime, task: Runnable) {
|
||||||
|
self.processor.lock().schedule(rt, task);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finds the next runnable task.
|
||||||
|
fn find_task(&self, rt: &Runtime) -> Steal<Runnable> {
|
||||||
|
let mut retry = false;
|
||||||
|
|
||||||
|
// First try finding a task in the local queue or in the global queue.
|
||||||
|
if let Some(task) = self.processor.lock().pop_task() {
|
||||||
|
return Steal::Success(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.processor.lock().steal_from_global(rt) {
|
||||||
|
Steal::Empty => {}
|
||||||
|
Steal::Retry => retry = true,
|
||||||
|
Steal::Success(task) => return Steal::Success(task),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try polling the reactor, but don't block on it.
|
||||||
|
let progress = rt.quick_poll().unwrap();
|
||||||
|
|
||||||
|
// Try finding a task in the local queue, which might hold tasks woken by the reactor. If
|
||||||
|
// the local queue is still empty, try stealing from other processors.
|
||||||
|
if progress {
|
||||||
|
if let Some(task) = self.processor.lock().pop_task() {
|
||||||
|
return Steal::Success(task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.processor.lock().steal_from_others(rt) {
|
||||||
|
Steal::Empty => {}
|
||||||
|
Steal::Retry => retry = true,
|
||||||
|
Steal::Success(task) => return Steal::Success(task),
|
||||||
|
}
|
||||||
|
|
||||||
|
if retry { Steal::Retry } else { Steal::Empty }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs the machine on the current thread.
|
||||||
|
fn run(&self, rt: &Runtime) {
|
||||||
|
/// Number of yields when no runnable task is found.
|
||||||
|
const YIELDS: u32 = 3;
|
||||||
|
/// Number of short sleeps when no runnable task in found.
|
||||||
|
const SLEEPS: u32 = 10;
|
||||||
|
/// Number of runs in a row before the global queue is inspected.
|
||||||
|
const RUNS: u32 = 64;
|
||||||
|
|
||||||
|
// The number of times the thread found work in a row.
|
||||||
|
let mut runs = 0;
|
||||||
|
// The number of times the thread didn't find work in a row.
|
||||||
|
let mut fails = 0;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
// Check if `task::yield_now()` was invoked and flush the slot if so.
|
||||||
|
YIELD_NOW.with(|flag| {
|
||||||
|
if flag.replace(false) {
|
||||||
|
self.processor.lock().flush_slot(rt);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// After a number of runs in a row, do some work to ensure no task is left behind
|
||||||
|
// indefinitely. Poll the reactor, steal tasks from the global queue, and flush the
|
||||||
|
// task slot.
|
||||||
|
if runs >= RUNS {
|
||||||
|
runs = 0;
|
||||||
|
rt.quick_poll().unwrap();
|
||||||
|
|
||||||
|
let mut p = self.processor.lock();
|
||||||
|
if let Steal::Success(task) = p.steal_from_global(rt) {
|
||||||
|
p.schedule(rt, task);
|
||||||
|
}
|
||||||
|
|
||||||
|
p.flush_slot(rt);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to find a runnable task.
|
||||||
|
if let Steal::Success(task) = self.find_task(rt) {
|
||||||
|
task.run();
|
||||||
|
runs += 1;
|
||||||
|
fails = 0;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
fails += 1;
|
||||||
|
|
||||||
|
// Yield the current thread a few times.
|
||||||
|
if fails <= YIELDS {
|
||||||
|
thread::yield_now();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put the current thread to sleep a few times.
|
||||||
|
if fails <= YIELDS + SLEEPS {
|
||||||
|
thread::sleep(Duration::from_micros(10));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// One final check for available tasks while the scheduler is locked.
|
||||||
|
if let Some(task) = iter::repeat_with(|| self.find_task(rt))
|
||||||
|
.find(|s| !s.is_retry())
|
||||||
|
.and_then(|s| s.success())
|
||||||
|
{
|
||||||
|
self.schedule(rt, task);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut sched = rt.sched.lock().unwrap();
|
||||||
|
|
||||||
|
if sched.polling {
|
||||||
|
thread::sleep(Duration::from_micros(10));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
sched.polling = true;
|
||||||
|
drop(sched);
|
||||||
|
|
||||||
|
rt.reactor.poll(None).unwrap();
|
||||||
|
|
||||||
|
let mut sched = rt.sched.lock().unwrap();
|
||||||
|
sched.polling = false;
|
||||||
|
|
||||||
|
runs = 0;
|
||||||
|
fails = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Processor {
|
||||||
|
/// The local task queue.
|
||||||
|
worker: Worker<Runnable>,
|
||||||
|
|
||||||
|
/// Contains the next task to run as an optimization that skips the queue.
|
||||||
|
slot: Option<Runnable>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Processor {
|
||||||
|
/// Creates a new processor.
|
||||||
|
fn new() -> Processor {
|
||||||
|
Processor {
|
||||||
|
worker: Worker::new_fifo(),
|
||||||
|
slot: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Schedules a task to run on this processor.
|
||||||
|
fn schedule(&mut self, rt: &Runtime, task: Runnable) {
|
||||||
|
match self.slot.replace(task) {
|
||||||
|
None => {}
|
||||||
|
Some(task) => {
|
||||||
|
self.worker.push(task);
|
||||||
|
rt.notify();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flushes a task from the slot into the local queue.
|
||||||
|
fn flush_slot(&mut self, rt: &Runtime) {
|
||||||
|
if let Some(task) = self.slot.take() {
|
||||||
|
self.worker.push(task);
|
||||||
|
rt.notify();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pops a task from this processor.
|
||||||
|
fn pop_task(&mut self) -> Option<Runnable> {
|
||||||
|
self.slot.take().or_else(|| self.worker.pop())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Steals a task from the global queue.
|
||||||
|
fn steal_from_global(&self, rt: &Runtime) -> Steal<Runnable> {
|
||||||
|
rt.injector.steal_batch_and_pop(&self.worker)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Steals a task from other processors.
|
||||||
|
fn steal_from_others(&self, rt: &Runtime) -> Steal<Runnable> {
|
||||||
|
// Pick a random starting point in the list of queues.
|
||||||
|
let len = rt.stealers.len();
|
||||||
|
let start = random(len as u32) as usize;
|
||||||
|
|
||||||
|
// Create an iterator over stealers that starts from the chosen point.
|
||||||
|
let (l, r) = rt.stealers.split_at(start);
|
||||||
|
let stealers = r.iter().chain(l.iter());
|
||||||
|
|
||||||
|
// Try stealing a batch of tasks from each queue.
|
||||||
|
stealers
|
||||||
|
.map(|s| s.steal_batch_and_pop(&self.worker))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,89 @@
|
|||||||
|
use std::cell::UnsafeCell;
|
||||||
|
use std::ops::{Deref, DerefMut};
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
use crossbeam_utils::Backoff;
|
||||||
|
|
||||||
|
/// A simple spinlock.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Spinlock<T> {
|
||||||
|
locked: AtomicBool,
|
||||||
|
value: UnsafeCell<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl<T: Send> Send for Spinlock<T> {}
|
||||||
|
unsafe impl<T: Send> Sync for Spinlock<T> {}
|
||||||
|
|
||||||
|
impl<T> Spinlock<T> {
|
||||||
|
/// Returns a new spinlock initialized with `value`.
|
||||||
|
pub const fn new(value: T) -> Spinlock<T> {
|
||||||
|
Spinlock {
|
||||||
|
locked: AtomicBool::new(false),
|
||||||
|
value: UnsafeCell::new(value),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Locks the spinlock.
|
||||||
|
pub fn lock(&self) -> SpinlockGuard<'_, T> {
|
||||||
|
let backoff = Backoff::new();
|
||||||
|
while self.locked.compare_and_swap(false, true, Ordering::Acquire) {
|
||||||
|
backoff.snooze();
|
||||||
|
}
|
||||||
|
SpinlockGuard { parent: self }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A guard holding a spinlock locked.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct SpinlockGuard<'a, T> {
|
||||||
|
parent: &'a Spinlock<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl<T: Send> Send for SpinlockGuard<'_, T> {}
|
||||||
|
unsafe impl<T: Sync> Sync for SpinlockGuard<'_, T> {}
|
||||||
|
|
||||||
|
impl<'a, T> Drop for SpinlockGuard<'a, T> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.parent.locked.store(false, Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a, T> Deref for SpinlockGuard<'a, T> {
|
||||||
|
type Target = T;
|
||||||
|
|
||||||
|
fn deref(&self) -> &T {
|
||||||
|
unsafe { &*self.parent.value.get() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a, T> DerefMut for SpinlockGuard<'a, T> {
|
||||||
|
fn deref_mut(&mut self) -> &mut T {
|
||||||
|
unsafe { &mut *self.parent.value.get() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn spinlock() {
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::sync::{Spinlock};
|
||||||
|
use crate::task;
|
||||||
|
|
||||||
|
task::block_on(async {
|
||||||
|
|
||||||
|
let m = Arc::new(Spinlock::new(0));
|
||||||
|
let mut tasks = vec![];
|
||||||
|
|
||||||
|
for _ in 0..10 {
|
||||||
|
let m = m.clone();
|
||||||
|
tasks.push(task::spawn(async move {
|
||||||
|
*m.lock() += 1;
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
for t in tasks {
|
||||||
|
t.await;
|
||||||
|
}
|
||||||
|
assert_eq!(*m.lock(), 10);
|
||||||
|
})
|
||||||
|
}
|
||||||
@ -1,13 +0,0 @@
|
|||||||
//! Task executor.
|
|
||||||
//!
|
|
||||||
//! API bindings between `crate::task` and this module are very simple:
|
|
||||||
//!
|
|
||||||
//! * The only export is the `schedule` function.
|
|
||||||
//! * The only import is the `crate::task::Runnable` type.
|
|
||||||
|
|
||||||
pub(crate) use pool::schedule;
|
|
||||||
|
|
||||||
use sleepers::Sleepers;
|
|
||||||
|
|
||||||
mod pool;
|
|
||||||
mod sleepers;
|
|
||||||
@ -1,179 +0,0 @@
|
|||||||
use std::cell::Cell;
|
|
||||||
use std::iter;
|
|
||||||
use std::thread;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use crossbeam_deque::{Injector, Stealer, Worker};
|
|
||||||
use once_cell::sync::Lazy;
|
|
||||||
use once_cell::unsync::OnceCell;
|
|
||||||
|
|
||||||
use crate::task::executor::Sleepers;
|
|
||||||
use crate::task::Runnable;
|
|
||||||
use crate::utils::{abort_on_panic, random};
|
|
||||||
|
|
||||||
/// The state of an executor.
|
|
||||||
struct Pool {
|
|
||||||
/// The global queue of tasks.
|
|
||||||
injector: Injector<Runnable>,
|
|
||||||
|
|
||||||
/// Handles to local queues for stealing work from worker threads.
|
|
||||||
stealers: Vec<Stealer<Runnable>>,
|
|
||||||
|
|
||||||
/// Used for putting idle workers to sleep and notifying them when new tasks come in.
|
|
||||||
sleepers: Sleepers,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Global executor that runs spawned tasks.
|
|
||||||
static POOL: Lazy<Pool> = Lazy::new(|| {
|
|
||||||
let num_threads = num_cpus::get().max(1);
|
|
||||||
let mut stealers = Vec::new();
|
|
||||||
|
|
||||||
// Spawn worker threads.
|
|
||||||
for _ in 0..num_threads {
|
|
||||||
let worker = Worker::new_fifo();
|
|
||||||
stealers.push(worker.stealer());
|
|
||||||
|
|
||||||
let proc = Processor {
|
|
||||||
worker,
|
|
||||||
slot: Cell::new(None),
|
|
||||||
slot_runs: Cell::new(0),
|
|
||||||
};
|
|
||||||
|
|
||||||
thread::Builder::new()
|
|
||||||
.name("async-std/executor".to_string())
|
|
||||||
.spawn(|| {
|
|
||||||
let _ = PROCESSOR.with(|p| p.set(proc));
|
|
||||||
abort_on_panic(main_loop);
|
|
||||||
})
|
|
||||||
.expect("cannot start a thread driving tasks");
|
|
||||||
}
|
|
||||||
|
|
||||||
Pool {
|
|
||||||
injector: Injector::new(),
|
|
||||||
stealers,
|
|
||||||
sleepers: Sleepers::new(),
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
/// The state of a worker thread.
|
|
||||||
struct Processor {
|
|
||||||
/// The local task queue.
|
|
||||||
worker: Worker<Runnable>,
|
|
||||||
|
|
||||||
/// Contains the next task to run as an optimization that skips queues.
|
|
||||||
slot: Cell<Option<Runnable>>,
|
|
||||||
|
|
||||||
/// How many times in a row tasks have been taked from the slot rather than the queue.
|
|
||||||
slot_runs: Cell<u32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
thread_local! {
|
|
||||||
/// Worker thread state.
|
|
||||||
static PROCESSOR: OnceCell<Processor> = OnceCell::new();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Schedules a new runnable task for execution.
|
|
||||||
pub(crate) fn schedule(task: Runnable) {
|
|
||||||
PROCESSOR.with(|proc| {
|
|
||||||
// If the current thread is a worker thread, store it into its task slot or push it into
|
|
||||||
// its local task queue. Otherwise, push it into the global task queue.
|
|
||||||
match proc.get() {
|
|
||||||
Some(proc) => {
|
|
||||||
// Replace the task in the slot.
|
|
||||||
if let Some(task) = proc.slot.replace(Some(task)) {
|
|
||||||
// If the slot already contained a task, push it into the local task queue.
|
|
||||||
proc.worker.push(task);
|
|
||||||
POOL.sleepers.notify_one();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
POOL.injector.push(task);
|
|
||||||
POOL.sleepers.notify_one();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Main loop running a worker thread.
|
|
||||||
fn main_loop() {
|
|
||||||
/// Number of yields when no runnable task is found.
|
|
||||||
const YIELDS: u32 = 3;
|
|
||||||
/// Number of short sleeps when no runnable task in found.
|
|
||||||
const SLEEPS: u32 = 1;
|
|
||||||
|
|
||||||
// The number of times the thread didn't find work in a row.
|
|
||||||
let mut fails = 0;
|
|
||||||
|
|
||||||
loop {
|
|
||||||
// Try to find a runnable task.
|
|
||||||
match find_runnable() {
|
|
||||||
Some(task) => {
|
|
||||||
fails = 0;
|
|
||||||
|
|
||||||
// Run the found task.
|
|
||||||
task.run();
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
fails += 1;
|
|
||||||
|
|
||||||
// Yield the current thread or put it to sleep.
|
|
||||||
if fails <= YIELDS {
|
|
||||||
thread::yield_now();
|
|
||||||
} else if fails <= YIELDS + SLEEPS {
|
|
||||||
thread::sleep(Duration::from_micros(10));
|
|
||||||
} else {
|
|
||||||
POOL.sleepers.wait();
|
|
||||||
fails = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Find the next runnable task.
|
|
||||||
fn find_runnable() -> Option<Runnable> {
|
|
||||||
/// Maximum number of times the slot can be used in a row.
|
|
||||||
const SLOT_LIMIT: u32 = 16;
|
|
||||||
|
|
||||||
PROCESSOR.with(|proc| {
|
|
||||||
let proc = proc.get().unwrap();
|
|
||||||
|
|
||||||
// Try taking a task from the slot.
|
|
||||||
let runs = proc.slot_runs.get();
|
|
||||||
if runs < SLOT_LIMIT {
|
|
||||||
if let Some(task) = proc.slot.take() {
|
|
||||||
proc.slot_runs.set(runs + 1);
|
|
||||||
return Some(task);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
proc.slot_runs.set(0);
|
|
||||||
|
|
||||||
// Pop a task from the local queue, if not empty.
|
|
||||||
proc.worker.pop().or_else(|| {
|
|
||||||
// Otherwise, we need to look for a task elsewhere.
|
|
||||||
iter::repeat_with(|| {
|
|
||||||
// Try stealing a batch of tasks from the global queue.
|
|
||||||
POOL.injector
|
|
||||||
.steal_batch_and_pop(&proc.worker)
|
|
||||||
// Or try stealing a batch of tasks from one of the other threads.
|
|
||||||
.or_else(|| {
|
|
||||||
// First, pick a random starting point in the list of local queues.
|
|
||||||
let len = POOL.stealers.len();
|
|
||||||
let start = random(len as u32) as usize;
|
|
||||||
|
|
||||||
// Try stealing a batch of tasks from each local queue starting from the
|
|
||||||
// chosen point.
|
|
||||||
let (l, r) = POOL.stealers.split_at(start);
|
|
||||||
let stealers = r.iter().chain(l.iter());
|
|
||||||
stealers
|
|
||||||
.map(|s| s.steal_batch_and_pop(&proc.worker))
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
// Loop while no task was stolen and any steal operation needs to be retried.
|
|
||||||
.find(|s| !s.is_retry())
|
|
||||||
// Extract the stolen task, if there is one.
|
|
||||||
.and_then(|s| s.success())
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@ -1,52 +0,0 @@
|
|||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::{Condvar, Mutex};
|
|
||||||
|
|
||||||
/// The place where worker threads go to sleep.
|
|
||||||
///
|
|
||||||
/// Similar to how thread parking works, if a notification comes up while no threads are sleeping,
|
|
||||||
/// the next thread that attempts to go to sleep will pick up the notification immediately.
|
|
||||||
pub struct Sleepers {
|
|
||||||
/// How many threads are currently a sleep.
|
|
||||||
sleep: Mutex<usize>,
|
|
||||||
|
|
||||||
/// A condvar for notifying sleeping threads.
|
|
||||||
wake: Condvar,
|
|
||||||
|
|
||||||
/// Set to `true` if a notification came up while nobody was sleeping.
|
|
||||||
notified: AtomicBool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Sleepers {
|
|
||||||
/// Creates a new `Sleepers`.
|
|
||||||
pub fn new() -> Sleepers {
|
|
||||||
Sleepers {
|
|
||||||
sleep: Mutex::new(0),
|
|
||||||
wake: Condvar::new(),
|
|
||||||
notified: AtomicBool::new(false),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Puts the current thread to sleep.
|
|
||||||
pub fn wait(&self) {
|
|
||||||
let mut sleep = self.sleep.lock().unwrap();
|
|
||||||
|
|
||||||
if !self.notified.swap(false, Ordering::SeqCst) {
|
|
||||||
*sleep += 1;
|
|
||||||
let _ = self.wake.wait(sleep).unwrap();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Notifies one thread.
|
|
||||||
pub fn notify_one(&self) {
|
|
||||||
if !self.notified.load(Ordering::SeqCst) {
|
|
||||||
let mut sleep = self.sleep.lock().unwrap();
|
|
||||||
|
|
||||||
if *sleep > 0 {
|
|
||||||
*sleep -= 1;
|
|
||||||
self.wake.notify_one();
|
|
||||||
} else {
|
|
||||||
self.notified.store(true, Ordering::SeqCst);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue