diff --git a/Cargo.lock b/Cargo.lock index 5bc9205de2..47512af1e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,6 +256,10 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" +[[package]] +name = "bit_set" +version = "0.1.0" + [[package]] name = "bitfield" version = "0.13.2" @@ -3141,6 +3145,8 @@ dependencies = [ name = "scheduler_epoch" version = "0.1.0" dependencies = [ + "bit_set", + "kernel_config", "log", "spin 0.9.4", "task", diff --git a/kernel/scheduler_epoch/Cargo.toml b/kernel/scheduler_epoch/Cargo.toml index e0b65c5a15..63d7c675ae 100644 --- a/kernel/scheduler_epoch/Cargo.toml +++ b/kernel/scheduler_epoch/Cargo.toml @@ -6,13 +6,11 @@ version = "0.1.0" edition = "2021" [dependencies] +bit_set = { path = "../../libs/bit_set" } +kernel_config = { path = "../kernel_config" } +log = "0.4.8" spin = "0.9.4" - -[dependencies.log] -version = "0.4.8" - -[dependencies.task] -path = "../task" +task = { path = "../task" } [lib] crate-type = ["rlib"] diff --git a/kernel/scheduler_epoch/src/lib.rs b/kernel/scheduler_epoch/src/lib.rs index 22a6917466..456691e0ee 100644 --- a/kernel/scheduler_epoch/src/lib.rs +++ b/kernel/scheduler_epoch/src/lib.rs @@ -1,34 +1,49 @@ -//! This crate implements a token-based epoch scheduling policy. +//! A token-based epoch scheduler. //! -//! At the begining of each scheduling epoch, a set of tokens is distributed -//! among all runnable tasks, based on their priority relative to all other -//! runnable tasks in the runqueue. The formula for this is: -//! ```ignore -//! tokens_assigned_to_task_i = (priority_task_i / sum_priority_all_tasks) * epoch_length; -//! ``` -//! * Each time a task is picked, its token count is decremented by 1. -//! * A task can only be selected for next execution if it has tokens remaining. -//! * When all tokens of all runnable task are exhausted, a new scheduling epoch begins. +//! The implementation is based on the [`O(1)` Linux +//! scheduler][linux-scheduler]. //! -//! This epoch scheduler is also a priority-based scheduler, so it allows -//! getting and setting the priorities of each task. +//! The scheduler is comprised of two run queues: an . +//! +//! Note that our implementation is not constant-time since we store +//! non-runnable tasks on the run queue. +//! +//! [linux-scheduler]: https://litux.nl/mirror/kerneldevelopment/0672327201/ch04lev1sec2.html #![no_std] +#![feature(core_intrinsics)] + +mod queue; extern crate alloc; -use alloc::{boxed::Box, collections::VecDeque, vec::Vec}; -use core::ops::{Deref, DerefMut}; +use alloc::{boxed::Box, vec::Vec}; +use core::{ + mem, + ops::{Deref, DerefMut}, + time::Duration, +}; + use task::TaskRef; -const MAX_PRIORITY: u8 = 40; +use crate::queue::RunQueue; + +const MAX_PRIORITY: u8 = 63; const DEFAULT_PRIORITY: u8 = 20; -const INITIAL_TOKENS: usize = 10; -/// An instance of an epoch scheduler, typically one per CPU. +/// The minimum amount of time for every runnable task to run. +/// +/// This is not strictly adhered to when the tasks are run +const TARGET_LATENCY: Duration = Duration::from_millis(15); + +/// An epoch scheduler. +/// +/// See crate-level docs for more information. pub struct Scheduler { idle_task: TaskRef, - queue: VecDeque, + active: RunQueue, + expired: RunQueue, + total_weight: usize, } impl Scheduler { @@ -36,188 +51,194 @@ impl Scheduler { pub const fn new(idle_task: TaskRef) -> Self { Self { idle_task, - queue: VecDeque::new(), + active: RunQueue::new(), + expired: RunQueue::new(), + // TODO: 0 or 1 + total_weight: 0, } } - /// Moves the `TaskRef` at the given `index` in this scheduler's runqueue - /// to the end (back) of the runqueue. - /// - /// Sets the number of tokens for that task to the given `tokens` - /// and increments that task's number of context switches. - /// - /// Returns a cloned reference to the `TaskRef` at the given `index`. - fn update_and_move_to_end(&mut self, index: usize, tokens: usize) -> Option { - if let Some(mut priority_task_ref) = self.queue.remove(index) { - priority_task_ref.tokens_remaining = tokens; - let task_ref = priority_task_ref.task.clone(); - self.queue.push_back(priority_task_ref); - Some(task_ref) + fn apply(&mut self, mut f: F) -> R + where + F: FnMut(&mut RunQueue) -> R, + R: Returnable, + { + let (first, second) = if self.active.len() >= self.expired.len() { + (&mut self.active, &mut self.expired) } else { - None - } - } + (&mut self.expired, &mut self.active) + }; + + let first_result = f(first); - fn try_next(&mut self) -> Option { - if let Some((task_index, _)) = self - .queue - .iter() - .enumerate() - .find(|(_, task)| task.is_runnable() && task.tokens_remaining > 0) - { - let chosen_task = self.queue.get(task_index).unwrap(); - let modified_tokens = chosen_task.tokens_remaining.saturating_sub(1); - self.update_and_move_to_end(task_index, modified_tokens) + if first_result.should_return() { + first_result } else { - None + f(second) } } +} - fn assign_tokens(&mut self) { - // We begin with total priorities = 1 to avoid division by zero - let mut total_priorities: usize = 1; - - // This loop calculates the total priorities of the runqueue - for (_i, t) in self.queue.iter().enumerate() { - // we assign tokens only to runnable tasks - if !t.is_runnable() { - continue; - } - - total_priorities = total_priorities - .saturating_add(1) - .saturating_add(t.priority as usize); - } - - // Each epoch lasts for a total of 100 tokens by default. - // However, as this granularity could skip over low priority tasks - // when many concurrent tasks are running, we increase the epoch in such cases. - let epoch: usize = core::cmp::max(total_priorities, 100); - - for (_i, t) in self.queue.iter_mut().enumerate() { - // we give zero tokens to the idle tasks - if t.is_an_idle_task { - continue; - } - - // we give zero tokens to non-runnable tasks - if !t.is_runnable() { - continue; - } +trait Returnable { + fn should_return(&self) -> bool; +} - // task_tokens = epoch * (taskref + 1) / total_priorities; - let task_tokens = epoch - .saturating_mul((t.priority as usize).saturating_add(1)) - .wrapping_div(total_priorities); +impl Returnable for bool { + fn should_return(&self) -> bool { + *self + } +} - t.tokens_remaining = task_tokens; - // debug!("assign_tokens(): CPU {} chose Task {:?}", cpu_id, &*t); - } +impl Returnable for Option { + fn should_return(&self) -> bool { + self.is_some() } } impl task::scheduler::Scheduler for Scheduler { + #[inline] fn next(&mut self) -> TaskRef { - self.try_next() - .or_else(|| { - self.assign_tokens(); - self.try_next() - }) - .unwrap_or_else(|| self.idle_task.clone()) + if self.active.is_empty() { + if !self.expired.is_empty() { + mem::swap(&mut self.active, &mut self.expired); + } else { + return self.idle_task.clone(); + } + } + self.active + .next(&mut self.expired, self.total_weight) + .unwrap_or(self.idle_task.clone()) } + #[inline] fn add(&mut self, task: TaskRef) { - let priority_task_ref = EpochTaskRef::new(task); - self.queue.push_back(priority_task_ref); + let weight = weight(DEFAULT_PRIORITY); + self.total_weight += weight; + + let task = EpochTaskRef::new( + task, + TaskConfiguration { + weight, + total_weight: self.total_weight, + }, + ); + self.expired.push(task, DEFAULT_PRIORITY); } + #[inline] fn busyness(&self) -> usize { - self.queue.len() + self.active.len() + self.expired.len() } + #[inline] fn remove(&mut self, task: &TaskRef) -> bool { - let mut task_index = None; - for (i, t) in self.queue.iter().enumerate() { - if **t == *task { - task_index = Some(i); - break; + match self.apply(|run_queue| run_queue.remove(task)) { + Some(weight) => { + self.total_weight -= weight; + true } - } - - if let Some(task_index) = task_index { - self.queue.remove(task_index); - true - } else { - false + None => false, } } + #[inline] fn as_priority_scheduler(&mut self) -> Option<&mut dyn task::scheduler::PriorityScheduler> { Some(self) } + #[inline] fn drain(&mut self) -> Box + '_> { - Box::new(self.queue.drain(..).map(|epoch_task| epoch_task.task)) + let mut active = RunQueue::new(); + let mut expired = RunQueue::new(); + + mem::swap(&mut self.active, &mut active); + mem::swap(&mut self.expired, &mut expired); + self.total_weight = 0; + + Box::new(active.drain().chain(expired.drain())) } + #[inline] fn tasks(&self) -> Vec { - self.queue + self.active .clone() - .into_iter() - .map(|epoch_task| epoch_task.task) + .drain() + .chain(self.expired.clone().drain()) .collect() } } impl task::scheduler::PriorityScheduler for Scheduler { + #[inline] fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { let priority = core::cmp::min(priority, MAX_PRIORITY); - for epoch_task in self.queue.iter_mut() { - if epoch_task.task == *task { - epoch_task.priority = priority; - return true; - } - } - false + self.apply(|run_queue| run_queue.set_priority(task, priority)) } + #[inline] fn priority(&mut self, task: &TaskRef) -> Option { - for epoch_task in self.queue.iter() { - if epoch_task.task == *task { - return Some(epoch_task.priority); - } - } - None + self.apply(|run_queue| run_queue.priority(task)) } } +#[inline] +fn weight(priority: u8) -> usize { + priority as usize + 1 +} + #[derive(Debug, Clone)] struct EpochTaskRef { task: TaskRef, - priority: u8, - tokens_remaining: usize, + tokens: usize, +} + +impl EpochTaskRef { + /// Creates a new task. + /// + /// Returns the task and the weight of the task. + #[must_use] + pub(crate) fn new(task: TaskRef, config: TaskConfiguration) -> Self { + let mut task = Self { task, tokens: 0 }; + task.recalculate_tokens(config); + task + } + + #[inline] + pub(crate) fn recalculate_tokens(&mut self, config: TaskConfiguration) { + const TOTAL_TOKENS: usize = TARGET_LATENCY.as_micros() as usize + / kernel_config::time::CONFIG_TIMESLICE_PERIOD_MICROSECONDS as usize; + + // TODO + self.tokens = core::cmp::max(TOTAL_TOKENS * config.weight / config.total_weight, 1); + } +} + +pub(crate) struct TaskConfiguration { + /// The weight of the task. + pub(crate) weight: usize, + /// The sum of the weights of all tasks on the run queue. + pub(crate) total_weight: usize, } impl Deref for EpochTaskRef { type Target = TaskRef; + #[inline] fn deref(&self) -> &TaskRef { &self.task } } impl DerefMut for EpochTaskRef { + #[inline] fn deref_mut(&mut self) -> &mut TaskRef { &mut self.task } } -impl EpochTaskRef { - fn new(task: TaskRef) -> EpochTaskRef { - EpochTaskRef { - task, - priority: DEFAULT_PRIORITY, - tokens_remaining: INITIAL_TOKENS, - } +impl From for TaskRef { + #[inline] + fn from(value: EpochTaskRef) -> Self { + value.task } } diff --git a/kernel/scheduler_epoch/src/queue.rs b/kernel/scheduler_epoch/src/queue.rs new file mode 100644 index 0000000000..f55f11192e --- /dev/null +++ b/kernel/scheduler_epoch/src/queue.rs @@ -0,0 +1,251 @@ +use alloc::collections::VecDeque; + +use bit_set::BitSet; +use task::TaskRef; + +use crate::{weight, EpochTaskRef, TaskConfiguration, MAX_PRIORITY}; + +/// A singular run queue. +/// +/// The scheduler contains two of these: an active one, and an expired one. +#[derive(Debug, Clone)] +pub(crate) struct RunQueue { + // TODO: Encode using MAX_PRIORITY + priorities: BitSet, + len: usize, + inner: [VecDeque; MAX_PRIORITY as usize], +} + +impl RunQueue { + #[inline] + pub(crate) const fn new() -> Self { + const INIT: VecDeque = VecDeque::new(); + + Self { + priorities: BitSet::new(), + len: 0, + inner: [INIT; MAX_PRIORITY as usize], + } + } + + #[inline] + pub(crate) fn len(&self) -> usize { + debug_assert_eq!( + self.inner.iter().map(|queue| queue.len()).sum::(), + self.len + ); + self.len + } + + #[inline] + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub(crate) fn push(&mut self, task: EpochTaskRef, priority: u8) { + self.priorities.insert(priority); + self.inner[priority as usize].push_back(task); + self.len += 1; + } + + #[inline] + pub(crate) fn next(&mut self, expired: &mut Self, total_weight: usize) -> Option { + let mut priorities = self.priorities.clone(); + + let mut top_index = priorities.max()?; + // TODO: top_queue.len() == 1 optimisation + let mut top_queue = &mut self.inner[top_index as usize]; + let mut next_task = top_queue.front().unwrap(); + + if !next_task.is_runnable() { + // TODO: This incredibly convoluted code is necessary because we store + // non-runnable tasks on the run queue. + + // Iterate through the queue to find the next runnable task and bring it to the + // front of its respective run queue. + + let mut vec_index = 0; + + while !next_task.is_runnable() { + assert!(!top_queue.is_empty()); + + if vec_index + 1 == top_queue.len() { + priorities.remove(top_index); + top_index = match priorities.max() { + Some(top) => top, + None => { + // There are no runnable tasks on the run queue. We + // must transfer all the tasks to the expired run + // queue and return None. + + let mut priorities = self.priorities.clone(); + + while let Some(top_index) = priorities.max() { + let top_queue = &mut self.inner[top_index as usize]; + + while let Some(mut task) = top_queue.pop_front() { + task.recalculate_tokens(TaskConfiguration { + weight: weight(top_index), + total_weight, + }); + expired.push(task, top_index); + } + + priorities.remove(top_index); + } + + self.priorities = BitSet::new(); + self.len = 0; + + return None; + } + }; + top_queue = &mut self.inner[top_index as usize]; + vec_index = 0; + } + + vec_index += 1; + next_task = &top_queue[vec_index]; + } + + for _ in 0..vec_index { + let task = top_queue.pop_front().unwrap(); + top_queue.push_back(task); + } + } + + let queue = &mut self.inner[top_index as usize]; + let next_task = queue.front().unwrap(); + + Some(if next_task.tokens <= 1 { + let mut next_task = queue.pop_front().unwrap(); + self.len -= 1; + + next_task.recalculate_tokens(TaskConfiguration { + weight: weight(top_index), + total_weight, + }); + expired.push(next_task.clone(), top_index); + + if queue.is_empty() { + self.priorities.remove(top_index); + } + + next_task.clone().task + } else { + let mut next_task = queue.pop_front().unwrap(); + + next_task.tokens -= 1; + queue.push_back(next_task.clone()); + + next_task.task + }) + } + + #[inline] + fn top_index(&self) -> Option { + self.priorities.max().map(|priority| priority as usize) + } + + #[inline] + pub(crate) fn remove(&mut self, task: &TaskRef) -> Option { + for priority in self.priorities.iter() { + let queue = &mut self.inner[priority]; + + for j in 0..queue.len() { + let element = &queue[j]; + + if **element == *task { + queue.remove(j); + self.len -= 1; + + if queue.is_empty() { + self.priorities.remove(priority as u8); + } + + return Some(weight(priority as u8)); + } + } + } + None + } + + /// Returns the priority of the given task. + #[inline] + pub(crate) fn priority(&self, task: &TaskRef) -> Option { + for i in self.priorities.iter() { + let queue = &self.inner[i]; + for t in queue { + if **t == *task { + return Some(i as u8); + } + } + } + None + } + + /// Sets the priority of the given task. + /// + /// Returns `true` if an action was performed i.e. if the task was in the + /// run queue. + #[inline] + pub(crate) fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { + // FIXME: Recalculate weights. + for i in self.priorities.iter() { + let queue = &mut self.inner[i]; + + for j in 0..queue.len() { + let element = &queue[j]; + + if **element == *task { + let task = queue.remove(j).unwrap(); + self.len -= 1; + + if queue.is_empty() { + self.priorities.remove(i as u8); + } + + self.push(task, priority); + return true; + } + } + } + // false + todo!(); + } + + #[inline] + pub(crate) fn drain(self) -> Drain { + Drain { inner: self } + } +} + +impl IntoIterator for RunQueue { + type Item = TaskRef; + + type IntoIter = Drain; + + fn into_iter(self) -> Self::IntoIter { + self.drain() + } +} + +pub(crate) struct Drain { + inner: RunQueue, +} + +impl Iterator for Drain { + type Item = TaskRef; + + fn next(&mut self) -> Option { + let top_index = self.inner.top_index()?; + let top_queue = &mut self.inner.inner[top_index]; + + if top_queue.len() == 1 { + self.inner.priorities.remove(top_index as u8); + } + + Some(top_queue.pop_front().unwrap().into()) + } +} diff --git a/libs/bit_set/Cargo.toml b/libs/bit_set/Cargo.toml new file mode 100644 index 0000000000..af52627a32 --- /dev/null +++ b/libs/bit_set/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "bit_set" +version = "0.1.0" +authors = ["Klim Tsoutsman "] +description = "A bit set storing integers less than 64" +edition = "2021" diff --git a/libs/bit_set/src/iter.rs b/libs/bit_set/src/iter.rs new file mode 100644 index 0000000000..135f36c3c2 --- /dev/null +++ b/libs/bit_set/src/iter.rs @@ -0,0 +1,67 @@ +use core::intrinsics::unlikely; + +/// An iterator over a [`BitSet`]. +/// +/// [`BitSet`]: crate::BitSet +pub struct Iter { + set: u64, + current_mask: u64, +} + +impl Iter { + pub(crate) const fn new(set: u64) -> Self { + Self { + set, + current_mask: u64::MAX, + } + } +} + +impl Iterator for Iter { + type Item = usize; + + fn next(&mut self) -> Option { + let next_index = (self.set & self.current_mask).trailing_zeros(); + + if unlikely(next_index == 64) { + None + } else { + // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 + self.current_mask = u64::MAX.checked_shl(next_index + 1).unwrap_or(0); + Some(next_index as usize) + } + } +} + +#[cfg(test)] +mod tests { + extern crate alloc; + + use alloc::vec::Vec; + + use crate::BitSet; + + #[test] + fn test_iter() { + let mut set = BitSet::new(); + set.insert(57); + set.insert(58); + set.insert(61); + set.insert(63); + assert_eq!(set.iter().collect::>(), [57, 58, 61, 63]); + + let mut set = BitSet::new(); + set.insert(0); + set.insert(8); + set.insert(16); + set.insert(24); + set.insert(32); + set.insert(40); + set.insert(48); + set.insert(56); + assert_eq!( + set.iter().collect::>(), + [0, 8, 16, 24, 32, 40, 48, 56] + ); + } +} diff --git a/libs/bit_set/src/lib.rs b/libs/bit_set/src/lib.rs new file mode 100644 index 0000000000..575ce64a90 --- /dev/null +++ b/libs/bit_set/src/lib.rs @@ -0,0 +1,166 @@ +//! A bit set backed by a [`u64`]. +//! +//! See [`BitSet`] for more details. + +#![no_std] +#![feature(const_likely, core_intrinsics)] + +mod iter; + +use core::intrinsics::likely; + +pub use iter::Iter; + +/// A bit set backed by a [`u64`]. +/// +/// This is equivalent to a `HashSet` storing integers in the range `[0, +/// 64)`. +#[derive(Debug, Clone)] +pub struct BitSet { + inner: u64, +} + +impl BitSet { + /// Constructs a new, empty `BitSet`. + pub const fn new() -> Self { + Self { inner: 0 } + } + + /// Returns an iterator over the elements of the set. + #[must_use] + pub const fn iter(&self) -> Iter { + Iter::new(self.inner) + } + + /// Returns `true` if the set contains the given element. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + #[must_use] + pub const fn contains(&self, element: u8) -> bool { + assert!(element < 64); + self.inner & (1 << element) != 0 + } + + /// Adds an element to the set. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + pub fn insert(&mut self, element: u8) { + assert!(element < 64); + self.inner |= 1 << element; + } + + /// Removes an element from the set. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + pub fn remove(&mut self, element: u8) { + assert!(element < 64); + self.inner &= !(1 << element); + } + + /// Returns the smallest element in the set. + /// + /// Returns `None` if the set is empty. + #[must_use] + pub const fn min(&self) -> Option { + if likely(self.inner != 0) { + Some(self.inner.trailing_zeros() as u8) + } else { + None + } + } + + /// Returns the largest element in the set. + /// + /// Returns `None` if the set is empty. + #[must_use] + pub const fn max(&self) -> Option { + if likely(self.inner != 0) { + // self.inner.leading_zeros() <= 63 + Some(63 - self.inner.leading_zeros() as u8) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_contains() { + let mut set = BitSet::new(); + + for i in 0..64 { + assert!(!set.contains(i)); + } + + set.insert(3); + + for i in 0..64 { + if i != 3 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + + set.insert(0); + + for i in 0..64 { + if i != 0 && i != 3 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + + set.insert(63); + + for i in 0..64 { + if i != 0 && i != 3 && i != 63 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + } + + #[test] + fn test_remove() { + let mut set = BitSet::new(); + + set.insert(3); + set.insert(63); + set.remove(3); + + for i in 0..64 { + if i != 63 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + } + + #[test] + fn test_min_max() { + let mut set = BitSet::new(); + assert_eq!(set.min(), None); + assert_eq!(set.max(), None); + + set.insert(5); + assert_eq!(set.min(), Some(5)); + assert_eq!(set.max(), Some(5)); + + set.insert(3); + assert_eq!(set.min(), Some(3)); + assert_eq!(set.max(), Some(5)); + } +}