#![allow(clippy::inline_always)] pub use crate::{ sll::{IntrusiveSortedLinkedList, Node as IntrusiveNode}, tq::{TaskNotReady, TimerQueue, WakerNotReady}, }; pub use bare_metal::CriticalSection; use core::{ cell::Cell, sync::atomic::{AtomicBool, Ordering}, }; pub use cortex_m::{ asm::nop, asm::wfi, interrupt, peripheral::{scb::SystemHandler, DWT, NVIC, SCB, SYST}, Peripherals, }; pub mod executor { use core::{ future::Future, mem, pin::Pin, task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; static WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new(waker_clone, waker_wake, waker_wake, waker_drop); unsafe fn waker_clone(p: *const ()) -> RawWaker { RawWaker::new(p, &WAKER_VTABLE) } unsafe fn waker_wake(p: *const ()) { // The only thing we need from a waker is the function to call to pend the async // dispatcher. let f: fn() = mem::transmute(p); f(); } unsafe fn waker_drop(_: *const ()) { // nop } //============ // AsyncTaskExecutor pub struct AsyncTaskExecutor { task: Option, } impl AsyncTaskExecutor { pub const fn new() -> Self { Self { task: None } } pub fn is_running(&self) -> bool { self.task.is_some() } pub fn spawn(&mut self, future: F) { self.task = Some(future); } pub fn poll(&mut self, wake: fn()) -> bool { if let Some(future) = &mut self.task { unsafe { let waker = Waker::from_raw(RawWaker::new(wake as *const (), &WAKER_VTABLE)); let mut cx = Context::from_waker(&waker); let future = Pin::new_unchecked(future); match future.poll(&mut cx) { Poll::Ready(_) => { self.task = None; true // Only true if we finished now } Poll::Pending => false, } } } else { false } } } } /// Mask is used to store interrupt masks on systems without a BASEPRI register (M0, M0+, M23). /// It needs to be large enough to cover all the relevant interrupts in use. /// For M0/M0+ there are only 32 interrupts so we only need one u32 value. /// For M23 there can be as many as 480 interrupts. /// Rather than providing space for all possible interrupts, we just detect the highest interrupt in /// use at compile time and allocate enough u32 chunks to cover them. #[derive(Copy, Clone)] pub struct Mask([u32; M]); impl core::ops::BitOrAssign for Mask { fn bitor_assign(&mut self, rhs: Self) { for i in 0..M { self.0[i] |= rhs.0[i]; } } } #[cfg(not(have_basepri))] impl Mask { /// Set a bit inside a Mask. const fn set_bit(mut self, bit: u32) -> Self { let block = bit / 32; if block as usize >= M { panic!("Generating masks for thumbv6/thumbv8m.base failed! Are you compiling for thumbv6 on an thumbv7 MCU or using an unsupported thumbv8m.base MCU?"); } let offset = bit - (block * 32); self.0[block as usize] |= 1 << offset; self } } #[cfg(have_basepri)] use cortex_m::register::basepri; #[cfg(have_basepri)] #[inline(always)] pub fn run(priority: u8, f: F) where F: FnOnce(), { if priority == 1 { // If the priority of this interrupt is `1` then BASEPRI can only be `0` f(); unsafe { basepri::write(0) } } else { let initial = basepri::read(); f(); unsafe { basepri::write(initial) } } } #[cfg(not(have_basepri))] #[inline(always)] pub fn run(_priority: u8, f: F) where F: FnOnce(), { f(); } pub struct Barrier { inner: AtomicBool, } impl Barrier { pub const fn new() -> Self { Barrier { inner: AtomicBool::new(false), } } pub fn release(&self) { self.inner.store(true, Ordering::Release); } pub fn wait(&self) { while !self.inner.load(Ordering::Acquire) { core::hint::spin_loop() } } } // Newtype over `Cell` that forbids mutation through a shared reference pub struct Priority { inner: Cell, } impl Priority { /// Create a new Priority /// /// # Safety /// /// Will overwrite the current Priority #[inline(always)] pub const unsafe fn new(value: u8) -> Self { Priority { inner: Cell::new(value), } } /// Change the current priority to `value` // These two methods are used by `lock` (see below) but can't be used from the RTIC application #[inline(always)] fn set(&self, value: u8) { self.inner.set(value); } /// Get the current priority #[inline(always)] fn get(&self) -> u8 { self.inner.get() } } /// Const helper to check architecture pub const fn have_basepri() -> bool { #[cfg(have_basepri)] { true } #[cfg(not(have_basepri))] { false } } #[inline(always)] pub fn assert_send() where T: Send, { } #[inline(always)] pub fn assert_sync() where T: Sync, { } /// Lock implementation using BASEPRI and global Critical Section (CS) /// /// # Safety /// /// The system ceiling is raised from current to ceiling /// by either /// - raising the BASEPRI to the ceiling value, or /// - disable all interrupts in case we want to /// mask interrupts with maximum priority /// /// Dereferencing a raw pointer inside CS /// /// The priority.set/priority.get can safely be outside the CS /// as being a context local cell (not affected by preemptions). /// It is merely used in order to omit masking in case current /// priority is current priority >= ceiling. /// /// Lock Efficiency: /// Experiments validate (sub)-zero cost for CS implementation /// (Sub)-zero as: /// - Either zero OH (lock optimized out), or /// - Amounting to an optimal assembly implementation /// - The BASEPRI value is folded to a constant at compile time /// - CS entry, single assembly instruction to write BASEPRI /// - CS exit, single assembly instruction to write BASEPRI /// - priority.set/get optimized out (their effect not) /// - On par or better than any handwritten implementation of SRP /// /// Limitations: /// The current implementation reads/writes BASEPRI once /// even in some edge cases where this may be omitted. /// Total OH of per task is max 2 clock cycles, negligible in practice /// but can in theory be fixed. /// #[cfg(have_basepri)] #[inline(always)] pub unsafe fn lock( ptr: *mut T, priority: &Priority, ceiling: u8, nvic_prio_bits: u8, _mask: &[Mask; 3], f: impl FnOnce(&mut T) -> R, ) -> R { let current = priority.get(); if current < ceiling { if ceiling == (1 << nvic_prio_bits) { priority.set(u8::max_value()); let r = interrupt::free(|_| f(&mut *ptr)); priority.set(current); r } else { priority.set(ceiling); basepri::write(logical2hw(ceiling, nvic_prio_bits)); let r = f(&mut *ptr); basepri::write(logical2hw(current, nvic_prio_bits)); priority.set(current); r } } else { f(&mut *ptr) } } /// Lock implementation using interrupt masking /// /// # Safety /// /// The system ceiling is raised from current to ceiling /// by computing a 32 bit `mask` (1 bit per interrupt) /// 1: ceiling >= priority > current /// 0: else /// /// On CS entry, `clear_enable_mask(mask)` disables interrupts /// On CS exit, `set_enable_mask(mask)` re-enables interrupts /// /// The priority.set/priority.get can safely be outside the CS /// as being a context local cell (not affected by preemptions). /// It is merely used in order to omit masking in case /// current priority >= ceiling. /// /// Dereferencing a raw pointer is done safely inside the CS /// /// Lock Efficiency: /// Early experiments validate (sub)-zero cost for CS implementation /// (Sub)-zero as: /// - Either zero OH (lock optimized out), or /// - Amounting to an optimal assembly implementation /// - if ceiling == (1 << nvic_prio_bits) /// - we execute the closure in a global critical section (interrupt free) /// - CS entry cost, single write to core register /// - CS exit cost, single write to core register /// else /// - The `mask` value is folded to a constant at compile time /// - CS entry, single write of the 32 bit `mask` to the `icer` register /// - CS exit, single write of the 32 bit `mask` to the `iser` register /// - priority.set/get optimized out (their effect not) /// - On par or better than any hand written implementation of SRP /// /// Limitations: /// Current implementation does not allow for tasks with shared resources /// to be bound to exception handlers, as these cannot be masked in HW. /// /// Possible solutions: /// - Mask exceptions by global critical sections (interrupt::free) /// - Temporary lower exception priority /// /// These possible solutions are set goals for future work #[cfg(not(have_basepri))] #[inline(always)] pub unsafe fn lock( ptr: *mut T, priority: &Priority, ceiling: u8, _nvic_prio_bits: u8, masks: &[Mask; 3], f: impl FnOnce(&mut T) -> R, ) -> R { let current = priority.get(); if current < ceiling { if ceiling >= 4 { // safe to manipulate outside critical section priority.set(ceiling); // execute closure under protection of raised system ceiling let r = interrupt::free(|_| f(&mut *ptr)); // safe to manipulate outside critical section priority.set(current); r } else { // safe to manipulate outside critical section priority.set(ceiling); let mask = compute_mask(current, ceiling, masks); clear_enable_mask(mask); // execute closure under protection of raised system ceiling let r = f(&mut *ptr); set_enable_mask(mask); // safe to manipulate outside critical section priority.set(current); r } } else { // execute closure without raising system ceiling f(&mut *ptr) } } #[cfg(not(have_basepri))] #[inline(always)] fn compute_mask(from_prio: u8, to_prio: u8, masks: &[Mask; 3]) -> Mask { let mut res = Mask([0; M]); masks[from_prio as usize..to_prio as usize] .iter() .for_each(|m| res |= *m); res } // enables interrupts #[cfg(not(have_basepri))] #[inline(always)] unsafe fn set_enable_mask(mask: Mask) { for i in 0..M { // This check should involve compile time constants and be optimized out. if mask.0[i] != 0 { (*NVIC::PTR).iser[i].write(mask.0[i]); } } } // disables interrupts #[cfg(not(have_basepri))] #[inline(always)] unsafe fn clear_enable_mask(mask: Mask) { for i in 0..M { // This check should involve compile time constants and be optimized out. if mask.0[i] != 0 { (*NVIC::PTR).icer[i].write(mask.0[i]); } } } #[inline] #[must_use] pub fn logical2hw(logical: u8, nvic_prio_bits: u8) -> u8 { ((1 << nvic_prio_bits) - logical) << (8 - nvic_prio_bits) } #[cfg(have_basepri)] pub const fn create_mask(_: [u32; N]) -> Mask { Mask([0; M]) } #[cfg(not(have_basepri))] pub const fn create_mask(list_of_shifts: [u32; N]) -> Mask { let mut mask = Mask([0; M]); let mut i = 0; while i < N { let shift = list_of_shifts[i]; i += 1; mask = mask.set_bit(shift); } mask } #[cfg(have_basepri)] pub const fn compute_mask_chunks(_: [u32; L]) -> usize { 0 } /// Compute the number of u32 chunks needed to store the Mask value. /// On M0, M0+ this should always end up being 1. /// On M23 we will pick a number that allows us to store the highest index used by the code. /// This means the amount of overhead will vary based on the actually interrupts used by the code. #[cfg(not(have_basepri))] pub const fn compute_mask_chunks(ids: [u32; L]) -> usize { let mut max: usize = 0; let mut i = 0; while i < L { let id = ids[i] as usize; i += 1; if id > max { max = id; } } (max + 32) / 32 } #[cfg(have_basepri)] pub const fn no_basepri_panic() { // For non-v6 all is fine } #[cfg(not(have_basepri))] pub const fn no_basepri_panic() { panic!("Exceptions with shared resources are not allowed when compiling for thumbv6 or thumbv8m.base. Use local resources or `#[lock_free]` shared resources"); }