time: handle wrapped top-level timer-wheel slots (#8334)

This commit is contained in:
Kiryl Mialeshka
2026-09-07 09:43:41 +08:00
committed by GitHub
parent a316820fa4
commit dde3f86922
5 changed files with 634 additions and 6 deletions
+124 -3
View File
@@ -111,8 +111,15 @@ impl Level {
return None;
}
// Get the slot for now using Maths
let now_slot = (now / slot_range(self.level)) as usize;
// Add the +1 offset for the `now_slot` to ignore the slot that `now` fits in,
// since it's the farthest timer that could appear from `now`.
// This is mostly relevant for the top level because it acts as a
// pseudo-ring buffer: timers that would logically go past the top level are
// fudged into it by `level_for` and the `MAX_DURATION` cap, so the slot holding
// `now` can be occupied by an entry that is a whole rotation away.
// For the lower levels `level_for` always places an entry in a slot other
// than the one holding `now`, so `now_slot` is always empty there.
let now_slot = ((now / slot_range(self.level)) % LEVEL_MULT as u64) as usize + 1;
let occupied = self.occupied.rotate_right(now_slot as u32);
let zeros = occupied.trailing_zeros() as usize;
let slot = (zeros + now_slot) % LEVEL_MULT;
@@ -131,7 +138,12 @@ impl Level {
pub(crate) unsafe fn remove_entry(&mut self, item: NonNull<TimerShared>) {
let slot = slot_for(unsafe { item.as_ref().registered_when() }, self.level);
unsafe { self.slot[slot].remove(item) };
unsafe {
assert!(
self.slot[slot].remove(item).is_some(),
"Attempt to remove item not present in the timing wheel"
)
};
if self.slot[slot].is_empty() {
// The bit is currently set
debug_assert!(self.occupied & occupied_bit(slot) != 0);
@@ -190,4 +202,113 @@ mod test {
}
}
}
fn level_with(level: usize, occupied: u64) -> Level {
let mut level = Level::new(level);
level.occupied = occupied;
level
}
#[test]
fn next_occupied_slot_on_an_empty_level() {
assert_eq!(Level::new(0).next_occupied_slot(0), None);
assert_eq!(Level::new(5).next_occupied_slot(1 << 36), None);
}
#[test]
fn next_occupied_slot_of_a_single_slot() {
// slot 10 of level 0, i.e. tick 10 of every 64-tick window
let level = level_with(0, 1 << 10);
assert_eq!(level.next_occupied_slot(0), Some(10));
assert_eq!(level.next_occupied_slot(9), Some(10));
// `now` inside the slot itself, and past it
assert_eq!(level.next_occupied_slot(10), Some(10));
assert_eq!(level.next_occupied_slot(11), Some(10));
// `now` past the window: the slot is taken modulo 64
assert_eq!(level.next_occupied_slot(64 + 3), Some(10));
}
#[test]
fn next_occupied_slot_picks_the_nearest_slot_forward() {
let level = level_with(0, (1 << 10) | (1 << 40));
assert_eq!(level.next_occupied_slot(0), Some(10));
assert_eq!(level.next_occupied_slot(20), Some(40));
// nothing left ahead in this window, so the scan wraps to slot 10
assert_eq!(level.next_occupied_slot(41), Some(10));
}
#[test]
fn next_occupied_slot_skips_the_slot_holding_now() {
// The occurrence of slot 0 in this rotation has already started, so it
// can only be processed a full `level_range` later. Slot 1 is still
// ahead of `now` in this rotation and therefore expires first.
let level = level_with(5, 0b11);
assert_eq!(level.next_occupied_slot(0), Some(1));
assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(1));
// The same holds for any later slot, not just the adjacent one.
let level = level_with(5, 1 | (1 << 40));
assert_eq!(level.next_occupied_slot(0), Some(40));
}
#[test]
fn next_occupied_slot_of_the_slot_holding_now_when_it_is_the_only_one() {
// Nothing is ahead of `now`, so slot 0 is the earliest expiration even
// though it is reached only in the next rotation.
let level = level_with(5, 1);
assert_eq!(level.next_occupied_slot(0), Some(0));
assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(0));
}
#[test]
fn next_occupied_slot_of_the_last_slot() {
let level = level_with(5, 1 << 63);
assert_eq!(level.next_occupied_slot(62 << 30), Some(63));
assert_eq!(level.next_occupied_slot(63 << 30), Some(63));
}
#[test]
fn next_expiration_reports_the_start_of_the_slot() {
// slot 3 of level 1: slots are 64 ticks wide, so it starts at tick 192
let expiration = level_with(1, 1 << 3).next_expiration(100).unwrap();
assert_eq!(expiration.level, 1);
assert_eq!(expiration.slot, 3);
assert_eq!(expiration.deadline, 192);
}
#[test]
fn next_expiration_below_the_top_level() {
let level = level_with(4, 1 << 1);
assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 24);
assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 24);
}
#[test]
fn next_expiration_at_the_top_level() {
let level = level_with(5, 1 << 1);
assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 30);
assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 30);
}
#[test]
fn next_expiration_wraps_a_slot_at_or_behind_now() {
// Slot 0 of the top level starts at tick 0, so its next occurrence is a
// full rotation of the level away.
let level = level_with(5, 1 << 0);
assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 36);
assert_eq!(
level.next_expiration((1 << 30) + 10).unwrap().deadline,
1 << 36
);
}
}
+176
View File
@@ -290,6 +290,8 @@ fn level_for(elapsed: u64, when: u64) -> usize {
#[cfg(all(test, not(loom)))]
mod test {
use std::pin::Pin;
use super::*;
#[test]
@@ -327,4 +329,178 @@ mod test {
}
}
}
#[must_use]
fn insert_entry(wheel: &mut Wheel, when: u64) -> Pin<Box<TimerShared>> {
let entry = Box::pin(TimerShared::new());
unsafe { entry.set_expiration(when) };
unsafe { wheel.insert(entry.as_ref().handle()).unwrap() };
entry
}
#[test]
fn test_next_expiration_to_level_4() {
let wheel = &mut Wheel::new();
// that should occupy slot 1 of the level 4 of the wheel
let when = (1 << 24) + 10;
let _entry = insert_entry(wheel, when);
let expiration = wheel.next_expiration_time();
// next expiration should be calculated as the start of the level 4 range
assert_eq!(expiration, Some(1 << 24));
// set the elapsed to the start of the previous expiration
wheel.poll(1 << 24);
let expiration = wheel.next_expiration_time();
// that should be equal of the LEVEL_WHEN which is 10 ms after previous expiration
assert_eq!(expiration, Some(when));
wheel.poll(when);
assert!(wheel.next_expiration().is_none());
}
#[test]
fn test_next_expiration_to_level_5() {
let wheel = &mut Wheel::new();
// that will occupy slot 1 of the level 5 of the wheel
let when = (1 << 30) + 10;
let _entry = insert_entry(wheel, when);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(1 << 30));
// set the elapsed to the start of the previous expiration
wheel.poll(1 << 30);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some((1 << 30) + 10));
wheel.poll(when);
assert!(wheel.next_expiration().is_none());
}
#[test]
fn test_next_expiration_after_level_5() {
let wheel = &mut Wheel::new();
// that will occupy slot 0 of the level 5 of the wheel
let when = (1 << 36) + 5;
let _entry = insert_entry(wheel, when);
let expiration = wheel.next_expiration_time();
// that should come after the wheel
assert_eq!(expiration, Some(1 << 36));
// set the elapsed to the start of the previous expiration
wheel.poll(1 << 36);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(when));
wheel.poll(when);
assert!(wheel.next_expiration_time().is_none());
}
#[test]
fn test_next_expiration_after_level_5_twice() {
let wheel = &mut Wheel::new();
// that will occupy slot 0 of the level 5 of the wheel
let when = (1 << 37) + 5;
let _entry = insert_entry(wheel, when);
let expiration = wheel.next_expiration_time();
// that should come after the wheel
assert_eq!(expiration, Some(1 << 36));
// set the elapsed to the start of the previous expiration
wheel.poll(1 << 36);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(1 << 37));
wheel.poll(1 << 37);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(when));
wheel.poll(when);
assert!(wheel.next_expiration_time().is_none());
}
#[test]
fn test_next_expiration_to_level_5_and_after_level_5() {
let wheel = &mut Wheel::new();
// that will occupy slot 0 of the level 5 of the wheel
let when = (1 << 36) + 1;
let _entry = insert_entry(wheel, when);
// that will occupy slot 1 of the level 5 of the wheel
let when = (1 << 30) + 10;
let _entry = insert_entry(wheel, when);
let expiration = wheel.next_expiration_time();
// this should point to the expiration of the slot 1 entry
// and not the slot 0 that is higher than 2^36
assert_eq!(expiration, Some(1 << 30));
// set the elapsed to the start of the previous expiration
wheel.poll(1 << 30);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some((1 << 30) + 10));
wheel.poll(when);
let expiration = wheel.next_expiration_time();
// The next expiration show refer the next loop of top level
assert_eq!(expiration, Some(1 << 36));
}
#[test]
fn test_next_expiration_at_slot_5_of_the_top_level() {
let wheel = &mut Wheel::new();
// move the wheel into slot 5 of the top level
wheel.poll(5 << 30);
// that will occupy slot 5 of the level 5 of the wheel, the same slot the
// elapsed time sits in, so it is reachable only one rotation of the
// level later
let when = (1 << 36) + (5 << 30) + 1;
let _entry = insert_entry(wheel, when);
// that will occupy slot 20 of the level 5 of the wheel, still ahead in
// the current rotation
let when = (20 << 30) + 10;
let _entry = insert_entry(wheel, when);
let expiration = wheel.next_expiration_time();
// this should point to the start of the slot 20 and not to the slot 5
// that comes around only after the whole level rotates
assert_eq!(expiration, Some(20 << 30));
}
}
+124 -3
View File
@@ -112,8 +112,15 @@ impl Level {
return None;
}
// Get the slot for now using Maths
let now_slot = (now / slot_range(self.level)) as usize;
// Add the +1 offset for the `now_slot` to ignore the slot that `now` fits in,
// since it's the farthest timer that could appear from `now`.
// This is mostly relevant for the top level because it acts as a
// pseudo-ring buffer: timers that would logically go past the top level are
// fudged into it by `level_for` and the `MAX_DURATION` cap, so the slot holding
// `now` can be occupied by an entry that is a whole rotation away.
// For the lower levels `level_for` always places an entry in a slot other
// than the one holding `now`, so `now_slot` is always empty there.
let now_slot = ((now / slot_range(self.level)) % LEVEL_MULT as u64) as usize + 1;
let occupied = self.occupied.rotate_right(now_slot as u32);
let zeros = occupied.trailing_zeros() as usize;
let slot = (zeros + now_slot) % LEVEL_MULT;
@@ -134,7 +141,12 @@ impl Level {
pub(crate) unsafe fn remove_entry(&mut self, hdl: EntryHandle) {
let slot = slot_for(hdl.deadline(), self.level);
unsafe { self.slot[slot].remove(NonNull::from(&hdl)) };
unsafe {
assert!(
self.slot[slot].remove(NonNull::from(&hdl)).is_some(),
"Attempt to remove item not present in the timing wheel"
)
};
if self.slot[slot].is_empty() {
// The bit is currently set
debug_assert!(self.occupied & occupied_bit(slot) != 0);
@@ -193,4 +205,113 @@ mod test {
}
}
}
fn level_with(level: usize, occupied: u64) -> Level {
let mut level = Level::new(level);
level.occupied = occupied;
level
}
#[test]
fn next_occupied_slot_on_an_empty_level() {
assert_eq!(Level::new(0).next_occupied_slot(0), None);
assert_eq!(Level::new(5).next_occupied_slot(1 << 36), None);
}
#[test]
fn next_occupied_slot_of_a_single_slot() {
// slot 10 of level 0, i.e. tick 10 of every 64-tick window
let level = level_with(0, 1 << 10);
assert_eq!(level.next_occupied_slot(0), Some(10));
assert_eq!(level.next_occupied_slot(9), Some(10));
// `now` inside the slot itself, and past it
assert_eq!(level.next_occupied_slot(10), Some(10));
assert_eq!(level.next_occupied_slot(11), Some(10));
// `now` past the window: the slot is taken modulo 64
assert_eq!(level.next_occupied_slot(64 + 3), Some(10));
}
#[test]
fn next_occupied_slot_picks_the_nearest_slot_forward() {
let level = level_with(0, (1 << 10) | (1 << 40));
assert_eq!(level.next_occupied_slot(0), Some(10));
assert_eq!(level.next_occupied_slot(20), Some(40));
// nothing left ahead in this window, so the scan wraps to slot 10
assert_eq!(level.next_occupied_slot(41), Some(10));
}
#[test]
fn next_occupied_slot_skips_the_slot_holding_now() {
// The occurrence of slot 0 in this rotation has already started, so it
// can only be processed a full `level_range` later. Slot 1 is still
// ahead of `now` in this rotation and therefore expires first.
let level = level_with(5, 0b11);
assert_eq!(level.next_occupied_slot(0), Some(1));
assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(1));
// The same holds for any later slot, not just the adjacent one.
let level = level_with(5, 1 | (1 << 40));
assert_eq!(level.next_occupied_slot(0), Some(40));
}
#[test]
fn next_occupied_slot_of_the_slot_holding_now_when_it_is_the_only_one() {
// Nothing is ahead of `now`, so slot 0 is the earliest expiration even
// though it is reached only in the next rotation.
let level = level_with(5, 1);
assert_eq!(level.next_occupied_slot(0), Some(0));
assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(0));
}
#[test]
fn next_occupied_slot_of_the_last_slot() {
let level = level_with(5, 1 << 63);
assert_eq!(level.next_occupied_slot(62 << 30), Some(63));
assert_eq!(level.next_occupied_slot(63 << 30), Some(63));
}
#[test]
fn next_expiration_reports_the_start_of_the_slot() {
// slot 3 of level 1: slots are 64 ticks wide, so it starts at tick 192
let expiration = level_with(1, 1 << 3).next_expiration(100).unwrap();
assert_eq!(expiration.level, 1);
assert_eq!(expiration.slot, 3);
assert_eq!(expiration.deadline, 192);
}
#[test]
fn next_expiration_below_the_top_level() {
let level = level_with(4, 1 << 1);
assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 24);
assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 24);
}
#[test]
fn next_expiration_at_the_top_level() {
let level = level_with(5, 1 << 1);
assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 30);
assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 30);
}
#[test]
fn next_expiration_wraps_a_slot_at_or_behind_now() {
// Slot 0 of the top level starts at tick 0, so its next occurrence is a
// full rotation of the level away.
let level = level_with(5, 1 << 0);
assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 36);
assert_eq!(
level.next_expiration((1 << 30) + 10).unwrap().deadline,
1 << 36
);
}
}
+177
View File
@@ -234,6 +234,7 @@ fn level_for(elapsed: u64, when: u64) -> usize {
#[cfg(all(test, not(loom)))]
mod test {
use super::super::cancellation_queue;
use super::*;
#[test]
@@ -271,4 +272,180 @@ mod test {
}
}
}
#[must_use]
fn insert_entry(wheel: &mut Wheel, when: u64) -> EntryHandle {
let (cancel_tx, _cancel_rx) = cancellation_queue::new();
let hdl = EntryHandle::new(when);
unsafe { wheel.insert(hdl.clone(), cancel_tx) };
hdl
}
fn poll(wheel: &mut Wheel, now: u64) {
let mut wq = WakeQueue::new();
wheel.take_expired(now, &mut wq);
}
#[test]
fn test_next_expiration_to_level_4() {
let mut wheel = Wheel::new();
// that should occupy slot 1 of the level 4 of the wheel
let when = (1 << 24) + 10;
let _entry = insert_entry(&mut wheel, when);
let expiration = wheel.next_expiration_time();
// next expiration should be calculated as the start of the level 4 range
assert_eq!(expiration, Some(1 << 24));
// set the elapsed to the start of the previous expiration
poll(&mut wheel, 1 << 24);
let expiration = wheel.next_expiration_time();
// that should be equal of the LEVEL_WHEN which is 10 ms after previous expiration
assert_eq!(expiration, Some(when));
poll(&mut wheel, when);
assert!(wheel.next_expiration_time().is_none());
}
#[test]
fn test_next_expiration_to_level_5() {
let mut wheel = Wheel::new();
// that will occupy slot 1 of the level 5 of the wheel
let when = (1 << 30) + 10;
let _entry = insert_entry(&mut wheel, when);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(1 << 30));
// set the elapsed to the start of the previous expiration
poll(&mut wheel, 1 << 30);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some((1 << 30) + 10));
poll(&mut wheel, when);
assert!(wheel.next_expiration_time().is_none());
}
#[test]
fn test_next_expiration_after_level_5() {
let mut wheel = Wheel::new();
// that will occupy slot 0 of the level 5 of the wheel
let when = (1 << 36) + 5;
let _entry = insert_entry(&mut wheel, when);
let expiration = wheel.next_expiration_time();
// that should come after the wheel
assert_eq!(expiration, Some(1 << 36));
// set the elapsed to the start of the previous expiration
poll(&mut wheel, 1 << 36);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(when));
poll(&mut wheel, when);
assert!(wheel.next_expiration_time().is_none());
}
#[test]
fn test_next_expiration_after_level_5_twice() {
let mut wheel = Wheel::new();
// that will occupy slot 0 of the level 5 of the wheel
let when = (1 << 37) + 5;
let _entry = insert_entry(&mut wheel, when);
let expiration = wheel.next_expiration_time();
// that should come after the wheel
assert_eq!(expiration, Some(1 << 36));
// set the elapsed to the start of the previous expiration
poll(&mut wheel, 1 << 36);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(1 << 37));
poll(&mut wheel, 1 << 37);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some(when));
poll(&mut wheel, when);
assert!(wheel.next_expiration_time().is_none());
}
#[test]
fn test_next_expiration_to_level_5_and_after_level_5() {
let mut wheel = Wheel::new();
// that will occupy slot 0 of the level 5 of the wheel
let when = (1 << 36) + 1;
let _entry = insert_entry(&mut wheel, when);
// that will occupy slot 1 of the level 5 of the wheel
let when = (1 << 30) + 10;
let _entry = insert_entry(&mut wheel, when);
let expiration = wheel.next_expiration_time();
// this should point to the expiration of the slot 1 entry
// and not the slot 0 that is higher than 2^36
assert_eq!(expiration, Some(1 << 30));
// set the elapsed to the start of the previous expiration
poll(&mut wheel, 1 << 30);
let expiration = wheel.next_expiration_time();
assert_eq!(expiration, Some((1 << 30) + 10));
poll(&mut wheel, when);
let expiration = wheel.next_expiration_time();
// The next expiration show refer the next loop of top level
assert_eq!(expiration, Some(1 << 36));
}
#[test]
fn test_next_expiration_at_slot_5_of_the_top_level() {
let mut wheel = Wheel::new();
// move the wheel into slot 5 of the top level
poll(&mut wheel, 5 << 30);
// that will occupy slot 5 of the level 5 of the wheel, the same slot the
// elapsed time sits in, so it is reachable only one rotation of the
// level later
let when = (1 << 36) + (5 << 30) + 1;
let _entry = insert_entry(&mut wheel, when);
// that will occupy slot 20 of the level 5 of the wheel, still ahead in
// the current rotation
let when = (20 << 30) + 10;
let _entry = insert_entry(&mut wheel, when);
let expiration = wheel.next_expiration_time();
// this should point to the start of the slot 20 and not to the slot 5
// that comes around only after the whole level rotates
assert_eq!(expiration, Some(20 << 30));
}
}
+33
View File
@@ -297,6 +297,39 @@ async fn no_out_of_bounds_close_to_max() {
time::sleep(Duration::MAX - Duration::from_millis(1)).await;
}
#[tokio::test(start_paused = true)]
async fn long_wait_sleep_does_not_break_other_timers() {
tokio::spawn(time::sleep(ms(10 << 36)));
time::advance(ms((1 << 30) - 1)).await;
let start = Instant::now();
time::sleep(ms(10)).await;
assert_elapsed!(start, ms(10));
}
#[tokio::test(start_paused = true)]
async fn long_wait_sleep_does_not_corrupt_wheel() {
use futures::poll;
tokio::spawn(time::sleep(ms(10 << 36)));
time::advance(ms((1 << 30) - 1)).await;
let mut first = Box::pin(time::sleep(ms(10)));
assert_pending!(poll!(first.as_mut()));
let mut second = Box::pin(time::sleep(ms(20)));
assert_pending!(poll!(second.as_mut()));
time::advance(ms(30)).await;
drop(first);
time::advance(ms(1 << 31)).await;
drop(second);
}
fn ms(n: u64) -> Duration {
Duration::from_millis(n)
}