diff --git a/tokio/src/runtime/scheduler/multi_thread/queue.rs b/tokio/src/runtime/scheduler/multi_thread/queue.rs index bfa33552b..373772357 100644 --- a/tokio/src/runtime/scheduler/multi_thread/queue.rs +++ b/tokio/src/runtime/scheduler/multi_thread/queue.rs @@ -27,6 +27,9 @@ cfg_not_has_atomic_u64! { /// Producer handle. May only be used from a single thread. pub(crate) struct Local { + // If this is `false`, we DEFINITELY don't have a task in the LIFO slot, + // allowing us to avoid an atomic swap to try and pop from it. + might_have_lifo: bool, inner: Arc>, } @@ -100,6 +103,7 @@ pub(crate) fn local() -> (Steal, Local) { }); let local = Local { + might_have_lifo: false, inner: inner.clone(), }; @@ -112,7 +116,7 @@ impl Local { /// Returns the number of entries in the queue pub(crate) fn len(&self) -> usize { let (_, head) = unpack(self.inner.head.load(Acquire)); - let lifo = self.inner.lifo.is_some() as usize; + let lifo = (self.might_have_lifo && self.inner.lifo.is_some()) as usize; // safety: this is the **only** thread that updates this cell. let tail = unsafe { self.inner.tail.unsync_load() }; len(head, tail) + lifo @@ -409,13 +413,20 @@ impl Local { /// Pushes a task to the LIFO slot, returning the task previously in the /// LIFO slot (if there was one). - pub(crate) fn push_lifo(&self, task: task::Notified) -> Option> { + pub(crate) fn push_lifo(&mut self, task: task::Notified) -> Option> { + self.might_have_lifo = true; self.inner.lifo.swap(Some(task)) } /// Pops the task currently held in the LIFO slot, if there is one; /// otherwise, returns `None`. - pub(crate) fn pop_lifo(&self) -> Option> { + pub(crate) fn pop_lifo(&mut self) -> Option> { + if !self.might_have_lifo { + return None; + } + + self.might_have_lifo = false; + // LIFO-suction! self.inner.lifo.take() }