rt: sketch out new wakeup knob

tokio/src/runtime/builder.rs

@@ -146,6 +146,10 @@ pub struct Builder {
     /// Whether or not to enable eager hand-off for the I/O and time drivers (in
     /// `tokio_unstable`).
     enable_eager_driver_handoff: bool,
+
+    /// Whether or not to wake parked worker threads on pushes to a worker's
+    /// LIFO slot.
+    wake_on_lifo_push: bool,
 }
 
 cfg_unstable! {
@@ -339,8 +343,9 @@ impl Builder {
 
             timer_flavor: TimerFlavor::Traditional,
 
-            // Eager driver handoff is disabled by default.
+            // Eager worker wakeup behavior is disabled by default.
             enable_eager_driver_handoff: false,
+            wake_on_lifo_push: false,
         }
     }
 
@@ -450,11 +455,20 @@ impl Builder {
     /// [unstable]: crate#unstable-features
     #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
     #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "rt-multi-thread"))))]
+    #[deprecated(since = "1.53.0", note = "use `enable_eager_worker_wakeup()` instead")]
     pub fn enable_eager_driver_handoff(&mut self) -> &mut Self {
         self.enable_eager_driver_handoff = true;
         self
     }
 
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+    #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "rt-multi-thread"))))]
+    pub fn enable_eager_worker_wakeup(&mut self) -> &mut Self {
+        self.enable_eager_driver_handoff = true;
+        self.wake_on_lifo_push = true;
+        self
+    }
+
     /// Sets the number of worker threads the `Runtime` will use.
     ///
     /// This can be any number above 0 though it is advised to keep this value
@@ -1708,6 +1722,9 @@ impl Builder {
                 enable_eager_driver_handoff: false,
                 seed_generator: seed_generator_1,
                 metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
+                // This setting never makes sense for the current thread
+                // runtime, as it has no notion of "waking a parked worker".
+                wake_on_lifo_push: false,
             },
             local_tid,
             self.name.clone(),
@@ -1889,6 +1906,7 @@ cfg_rt_multi_thread! {
                     unhandled_panic: self.unhandled_panic.clone(),
                     disable_lifo_slot: self.disable_lifo_slot,
                     enable_eager_driver_handoff: self.enable_eager_driver_handoff,
+                    wake_on_lifo_push: self.wake_on_lifo_push,
                     seed_generator: seed_generator_1,
                     metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
                 },

tokio/src/runtime/config.rs

@@ -58,4 +58,10 @@ pub(crate) struct Config {
     /// from polling the I/O driver to polling its own tasks (requires
     /// `tokio_unstable`).
     pub(crate) enable_eager_driver_handoff: bool,
+
+    /// If `true`, a parked worker is woken whenever a task is pushed to a
+    /// worker's LIFO slot, to ensure that the LIFO task is always
+    /// stealable.`Otherwise, pushing a task to the LIFO slot does not wake a
+    /// parked worker.
+    pub(crate) wake_on_lifo_push: bool,
 }

tokio/src/runtime/scheduler/multi_thread/worker.rs

@@ -1349,9 +1349,12 @@ impl Handle {
         // task must always be pushed to the back of the queue, enabling other
         // tasks to be executed. If **not** a yield, then there is more
         // flexibility and the task may go to the front of the queue.
-        if is_yield || !core.lifo_enabled {
+        let should_notify = if is_yield || !core.lifo_enabled {
             core.run_queue
                 .push_back_or_overflow(task, self, &mut core.stats);
+            // Always notify a worker, as we have pushed to the end of the
+            // queue.
+            true
         } else {
             // Push to the LIFO slot
             if let Some(prev) = core.run_queue.push_lifo(task) {
@@ -1359,13 +1362,18 @@ impl Handle {
                 // to be pushed to the back of the run queue.
                 core.run_queue
                     .push_back_or_overflow(prev, self, &mut core.stats);
+                // Again, we have pushed the previous LIFO task to the end of
+                // the queue, so we should always notify a parked worker.
+                true
+            } else {
+                self.shared.config.wake_on_lifo_push
             }
         };
 
         // Only notify if not currently parked. If `park` is `None`, then the
         // scheduling is from a resource driver. As notifications often come in
         // batches, the notification is delayed until the park is complete.
-        if core.park.is_some() {
+        if should_notify && core.park.is_some() {
             self.notify_parked_local();
         }
     }