rt(threaded): basic self-tuning of injection queue

Proof of concept showing how we could implement some basic self-tuning
to check the interval queue (and other maintenance tasks potentially) at
variable interval rates.

.cargo/config

@@ -1,2 +0,0 @@
-# [build]
-# rustflags = ["--cfg", "tokio_unstable"]

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ env:
   RUST_BACKTRACE: 1
   # Change to specific Rust release to pin
   rust_stable: stable
-  rust_nightly: nightly-2022-11-03
+  rust_nightly: nightly-2023-05-18
   rust_clippy: 1.65.0
   # When updating this, also update:
   # - README.md
@@ -59,6 +59,8 @@ jobs:
       - wasm32-unknown-unknown
       - wasm32-wasi
       - check-external-types
+      - check-fuzzing
+      - check-unstable-mt-counters
     steps:
       - run: exit 0
 
@@ -186,10 +188,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os:
-          - windows-latest
-          - ubuntu-latest
-          - macos-latest
+        include:
+          - os: windows-latest
+          - os: ubuntu-latest
+          - os: macos-latest
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
@@ -206,6 +208,56 @@ jobs:
           # in order to run doctests for unstable features, we must also pass
           # the unstable cfg to RustDoc
           RUSTDOCFLAGS: --cfg tokio_unstable
+    
+  test-unstable-taskdump:
+    name: test tokio full --unstable --taskdump
+    needs: basics
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      # Run `tokio` with "unstable" and "taskdump" cfg flags.
+      - name: test tokio full --cfg unstable --cfg taskdump
+        run: cargo test --all-features
+        working-directory: tokio
+        env:
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
+          # in order to run doctests for unstable features, we must also pass
+          # the unstable cfg to RustDoc
+          RUSTDOCFLAGS: --cfg tokio_unstable --cfg tokio_taskdump
+
+  check-unstable-mt-counters:
+    name: check tokio full --internal-mt-counters
+    needs: basics
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      # Run `tokio` with "unstable" and "taskdump" cfg flags.
+      - name: check tokio full --cfg unstable --cfg internal-mt-counters
+        run: cargo test --all-features
+        working-directory: tokio
+        env:
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_internal_mt_counters -Dwarnings
+          # in order to run doctests for unstable features, we must also pass
+          # the unstable cfg to RustDoc
+          RUSTDOCFLAGS: --cfg tokio_unstable --cfg tokio_internal_mt_counters
 
   miri:
     name: miri
@@ -293,9 +345,11 @@ jobs:
       matrix:
         include:
           - target: i686-unknown-linux-gnu
+            rustflags: --cfg tokio_taskdump
           - target: arm-unknown-linux-gnueabihf
           - target: armv7-unknown-linux-gnueabihf
           - target: aarch64-unknown-linux-gnu
+            rustflags: --cfg tokio_taskdump
 
           # Run a platform without AtomicU64 and no const Mutex::new
           - target: arm-unknown-linux-gnueabihf
@@ -341,15 +395,15 @@ jobs:
           target: i686-unknown-linux-gnu
       - run: cargo test -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64
       # https://github.com/tokio-rs/tokio/pull/5356
       # https://github.com/tokio-rs/tokio/issues/5373
       - run: cargo hack build -p tokio --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64 --cfg tokio_no_const_mutex_new
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64 --cfg tokio_no_const_mutex_new
       - run: cargo hack build -p tokio --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64
 
   features:
     name: features
@@ -372,6 +426,11 @@ jobs:
         run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
         env:
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+      # Try with unstable and taskdump feature flags
+      - name: check --feature-powerset --unstable --taskdump
+        run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
+        env:
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
 
   minrust:
     name: minrust
@@ -424,7 +483,7 @@ jobs:
           cargo hack check --all-features --ignore-private
       - name: "check --all-features --unstable -Z minimal-versions"
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
         run: |
           # Remove dev-dependencies from Cargo.toml to prevent the next `cargo update`
           # from determining minimal versions based on dev-dependencies.
@@ -481,8 +540,8 @@ jobs:
       - name: "doc --lib --all-features"
         run: cargo doc --lib --no-deps --all-features --document-private-items
         env:
-          RUSTFLAGS: --cfg docsrs --cfg tokio_unstable
-          RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable -Dwarnings
+          RUSTFLAGS: --cfg docsrs --cfg tokio_unstable --cfg tokio_taskdump
+          RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
 
   loom-compile:
     name: build loom tests
@@ -653,3 +712,23 @@ jobs:
           cargo install cargo-check-external-types --locked --version 0.1.6
           cargo check-external-types --all-features --config external-types.toml
         working-directory: tokio
+
+  check-fuzzing:
+    name: check-fuzzing
+    needs: basics
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.rust_nightly }}
+      - uses: Swatinem/rust-cache@v2
+      - name: Install cargo-fuzz
+        run: cargo install cargo-fuzz
+      - name: Check /tokio/
+        run: cargo fuzz check --all-features
+        working-directory: tokio
+      - name: Check /tokio-stream/
+        run: cargo fuzz check --all-features
+        working-directory: tokio-stream

.github/workflows/loom.yml

@@ -24,13 +24,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        scope:
-          - --skip loom_pool
-          - loom_pool::group_a
-          - loom_pool::group_b
-          - loom_pool::group_c
-          - loom_pool::group_d
-          - time::driver
+        include:
+          - scope: --skip loom_pool
+            max_preemptions: 2
+          - scope: loom_pool::group_a
+            max_preemptions: 1
+          - scope: loom_pool::group_b
+            max_preemptions: 2
+          - scope: loom_pool::group_c
+            max_preemptions: 1
+          - scope: loom_pool::group_d
+            max_preemptions: 1
+          - scope: time::driver
+            max_preemptions: 2
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
@@ -42,7 +48,7 @@ jobs:
         run: cargo test --lib --release --features full -- --nocapture $SCOPE
         working-directory: tokio
         env:
-          RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings
-          LOOM_MAX_PREEMPTIONS: 2
+          RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings -C debug-assertions
+          LOOM_MAX_PREEMPTIONS: ${{ matrix.max_preemptions }}
           LOOM_MAX_BRANCHES: 10000
           SCOPE: ${{ matrix.scope }}

.gitignore

@@ -2,3 +2,4 @@ target
 Cargo.lock
 
 .cargo/config.toml
+.cargo/config

benches/Cargo.toml

@@ -40,6 +40,11 @@ name = "sync_watch"
 path = "sync_watch.rs"
 harness = false
 
+[[bench]]
+name = "rt_current_thread"
+path = "rt_current_thread.rs"
+harness = false
+
 [[bench]]
 name = "rt_multi_threaded"
 path = "rt_multi_threaded.rs"

benches/rt_current_thread.rs

@@ -0,0 +1,83 @@
+//! Benchmark implementation details of the threaded scheduler. These benches are
+//! intended to be used as a form of regression testing and not as a general
+//! purpose benchmark demonstrating real-world performance.
+
+use tokio::runtime::{self, Runtime};
+
+use bencher::{benchmark_group, benchmark_main, Bencher};
+
+const NUM_SPAWN: usize = 1_000;
+
+fn spawn_many_local(b: &mut Bencher) {
+    let rt = rt();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    b.iter(|| {
+        rt.block_on(async {
+            for _ in 0..NUM_SPAWN {
+                handles.push(tokio::spawn(async move {}));
+            }
+
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn spawn_many_remote_idle(b: &mut Bencher) {
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn spawn_many_remote_busy(b: &mut Bencher) {
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    rt.spawn(async {
+        fn iter() {
+            tokio::spawn(async { iter() });
+        }
+
+        iter()
+    });
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn rt() -> Runtime {
+    runtime::Builder::new_current_thread().build().unwrap()
+}
+
+benchmark_group!(
+    scheduler,
+    spawn_many_local,
+    spawn_many_remote_idle,
+    spawn_many_remote_busy
+);
+
+benchmark_main!(scheduler);

benches/rt_multi_threaded.rs

@@ -10,9 +10,10 @@ use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering::Relaxed;
 use std::sync::{mpsc, Arc};
 
-fn spawn_many(b: &mut Bencher) {
-    const NUM_SPAWN: usize = 10_000;
+const NUM_WORKERS: usize = 4;
+const NUM_SPAWN: usize = 10_000;
 
+fn spawn_many_local(b: &mut Bencher) {
     let rt = rt();
 
     let (tx, rx) = mpsc::sync_channel(1000);
@@ -38,6 +39,52 @@ fn spawn_many(b: &mut Bencher) {
     });
 }
 
+fn spawn_many_remote_idle(b: &mut Bencher) {
+    let rt = rt();
+
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn spawn_many_remote_busy(b: &mut Bencher) {
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    // Spawn some tasks to keep the runtimes busy
+    for _ in 0..(2 * NUM_WORKERS) {
+        rt.spawn(async {
+            loop {
+                tokio::task::yield_now().await;
+                std::thread::sleep(std::time::Duration::from_micros(10));
+            }
+        });
+    }
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
 fn yield_many(b: &mut Bencher) {
     const NUM_YIELD: usize = 1_000;
     const TASKS: usize = 200;
@@ -140,12 +187,20 @@ fn chained_spawn(b: &mut Bencher) {
 
 fn rt() -> Runtime {
     runtime::Builder::new_multi_thread()
-        .worker_threads(4)
+        .worker_threads(NUM_WORKERS)
         .enable_all()
         .build()
         .unwrap()
 }
 
-benchmark_group!(scheduler, spawn_many, ping_pong, yield_many, chained_spawn,);
+benchmark_group!(
+    scheduler,
+    spawn_many_local,
+    spawn_many_remote_idle,
+    spawn_many_remote_busy,
+    ping_pong,
+    yield_many,
+    chained_spawn,
+);
 
 benchmark_main!(scheduler);

examples/Cargo.toml

@@ -90,3 +90,7 @@ path = "named-pipe-ready.rs"
 [[example]]
 name = "named-pipe-multi-client"
 path = "named-pipe-multi-client.rs"
+
+[[example]]
+name = "dump"
+path = "dump.rs"

examples/dump.rs

@@ -0,0 +1,50 @@
+//! This example demonstrates tokio's experimental taskdumping functionality.
+
+#[cfg(all(
+    tokio_unstable,
+    tokio_taskdump,
+    target_os = "linux",
+    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+))]
+#[tokio::main(flavor = "current_thread")]
+async fn main() {
+    use std::hint::black_box;
+
+    #[inline(never)]
+    async fn a() {
+        black_box(b()).await
+    }
+
+    #[inline(never)]
+    async fn b() {
+        black_box(c()).await
+    }
+
+    #[inline(never)]
+    async fn c() {
+        black_box(tokio::task::yield_now()).await
+    }
+
+    tokio::spawn(a());
+    tokio::spawn(b());
+    tokio::spawn(c());
+
+    let handle = tokio::runtime::Handle::current();
+    let dump = handle.dump();
+
+    for (i, task) in dump.tasks().iter().enumerate() {
+        let trace = task.trace();
+        println!("task {i} trace:");
+        println!("{trace}");
+    }
+}
+
+#[cfg(not(all(
+    tokio_unstable,
+    tokio_taskdump,
+    target_os = "linux",
+    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+)))]
+fn main() {
+    println!("task dumps are not available")
+}

tokio-macros/src/lib.rs

@@ -209,7 +209,7 @@ pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
 
 /// Marks async function to be executed by selected runtime. This macro helps set up a `Runtime`
 /// without requiring the user to use [Runtime](../tokio/runtime/struct.Runtime.html) or
-/// [Builder](../tokio/runtime/struct.builder.html) directly.
+/// [Builder](../tokio/runtime/struct.Builder.html) directly.
 ///
 /// ## Function arguments:
 ///

tokio-stream/CHANGELOG.md

@@ -1,3 +1,22 @@
+# 0.1.14 (April 26th, 2023)
+
+This bugfix release bumps the minimum version of Tokio to 1.15, which is
+necessary for `timeout_repeating` to compile. ([#5657])
+
+[#5657]: https://github.com/tokio-rs/tokio/pull/5657
+
+# 0.1.13 (April 25th, 2023)
+
+This release bumps the MSRV of tokio-stream to 1.56.
+
+- stream: add "full" feature flag ([#5639])
+- stream: add `StreamExt::timeout_repeating` ([#5577])
+- stream: add `StreamNotifyClose` ([#4851])
+
+[#4851]: https://github.com/tokio-rs/tokio/pull/4851
+[#5577]: https://github.com/tokio-rs/tokio/pull/5577
+[#5639]: https://github.com/tokio-rs/tokio/pull/5639
+
 # 0.1.12 (January 20, 2023)
 
 - time: remove `Unpin` bound on `Throttle` methods ([#5105])

tokio-stream/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-stream"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-stream-0.1.x" git tag.
-version = "0.1.12"
+version = "0.1.14"
 edition = "2021"
 rust-version = "1.56"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -38,7 +38,7 @@ signal = ["tokio/signal"]
 [dependencies]
 futures-core = { version = "0.3.0" }
 pin-project-lite = "0.2.0"
-tokio = { version = "1.8.0", path = "../tokio", features = ["sync"] }
+tokio = { version = "1.15.0", path = "../tokio", features = ["sync"] }
 tokio-util = { version = "0.7.0", path = "../tokio-util", optional = true }
 
 [dev-dependencies]

tokio-stream/fuzz/fuzz_targets/fuzz_stream_map.rs

@@ -3,17 +3,8 @@
 use libfuzzer_sys::fuzz_target;
 use std::pin::Pin;
 
-use tokio_stream::{self as stream, pending, Stream, StreamExt, StreamMap};
-use tokio_test::{assert_ok, assert_pending, assert_ready, task};
-
-macro_rules! assert_ready_some {
-    ($($t:tt)*) => {
-        match assert_ready!($($t)*) {
-            Some(v) => v,
-            None => panic!("expected `Some`, got `None`"),
-        }
-    };
-}
+use tokio_stream::{self as stream, Stream, StreamMap};
+use tokio_test::{assert_pending, assert_ready, task};
 
 macro_rules! assert_ready_none {
     ($($t:tt)*) => {

tokio-stream/src/lib.rs

@@ -63,12 +63,12 @@
 //! [`tokio-util`] provides the [`StreamReader`] and [`ReaderStream`]
 //! types when the io feature is enabled.
 //!
-//! [`tokio-util`]: https://docs.rs/tokio-util/0.4/tokio_util/codec/index.html
-//! [`tokio::io`]: https://docs.rs/tokio/1.0/tokio/io/index.html
-//! [`AsyncRead`]: https://docs.rs/tokio/1.0/tokio/io/trait.AsyncRead.html
-//! [`AsyncWrite`]: https://docs.rs/tokio/1.0/tokio/io/trait.AsyncWrite.html
-//! [`ReaderStream`]: https://docs.rs/tokio-util/0.4/tokio_util/io/struct.ReaderStream.html
-//! [`StreamReader`]: https://docs.rs/tokio-util/0.4/tokio_util/io/struct.StreamReader.html
+//! [`tokio-util`]: https://docs.rs/tokio-util/latest/tokio_util/codec/index.html
+//! [`tokio::io`]: https://docs.rs/tokio/latest/tokio/io/index.html
+//! [`AsyncRead`]: https://docs.rs/tokio/latest/tokio/io/trait.AsyncRead.html
+//! [`AsyncWrite`]: https://docs.rs/tokio/latest/tokio/io/trait.AsyncWrite.html
+//! [`ReaderStream`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.ReaderStream.html
+//! [`StreamReader`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.StreamReader.html
 
 #[macro_use]
 mod macros;

tokio-util/CHANGELOG.md

@@ -1,3 +1,30 @@
+# 0.7.8 (April 25th, 2023)
+
+This release bumps the MSRV of tokio-util to 1.56.
+
+### Added
+
+- time: add `DelayQueue::peek` ([#5569])
+
+### Changed
+
+This release contains one performance improvement:
+
+- sync: try to lock the parent first in `CancellationToken` ([#5561])
+
+### Fixed
+
+- time: fix panic in `DelayQueue` ([#5630])
+
+### Documented
+
+- sync: improve `CancellationToken` doc on child tokens ([#5632])
+
+[#5561]: https://github.com/tokio-rs/tokio/pull/5561
+[#5569]: https://github.com/tokio-rs/tokio/pull/5569
+[#5630]: https://github.com/tokio-rs/tokio/pull/5630
+[#5632]: https://github.com/tokio-rs/tokio/pull/5632
+
 # 0.7.7 (February 12, 2023)
 
 This release reverts the removal of the `Encoder` bound on the `FramedParts`

tokio-util/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-util"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-util-0.7.x" git tag.
-version = "0.7.7"
+version = "0.7.8"
 edition = "2021"
 rust-version = "1.56"
 authors = ["Tokio Contributors <[email protected]>"]

tokio-util/src/sync/mpsc.rs

@@ -44,7 +44,7 @@ enum State<T> {
 pub struct PollSender<T> {
     sender: Option<Sender<T>>,
     state: State<T>,
-    acquire: ReusableBoxFuture<'static, Result<OwnedPermit<T>, PollSendError<T>>>,
+    acquire: PollSenderFuture<T>,
 }
 
 // Creates a future for acquiring a permit from the underlying channel.  This is used to ensure
@@ -64,13 +64,56 @@ async fn make_acquire_future<T>(
     }
 }
 
-impl<T: Send + 'static> PollSender<T> {
+type InnerFuture<'a, T> = ReusableBoxFuture<'a, Result<OwnedPermit<T>, PollSendError<T>>>;
+
+#[derive(Debug)]
+// TODO: This should be replace with a type_alias_impl_trait to eliminate `'static` and all the transmutes
+struct PollSenderFuture<T>(InnerFuture<'static, T>);
+
+impl<T> PollSenderFuture<T> {
+    /// Create with an empty inner future with no `Send` bound.
+    fn empty() -> Self {
+        // We don't use `make_acquire_future` here because our relaxed bounds on `T` are not
+        // compatible with the transitive bounds required by `Sender<T>`.
+        Self(ReusableBoxFuture::new(async { unreachable!() }))
+    }
+}
+
+impl<T: Send> PollSenderFuture<T> {
+    /// Create with an empty inner future.
+    fn new() -> Self {
+        let v = InnerFuture::new(make_acquire_future(None));
+        // This is safe because `make_acquire_future(None)` is actually `'static`
+        Self(unsafe { mem::transmute::<InnerFuture<'_, T>, InnerFuture<'static, T>>(v) })
+    }
+
+    /// Poll the inner future.
+    fn poll(&mut self, cx: &mut Context<'_>) -> Poll<Result<OwnedPermit<T>, PollSendError<T>>> {
+        self.0.poll(cx)
+    }
+
+    /// Replace the inner future.
+    fn set(&mut self, sender: Option<Sender<T>>) {
+        let inner: *mut InnerFuture<'static, T> = &mut self.0;
+        let inner: *mut InnerFuture<'_, T> = inner.cast();
+        // SAFETY: The `make_acquire_future(sender)` future must not exist after the type `T`
+        // becomes invalid, and this casts away the type-level lifetime check for that. However, the
+        // inner future is never moved out of this `PollSenderFuture<T>`, so the future will not
+        // live longer than the `PollSenderFuture<T>` lives. A `PollSenderFuture<T>` is guaranteed
+        // to not exist after the type `T` becomes invalid, because it is annotated with a `T`, so
+        // this is ok.
+        let inner = unsafe { &mut *inner };
+        inner.set(make_acquire_future(sender));
+    }
+}
+
+impl<T: Send> PollSender<T> {
     /// Creates a new `PollSender`.
     pub fn new(sender: Sender<T>) -> Self {
         Self {
             sender: Some(sender.clone()),
             state: State::Idle(sender),
-            acquire: ReusableBoxFuture::new(make_acquire_future(None)),
+            acquire: PollSenderFuture::new(),
         }
     }
 
@@ -97,7 +140,7 @@ impl<T: Send + 'static> PollSender<T> {
                 State::Idle(sender) => {
                     // Start trying to acquire a permit to reserve a slot for our send, and
                     // immediately loop back around to poll it the first time.
-                    self.acquire.set(make_acquire_future(Some(sender)));
+                    self.acquire.set(Some(sender));
                     (None, State::Acquiring)
                 }
                 State::Acquiring => match self.acquire.poll(cx) {
@@ -194,7 +237,7 @@ impl<T: Send + 'static> PollSender<T> {
         match self.state {
             State::Idle(_) => self.state = State::Closed,
             State::Acquiring => {
-                self.acquire.set(make_acquire_future(None));
+                self.acquire.set(None);
                 self.state = State::Closed;
             }
             _ => {}
@@ -215,7 +258,7 @@ impl<T: Send + 'static> PollSender<T> {
             // We're currently trying to reserve a slot to send into.
             State::Acquiring => {
                 // Replacing the future drops the in-flight one.
-                self.acquire.set(make_acquire_future(None));
+                self.acquire.set(None);
 
                 // If we haven't closed yet, we have to clone our stored sender since we have no way
                 // to get it back from the acquire future we just dropped.
@@ -255,9 +298,7 @@ impl<T> Clone for PollSender<T> {
         Self {
             sender,
             state,
-            // We don't use `make_acquire_future` here because our relaxed bounds on `T` are not
-            // compatible with the transitive bounds required by `Sender<T>`.
-            acquire: ReusableBoxFuture::new(async { unreachable!() }),
+            acquire: PollSenderFuture::empty(),
         }
     }
 }

tokio-util/tests/mpsc.rs

@@ -27,6 +27,29 @@ async fn simple() {
     send.send_item(42).unwrap();
 }
 
+#[tokio::test]
+async fn simple_ref() {
+    let v = vec![1, 2, 3i32];
+
+    let (send, mut recv) = channel(3);
+    let mut send = PollSender::new(send);
+
+    for vi in v.iter() {
+        let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+        assert_ready_ok!(reserve.poll());
+        send.send_item(vi).unwrap();
+    }
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+
+    assert_eq!(*recv.recv().await.unwrap(), 1);
+    assert!(reserve.is_woken());
+    assert_ready_ok!(reserve.poll());
+    drop(recv);
+    send.send_item(&42).unwrap();
+}
+
 #[tokio::test]
 async fn repeated_poll_reserve() {
     let (send, mut recv) = channel::<i32>(1);

tokio/Cargo.toml

@@ -115,6 +115,11 @@ socket2 = { version = "0.4.9", optional = true, features = [ "all" ] }
 [target.'cfg(tokio_unstable)'.dependencies]
 tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true } # Not in full
 
+# Currently unstable. The API exposed by these features may be broken at any time.
+# Requires `--cfg tokio_unstable` to enable.
+[target.'cfg(tokio_taskdump)'.dependencies]
+backtrace = { version = "0.3.58" }
+
 [target.'cfg(unix)'.dependencies]
 libc = { version = "0.2.42", optional = true }
 signal-hook-registry = { version = "1.1.1", optional = true }

tokio/src/fs/file.rs

@@ -498,6 +498,7 @@ impl AsyncRead for File {
         cx: &mut Context<'_>,
         dst: &mut ReadBuf<'_>,
     ) -> Poll<io::Result<()>> {
+        ready!(crate::trace::trace_leaf(cx));
         let me = self.get_mut();
         let inner = me.inner.get_mut();
 
@@ -594,6 +595,7 @@ impl AsyncSeek for File {
     }
 
     fn poll_complete(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
+        ready!(crate::trace::trace_leaf(cx));
         let inner = self.inner.get_mut();
 
         loop {
@@ -629,6 +631,7 @@ impl AsyncWrite for File {
         cx: &mut Context<'_>,
         src: &[u8],
     ) -> Poll<io::Result<usize>> {
+        ready!(crate::trace::trace_leaf(cx));
         let me = self.get_mut();
         let inner = me.inner.get_mut();
 
@@ -695,11 +698,13 @@ impl AsyncWrite for File {
     }
 
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
+        ready!(crate::trace::trace_leaf(cx));
         let inner = self.inner.get_mut();
         inner.poll_flush(cx)
     }
 
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
+        ready!(crate::trace::trace_leaf(cx));
         self.poll_flush(cx)
     }
 }
@@ -774,8 +779,18 @@ impl Inner {
     async fn complete_inflight(&mut self) {
         use crate::future::poll_fn;
 
-        if let Err(e) = poll_fn(|cx| Pin::new(&mut *self).poll_flush(cx)).await {
-            self.last_write_err = Some(e.kind());
+        poll_fn(|cx| self.poll_complete_inflight(cx)).await
+    }
+
+    fn poll_complete_inflight(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+        ready!(crate::trace::trace_leaf(cx));
+        match self.poll_flush(cx) {
+            Poll::Ready(Err(e)) => {
+                self.last_write_err = Some(e.kind());
+                Poll::Ready(())
+            }
+            Poll::Ready(Ok(())) => Poll::Ready(()),
+            Poll::Pending => Poll::Pending,
         }
     }
 

tokio/src/fs/read_dir.rs

@@ -33,11 +33,11 @@ const CHUNK_SIZE: usize = 32;
 pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
     let path = path.as_ref().to_owned();
     asyncify(|| -> io::Result<ReadDir> {
-        let mut std = std::fs::read_dir(path)?.fuse();
+        let mut std = std::fs::read_dir(path)?;
         let mut buf = VecDeque::with_capacity(CHUNK_SIZE);
-        ReadDir::next_chunk(&mut buf, &mut std);
+        let remain = ReadDir::next_chunk(&mut buf, &mut std);
 
-        Ok(ReadDir(State::Idle(Some((buf, std)))))
+        Ok(ReadDir(State::Idle(Some((buf, std, remain)))))
     })
     .await
 }
@@ -64,12 +64,10 @@ pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
 #[must_use = "streams do nothing unless polled"]
 pub struct ReadDir(State);
 
-type StdReadDir = std::iter::Fuse<std::fs::ReadDir>;
-
 #[derive(Debug)]
 enum State {
-    Idle(Option<(VecDeque<io::Result<DirEntry>>, StdReadDir)>),
-    Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, StdReadDir)>),
+    Idle(Option<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir, bool)>),
+    Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir, bool)>),
 }
 
 impl ReadDir {
@@ -105,38 +103,35 @@ impl ReadDir {
         loop {
             match self.0 {
                 State::Idle(ref mut data) => {
-                    let (buf, _) = data.as_mut().unwrap();
+                    let (buf, _, ref remain) = data.as_mut().unwrap();
 
                     if let Some(ent) = buf.pop_front() {
                         return Poll::Ready(ent.map(Some));
-                    };
+                    } else if !remain {
+                        return Poll::Ready(Ok(None));
+                    }
 
-                    let (mut buf, mut std) = data.take().unwrap();
+                    let (mut buf, mut std, _) = data.take().unwrap();
 
                     self.0 = State::Pending(spawn_blocking(move || {
-                        ReadDir::next_chunk(&mut buf, &mut std);
-                        (buf, std)
+                        let remain = ReadDir::next_chunk(&mut buf, &mut std);
+                        (buf, std, remain)
                     }));
                 }
                 State::Pending(ref mut rx) => {
-                    let (mut buf, std) = ready!(Pin::new(rx).poll(cx))?;
-
-                    let ret = match buf.pop_front() {
-                        Some(Ok(x)) => Ok(Some(x)),
-                        Some(Err(e)) => Err(e),
-                        None => Ok(None),
-                    };
-
-                    self.0 = State::Idle(Some((buf, std)));
-
-                    return Poll::Ready(ret);
+                    self.0 = State::Idle(Some(ready!(Pin::new(rx).poll(cx))?));
                 }
             }
         }
     }
 
-    fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut StdReadDir) {
-        for ret in std.by_ref().take(CHUNK_SIZE) {
+    fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut std::fs::ReadDir) -> bool {
+        for _ in 0..CHUNK_SIZE {
+            let ret = match std.next() {
+                Some(ret) => ret,
+                None => return false,
+            };
+
             let success = ret.is_ok();
 
             buf.push_back(ret.map(|std| DirEntry {
@@ -154,6 +149,8 @@ impl ReadDir {
                 break;
             }
         }
+
+        true
     }
 }
 

tokio/src/io/util/async_buf_read_ext.rs

@@ -146,7 +146,7 @@ cfg_io_util! {
         ///    [`next_line`] method.
         ///  * Use [`tokio_util::codec::LinesCodec`][LinesCodec].
         ///
-        /// [LinesCodec]: https://docs.rs/tokio-util/0.6/tokio_util/codec/struct.LinesCodec.html
+        /// [LinesCodec]: https://docs.rs/tokio-util/latest/tokio_util/codec/struct.LinesCodec.html
         /// [`read_until`]: Self::read_until
         /// [`lines`]: Self::lines
         /// [`next_line`]: crate::io::Lines::next_line

tokio/src/lib.rs

@@ -487,6 +487,21 @@ compile_error!("Tokio's build script has incorrectly detected wasm.");
 ))]
 compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm.");
 
+#[cfg(all(not(tokio_unstable), tokio_taskdump))]
+compile_error!("The `tokio_taskdump` feature requires `--cfg tokio_unstable`.");
+
+#[cfg(all(
+    tokio_taskdump,
+    not(all(
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))
+))]
+compile_error!(
+    "The `tokio_taskdump` feature is only currently supported on \
+linux, on `aarch64`, `x86` and `x86_64`."
+);
+
 // Includes re-exports used by macros.
 //
 // This module is not intended to be part of the public API. In general, any
@@ -552,6 +567,20 @@ cfg_time! {
     pub mod time;
 }
 
+mod trace {
+    cfg_taskdump! {
+        pub(crate) use crate::runtime::task::trace::trace_leaf;
+    }
+
+    cfg_not_taskdump! {
+        #[inline(always)]
+        #[allow(dead_code)]
+        pub(crate) fn trace_leaf(_: &mut std::task::Context<'_>) -> std::task::Poll<()> {
+            std::task::Poll::Ready(())
+        }
+    }
+}
+
 mod util;
 
 /// Due to the `Stream` trait's inclusion in `std` landing later than Tokio's 1.0

tokio/src/macros/cfg.rs

@@ -373,6 +373,44 @@ macro_rules! cfg_not_rt_multi_thread {
     }
 }
 
+macro_rules! cfg_taskdump {
+    ($($item:item)*) => {
+        $(
+            #[cfg(all(
+                tokio_unstable,
+                tokio_taskdump,
+                feature = "rt",
+                target_os = "linux",
+                any(
+                    target_arch = "aarch64",
+                    target_arch = "x86",
+                    target_arch = "x86_64"
+                )
+            ))]
+            $item
+        )*
+    };
+}
+
+macro_rules! cfg_not_taskdump {
+    ($($item:item)*) => {
+        $(
+            #[cfg(not(all(
+                tokio_unstable,
+                tokio_taskdump,
+                feature = "rt",
+                target_os = "linux",
+                any(
+                    target_arch = "aarch64",
+                    target_arch = "x86",
+                    target_arch = "x86_64"
+                )
+            )))]
+            $item
+        )*
+    };
+}
+
 macro_rules! cfg_test_util {
     ($($item:item)*) => {
         $(

tokio/src/macros/loom.rs

@@ -1,11 +1,7 @@
 macro_rules! if_loom {
     ($($t:tt)*) => {{
         #[cfg(loom)]
-        const LOOM: bool = true;
-        #[cfg(not(loom))]
-        const LOOM: bool = false;
-
-        if LOOM {
+        {
             $($t)*
         }
     }}

tokio/src/net/tcp/socket.rs

@@ -404,6 +404,51 @@ impl TcpSocket {
         self.inner.linger()
     }
 
+    /// Sets the value of the `TCP_NODELAY` option on this socket.
+    ///
+    /// If set, this option disables the Nagle algorithm. This means that segments are always
+    /// sent as soon as possible, even if there is only a small amount of data. When not set,
+    /// data is buffered until there is a sufficient amount to send out, thereby avoiding
+    /// the frequent sending of small packets.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use tokio::net::TcpSocket;
+    ///
+    /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
+    /// let socket = TcpSocket::new_v4()?;
+    ///
+    /// println!("{:?}", socket.nodelay()?);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> {
+        self.inner.set_nodelay(nodelay)
+    }
+
+    /// Gets the value of the `TCP_NODELAY` option on this socket.
+    ///
+    /// For more information about this option, see [`set_nodelay`].
+    ///
+    /// [`set_nodelay`]: TcpSocket::set_nodelay
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use tokio::net::TcpSocket;
+    ///
+    /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
+    /// let stream = TcpSocket::new_v4()?;
+    ///
+    /// stream.set_nodelay(true)?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn nodelay(&self) -> io::Result<bool> {
+        self.inner.nodelay()
+    }
+
     /// Gets the value of the `IP_TOS` option for this socket.
     ///
     /// For more information about this option, see [`set_tos`].

tokio/src/net/unix/datagram/socket.rs

@@ -91,6 +91,7 @@ cfg_net_unix! {
     /// # Ok(())
     /// # }
     /// ```
+    #[cfg_attr(docsrs, doc(alias = "uds"))]
     pub struct UnixDatagram {
         io: PollEvented<mio::net::UnixDatagram>,
     }

tokio/src/net/unix/listener.rs

@@ -45,6 +45,7 @@ cfg_net_unix! {
     ///     }
     /// }
     /// ```
+    #[cfg_attr(docsrs, doc(alias = "uds"))]
     pub struct UnixListener {
         io: PollEvented<mio::net::UnixListener>,
     }

tokio/src/net/unix/stream.rs

@@ -34,6 +34,7 @@ cfg_net_unix! {
     ///
     /// [`shutdown()`]: fn@crate::io::AsyncWriteExt::shutdown
     /// [`UnixListener::accept`]: crate::net::UnixListener::accept
+    #[cfg_attr(docsrs, doc(alias = "uds"))]
     pub struct UnixStream {
         io: PollEvented<mio::net::UnixStream>,
     }

tokio/src/process/mod.rs

@@ -400,6 +400,22 @@ impl Command {
         self
     }
 
+    /// Append literal text to the command line without any quoting or escaping.
+    ///
+    /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow
+    /// `CommandLineToArgvW` escaping rules.
+    ///
+    /// **Note**: This is an [unstable API][unstable] but will be stabilised once
+    /// tokio's MSRV is sufficiently new. See [the documentation on
+    /// unstable features][unstable] for details about using unstable features.
+    #[cfg(windows)]
+    #[cfg(tokio_unstable)]
+    #[cfg_attr(docsrs, doc(cfg(all(windows, tokio_unstable))))]
+    pub fn raw_arg<S: AsRef<OsStr>>(&mut self, text_to_append_as_is: S) -> &mut Command {
+        self.std.raw_arg(text_to_append_as_is);
+        self
+    }
+
     /// Inserts or updates an environment variable mapping.
     ///
     /// Note that environment variable names are case-insensitive (but case-preserving) on Windows,

tokio/src/runtime/builder.rs

@@ -1,5 +1,5 @@
 use crate::runtime::handle::Handle;
-use crate::runtime::{blocking, driver, Callback, Runtime};
+use crate::runtime::{blocking, driver, Callback, HistogramBuilder, Runtime};
 use crate::util::rand::{RngSeed, RngSeedGenerator};
 
 use std::fmt;
@@ -95,6 +95,12 @@ pub struct Builder {
     /// Specify a random number generator seed to provide deterministic results
     pub(super) seed_generator: RngSeedGenerator,
 
+    /// When true, enables task poll count histogram instrumentation.
+    pub(super) metrics_poll_count_histogram_enable: bool,
+
+    /// Configures the task poll count histogram
+    pub(super) metrics_poll_count_histogram: HistogramBuilder,
+
     #[cfg(tokio_unstable)]
     pub(super) unhandled_panic: UnhandledPanic,
 }
@@ -268,6 +274,10 @@ impl Builder {
             #[cfg(tokio_unstable)]
             unhandled_panic: UnhandledPanic::Ignore,
 
+            metrics_poll_count_histogram_enable: false,
+
+            metrics_poll_count_histogram: Default::default(),
+
             disable_lifo_slot: false,
         }
     }
@@ -877,6 +887,133 @@ impl Builder {
         }
     }
 
+    cfg_metrics! {
+        /// Enables tracking the distribution of task poll times.
+        ///
+        /// Task poll times are not instrumented by default as doing so requires
+        /// calling [`Instant::now()`] twice per task poll, which could add
+        /// measurable overhead. Use the [`Handle::metrics()`] to access the
+        /// metrics data.
+        ///
+        /// The histogram uses fixed bucket sizes. In other words, the histogram
+        /// buckets are not dynamic based on input values. Use the
+        /// `metrics_poll_count_histogram_` builder methods to configure the
+        /// histogram details.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .build()
+        ///     .unwrap();
+        /// # // Test default values here
+        /// # fn us(n: u64) -> std::time::Duration { std::time::Duration::from_micros(n) }
+        /// # let m = rt.handle().metrics();
+        /// # assert_eq!(m.poll_count_histogram_num_buckets(), 10);
+        /// # assert_eq!(m.poll_count_histogram_bucket_range(0), us(0)..us(100));
+        /// # assert_eq!(m.poll_count_histogram_bucket_range(1), us(100)..us(200));
+        /// ```
+        ///
+        /// [`Handle::metrics()`]: crate::runtime::Handle::metrics
+        /// [`Instant::now()`]: std::time::Instant::now
+        pub fn enable_metrics_poll_count_histogram(&mut self) -> &mut Self {
+            self.metrics_poll_count_histogram_enable = true;
+            self
+        }
+
+        /// Sets the histogram scale for tracking the distribution of task poll
+        /// times.
+        ///
+        /// Tracking the distribution of task poll times can be done using a
+        /// linear or log scale. When using linear scale, each histogram bucket
+        /// will represent the same range of poll times. When using log scale,
+        /// each histogram bucket will cover a range twice as big as the
+        /// previous bucket.
+        ///
+        /// **Default:** linear scale.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::{self, HistogramScale};
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .metrics_poll_count_histogram_scale(HistogramScale::Log)
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn metrics_poll_count_histogram_scale(&mut self, histogram_scale: crate::runtime::HistogramScale) -> &mut Self {
+            self.metrics_poll_count_histogram.scale = histogram_scale;
+            self
+        }
+
+        /// Sets the histogram resolution for tracking the distribution of task
+        /// poll times.
+        ///
+        /// The resolution is the histogram's first bucket's range. When using a
+        /// linear histogram scale, each bucket will cover the same range. When
+        /// using a log scale, each bucket will cover a range twice as big as
+        /// the previous bucket. In the log case, the resolution represents the
+        /// smallest bucket range.
+        ///
+        /// Note that, when using log scale, the resolution is rounded up to the
+        /// nearest power of 2 in nanoseconds.
+        ///
+        /// **Default:** 100 microseconds.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        /// use std::time::Duration;
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .metrics_poll_count_histogram_resolution(Duration::from_micros(100))
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn metrics_poll_count_histogram_resolution(&mut self, resolution: Duration) -> &mut Self {
+            assert!(resolution > Duration::from_secs(0));
+            // Sanity check the argument and also make the cast below safe.
+            assert!(resolution <= Duration::from_secs(1));
+
+            let resolution = resolution.as_nanos() as u64;
+            self.metrics_poll_count_histogram.resolution = resolution;
+            self
+        }
+
+        /// Sets the number of buckets for the histogram tracking the
+        /// distribution of task poll times.
+        ///
+        /// The last bucket tracks all greater values that fall out of other
+        /// ranges. So, configuring the histogram using a linear scale,
+        /// resolution of 50ms, and 10 buckets, the 10th bucket will track task
+        /// polls that take more than 450ms to complete.
+        ///
+        /// **Default:** 10
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .metrics_poll_count_histogram_buckets(15)
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn metrics_poll_count_histogram_buckets(&mut self, buckets: usize) -> &mut Self {
+            self.metrics_poll_count_histogram.num_buckets = buckets;
+            self
+        }
+    }
+
     fn build_current_thread_runtime(&mut self) -> io::Result<Runtime> {
         use crate::runtime::scheduler::{self, CurrentThread};
         use crate::runtime::{runtime::Scheduler, Config};
@@ -909,6 +1046,7 @@ impl Builder {
                 unhandled_panic: self.unhandled_panic.clone(),
                 disable_lifo_slot: self.disable_lifo_slot,
                 seed_generator: seed_generator_1,
+                metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
             },
         );
 
@@ -922,6 +1060,14 @@ impl Builder {
             blocking_pool,
         ))
     }
+
+    fn metrics_poll_count_histogram_builder(&self) -> Option<HistogramBuilder> {
+        if self.metrics_poll_count_histogram_enable {
+            Some(self.metrics_poll_count_histogram.clone())
+        } else {
+            None
+        }
+    }
 }
 
 cfg_io_driver! {
@@ -1050,6 +1196,7 @@ cfg_rt_multi_thread! {
                     unhandled_panic: self.unhandled_panic.clone(),
                     disable_lifo_slot: self.disable_lifo_slot,
                     seed_generator: seed_generator_1,
+                    metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
                 },
             );
 

tokio/src/runtime/config.rs

@@ -28,6 +28,9 @@ pub(crate) struct Config {
     /// deterministic way.
     pub(crate) seed_generator: RngSeedGenerator,
 
+    /// How to build poll time histograms
+    pub(crate) metrics_poll_count_histogram: Option<crate::runtime::HistogramBuilder>,
+
     #[cfg(tokio_unstable)]
     /// How to respond to unhandled task panics.
     pub(crate) unhandled_panic: crate::runtime::UnhandledPanic,

tokio/src/runtime/context.rs

@@ -12,6 +12,10 @@ cfg_rt! {
     use std::cell::RefCell;
     use std::marker::PhantomData;
     use std::time::Duration;
+
+    cfg_taskdump! {
+        use crate::runtime::task::trace;
+    }
 }
 
 struct Context {
@@ -45,6 +49,15 @@ struct Context {
     /// Tracks the amount of "work" a task may still do before yielding back to
     /// the sheduler
     budget: Cell<coop::Budget>,
+
+    #[cfg(all(
+        tokio_unstable,
+        tokio_taskdump,
+        feature = "rt",
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))]
+    trace: trace::Context,
 }
 
 tokio_thread_local! {
@@ -75,6 +88,19 @@ tokio_thread_local! {
             rng: FastRand::new(RngSeed::new()),
 
             budget: Cell::new(coop::Budget::unconstrained()),
+
+            #[cfg(all(
+                tokio_unstable,
+                tokio_taskdump,
+                feature = "rt",
+                target_os = "linux",
+                any(
+                    target_arch = "aarch64",
+                    target_arch = "x86",
+                    target_arch = "x86_64"
+                )
+            ))]
+            trace: trace::Context::new(),
         }
     }
 }
@@ -378,6 +404,14 @@ cfg_rt! {
             matches!(self, EnterRuntime::Entered { .. })
         }
     }
+
+    cfg_taskdump! {
+        /// SAFETY: Callers of this function must ensure that trace frames always
+        /// form a valid linked list.
+        pub(crate) unsafe fn with_trace<R>(f: impl FnOnce(&trace::Context) -> R) -> R {
+            CONTEXT.with(|c| f(&c.trace))
+        }
+    }
 }
 
 // Forces the current "entered" state to be cleared while the closure

tokio/src/runtime/defer.rs

@@ -11,8 +11,14 @@ impl Defer {
         }
     }
 
-    pub(crate) fn defer(&mut self, waker: Waker) {
-        self.deferred.push(waker);
+    pub(crate) fn defer(&mut self, waker: &Waker) {
+        // If the same task adds itself a bunch of times, then only add it once.
+        if let Some(last) = self.deferred.last() {
+            if last.will_wake(waker) {
+                return;
+            }
+        }
+        self.deferred.push(waker.clone());
     }
 
     pub(crate) fn is_empty(&self) -> bool {
@@ -24,4 +30,9 @@ impl Defer {
             waker.wake();
         }
     }
+
+    #[cfg(tokio_taskdump)]
+    pub(crate) fn take_deferred(&mut self) -> Vec<Waker> {
+        std::mem::take(&mut self.deferred)
+    }
 }

tokio/src/runtime/dump.rs

@@ -0,0 +1,66 @@
+//! Snapshots of runtime state.
+
+use std::fmt;
+
+/// A snapshot of a runtime's state.
+#[derive(Debug)]
+pub struct Dump {
+    tasks: Tasks,
+}
+
+/// Snapshots of tasks.
+#[derive(Debug)]
+pub struct Tasks {
+    tasks: Vec<Task>,
+}
+
+/// A snapshot of a task.
+#[derive(Debug)]
+pub struct Task {
+    trace: Trace,
+}
+
+/// An execution trace of a task's last poll.
+#[derive(Debug)]
+pub struct Trace {
+    inner: super::task::trace::Trace,
+}
+
+impl Dump {
+    pub(crate) fn new(tasks: Vec<Task>) -> Self {
+        Self {
+            tasks: Tasks { tasks },
+        }
+    }
+
+    /// Tasks in this snapshot.
+    pub fn tasks(&self) -> &Tasks {
+        &self.tasks
+    }
+}
+
+impl Tasks {
+    /// Iterate over tasks.
+    pub fn iter(&self) -> impl Iterator<Item = &Task> {
+        self.tasks.iter()
+    }
+}
+
+impl Task {
+    pub(crate) fn new(trace: super::task::trace::Trace) -> Self {
+        Self {
+            trace: Trace { inner: trace },
+        }
+    }
+
+    /// A trace of this task's state.
+    pub fn trace(&self) -> &Trace {
+        &self.trace
+    }
+}
+
+impl fmt::Display for Trace {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.inner.fmt(f)
+    }
+}

tokio/src/runtime/handle.rs

@@ -252,6 +252,15 @@ impl Handle {
     /// [`tokio::time`]: crate::time
     #[track_caller]
     pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        ))]
+        let future = super::task::trace::Trace::root(future);
+
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let future =
             crate::util::trace::task(future, "block_on", None, super::task::Id::next().as_u64());
@@ -274,6 +283,14 @@ impl Handle {
         F::Output: Send + 'static,
     {
         let id = crate::runtime::task::Id::next();
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        ))]
+        let future = super::task::trace::Trace::root(future);
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let future = crate::util::trace::task(future, "task", _name, id.as_u64());
         self.inner.spawn(future, id)
@@ -321,6 +338,20 @@ cfg_metrics! {
     }
 }
 
+cfg_taskdump! {
+    impl Handle {
+        /// Capture a snapshot of this runtime's state.
+        pub fn dump(&self) -> crate::runtime::Dump {
+            match &self.inner {
+                scheduler::Handle::CurrentThread(handle) => handle.dump(),
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThread(_) =>
+                    unimplemented!("taskdumps are unsupported on the multi-thread runtime"),
+            }
+        }
+    }
+}
+
 /// Error returned by `try_current` when no Runtime has been started
 #[derive(Debug)]
 pub struct TryCurrentError {

tokio/src/runtime/metrics/batch.rs

@@ -1,7 +1,7 @@
-use crate::runtime::WorkerMetrics;
+use crate::runtime::metrics::{HistogramBatch, WorkerMetrics};
 
 use std::sync::atomic::Ordering::Relaxed;
-use std::time::Instant;
+use std::time::{Duration, Instant};
 
 pub(crate) struct MetricsBatch {
     /// Number of times the worker parked.
@@ -32,11 +32,26 @@ pub(crate) struct MetricsBatch {
 
     /// The total busy duration in nanoseconds.
     busy_duration_total: u64,
+
+    /// Instant at which work last resumed (continued after park).
     last_resume_time: Instant,
+
+    /// If `Some`, tracks poll times in nanoseconds
+    poll_timer: Option<PollTimer>,
+}
+
+struct PollTimer {
+    /// Histogram of poll counts within each band.
+    poll_counts: HistogramBatch,
+
+    /// Instant when the most recent task started polling.
+    poll_started_at: Instant,
 }
 
 impl MetricsBatch {
-    pub(crate) fn new() -> MetricsBatch {
+    pub(crate) fn new(worker_metrics: &WorkerMetrics) -> MetricsBatch {
+        let now = Instant::now();
+
         MetricsBatch {
             park_count: 0,
             noop_count: 0,
@@ -47,7 +62,14 @@ impl MetricsBatch {
             local_schedule_count: 0,
             overflow_count: 0,
             busy_duration_total: 0,
-            last_resume_time: Instant::now(),
+            last_resume_time: now,
+            poll_timer: worker_metrics
+                .poll_count_histogram
+                .as_ref()
+                .map(|worker_poll_counts| PollTimer {
+                    poll_counts: HistogramBatch::from_histogram(worker_poll_counts),
+                    poll_started_at: now,
+                }),
         }
     }
 
@@ -68,6 +90,11 @@ impl MetricsBatch {
             .local_schedule_count
             .store(self.local_schedule_count, Relaxed);
         worker.overflow_count.store(self.overflow_count, Relaxed);
+
+        if let Some(poll_timer) = &self.poll_timer {
+            let dst = worker.poll_count_histogram.as_ref().unwrap();
+            poll_timer.poll_counts.submit(dst);
+        }
     }
 
     /// The worker is about to park.
@@ -81,8 +108,22 @@ impl MetricsBatch {
         }
 
         let busy_duration = self.last_resume_time.elapsed();
-        let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX);
-        self.busy_duration_total += busy_duration;
+        self.busy_duration_total += duration_as_u64(busy_duration);
+    }
+
+    pub(crate) fn start_poll(&mut self) {
+        self.poll_count += 1;
+
+        if let Some(poll_timer) = &mut self.poll_timer {
+            poll_timer.poll_started_at = Instant::now();
+        }
+    }
+
+    pub(crate) fn end_poll(&mut self) {
+        if let Some(poll_timer) = &mut self.poll_timer {
+            let elapsed = duration_as_u64(poll_timer.poll_started_at.elapsed());
+            poll_timer.poll_counts.measure(elapsed, 1);
+        }
     }
 
     pub(crate) fn returned_from_park(&mut self) {
@@ -92,10 +133,6 @@ impl MetricsBatch {
     pub(crate) fn inc_local_schedule_count(&mut self) {
         self.local_schedule_count += 1;
     }
-
-    pub(crate) fn incr_poll_count(&mut self) {
-        self.poll_count += 1;
-    }
 }
 
 cfg_rt_multi_thread! {
@@ -113,3 +150,7 @@ cfg_rt_multi_thread! {
         }
     }
 }
+
+fn duration_as_u64(dur: Duration) -> u64 {
+    u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX)
+}

tokio/src/runtime/metrics/histogram.rs

@@ -0,0 +1,502 @@
+use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed};
+
+use std::cmp;
+use std::ops::Range;
+
+#[derive(Debug)]
+pub(crate) struct Histogram {
+    /// The histogram buckets
+    buckets: Box<[AtomicU64]>,
+
+    /// Bucket scale, linear or log
+    scale: HistogramScale,
+
+    /// Minimum resolution
+    resolution: u64,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct HistogramBuilder {
+    /// Histogram scale
+    pub(crate) scale: HistogramScale,
+
+    /// Must be a power of 2
+    pub(crate) resolution: u64,
+
+    /// Number of buckets
+    pub(crate) num_buckets: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct HistogramBatch {
+    buckets: Box<[u64]>,
+    scale: HistogramScale,
+    resolution: u64,
+}
+
+cfg_unstable! {
+    /// Whether the histogram used to aggregate a metric uses a linear or
+    /// logarithmic scale.
+    #[derive(Debug, Copy, Clone, Eq, PartialEq)]
+    #[non_exhaustive]
+    pub enum HistogramScale {
+        /// Linear bucket scale
+        Linear,
+
+        /// Logarithmic bucket scale
+        Log,
+    }
+}
+
+impl Histogram {
+    pub(crate) fn num_buckets(&self) -> usize {
+        self.buckets.len()
+    }
+
+    pub(crate) fn get(&self, bucket: usize) -> u64 {
+        self.buckets[bucket].load(Relaxed)
+    }
+
+    pub(crate) fn bucket_range(&self, bucket: usize) -> Range<u64> {
+        match self.scale {
+            HistogramScale::Log => Range {
+                start: if bucket == 0 {
+                    0
+                } else {
+                    self.resolution << (bucket - 1)
+                },
+                end: if bucket == self.buckets.len() - 1 {
+                    u64::MAX
+                } else {
+                    self.resolution << bucket
+                },
+            },
+            HistogramScale::Linear => Range {
+                start: self.resolution * bucket as u64,
+                end: if bucket == self.buckets.len() - 1 {
+                    u64::MAX
+                } else {
+                    self.resolution * (bucket as u64 + 1)
+                },
+            },
+        }
+    }
+}
+
+impl HistogramBatch {
+    pub(crate) fn from_histogram(histogram: &Histogram) -> HistogramBatch {
+        let buckets = vec![0; histogram.buckets.len()].into_boxed_slice();
+
+        HistogramBatch {
+            buckets,
+            scale: histogram.scale,
+            resolution: histogram.resolution,
+        }
+    }
+
+    pub(crate) fn measure(&mut self, value: u64, count: u64) {
+        self.buckets[self.value_to_bucket(value)] += count;
+    }
+
+    pub(crate) fn submit(&self, histogram: &Histogram) {
+        debug_assert_eq!(self.scale, histogram.scale);
+        debug_assert_eq!(self.resolution, histogram.resolution);
+        debug_assert_eq!(self.buckets.len(), histogram.buckets.len());
+
+        for i in 0..self.buckets.len() {
+            histogram.buckets[i].store(self.buckets[i], Relaxed);
+        }
+    }
+
+    fn value_to_bucket(&self, value: u64) -> usize {
+        match self.scale {
+            HistogramScale::Linear => {
+                let max = self.buckets.len() - 1;
+                cmp::min(value / self.resolution, max as u64) as usize
+            }
+            HistogramScale::Log => {
+                let max = self.buckets.len() - 1;
+
+                if value < self.resolution {
+                    0
+                } else {
+                    let significant_digits = 64 - value.leading_zeros();
+                    let bucket_digits = 64 - (self.resolution - 1).leading_zeros();
+                    cmp::min(significant_digits as usize - bucket_digits as usize, max)
+                }
+            }
+        }
+    }
+}
+
+impl HistogramBuilder {
+    pub(crate) fn new() -> HistogramBuilder {
+        HistogramBuilder {
+            scale: HistogramScale::Linear,
+            // Resolution is in nanoseconds.
+            resolution: 100_000,
+            num_buckets: 10,
+        }
+    }
+
+    pub(crate) fn build(&self) -> Histogram {
+        let mut resolution = self.resolution;
+
+        assert!(resolution > 0);
+
+        if matches!(self.scale, HistogramScale::Log) {
+            resolution = resolution.next_power_of_two();
+        }
+
+        Histogram {
+            buckets: (0..self.num_buckets)
+                .map(|_| AtomicU64::new(0))
+                .collect::<Vec<_>>()
+                .into_boxed_slice(),
+            resolution,
+            scale: self.scale,
+        }
+    }
+}
+
+impl Default for HistogramBuilder {
+    fn default() -> HistogramBuilder {
+        HistogramBuilder::new()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    macro_rules! assert_bucket_eq {
+        ($h:expr, $bucket:expr, $val:expr) => {{
+            assert_eq!($h.buckets[$bucket], $val);
+        }};
+    }
+
+    #[test]
+    fn log_scale_resolution_1() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Log,
+            resolution: 1,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..1);
+        assert_eq!(h.bucket_range(1), 1..2);
+        assert_eq!(h.bucket_range(2), 2..4);
+        assert_eq!(h.bucket_range(3), 4..8);
+        assert_eq!(h.bucket_range(9), 256..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(1, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(2, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 1);
+
+        b.measure(3, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 2);
+
+        b.measure(4, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 2);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(100, 1);
+        assert_bucket_eq!(b, 7, 1);
+
+        b.measure(128, 1);
+        assert_bucket_eq!(b, 8, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+    }
+
+    #[test]
+    fn log_scale_resolution_2() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Log,
+            resolution: 2,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..2);
+        assert_eq!(h.bucket_range(1), 2..4);
+        assert_eq!(h.bucket_range(2), 4..8);
+        assert_eq!(h.bucket_range(3), 8..16);
+        assert_eq!(h.bucket_range(9), 512..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(1, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(2, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(3, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(4, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 1);
+
+        b.measure(5, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 2);
+
+        b.measure(6, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 3);
+
+        b.measure(7, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 4);
+
+        b.measure(8, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 4);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(100, 1);
+        assert_bucket_eq!(b, 6, 1);
+
+        b.measure(128, 1);
+        assert_bucket_eq!(b, 7, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+
+    #[test]
+    fn linear_scale_resolution_1() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Linear,
+            resolution: 1,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..1);
+        assert_eq!(h.bucket_range(1), 1..2);
+        assert_eq!(h.bucket_range(2), 2..3);
+        assert_eq!(h.bucket_range(3), 3..4);
+        assert_eq!(h.bucket_range(9), 9..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(1, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(2, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 1);
+        assert_bucket_eq!(b, 3, 0);
+
+        b.measure(3, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 1);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(5, 1);
+        assert_bucket_eq!(b, 5, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+
+    #[test]
+    fn linear_scale_resolution_100() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Linear,
+            resolution: 100,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..100);
+        assert_eq!(h.bucket_range(1), 100..200);
+        assert_eq!(h.bucket_range(2), 200..300);
+        assert_eq!(h.bucket_range(3), 300..400);
+        assert_eq!(h.bucket_range(9), 900..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(50, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(100, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(101, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(200, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 1);
+
+        b.measure(299, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 2);
+
+        b.measure(222, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 3);
+
+        b.measure(300, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 3);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(888, 1);
+        assert_bucket_eq!(b, 8, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+
+    #[test]
+    fn inc_by_more_than_one() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Linear,
+            resolution: 100,
+            num_buckets: 10,
+        }
+        .build();
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 3);
+        assert_bucket_eq!(b, 0, 3);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(50, 5);
+        assert_bucket_eq!(b, 0, 8);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(100, 2);
+        assert_bucket_eq!(b, 0, 8);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(101, 19);
+        assert_bucket_eq!(b, 0, 8);
+        assert_bucket_eq!(b, 1, 21);
+        assert_bucket_eq!(b, 2, 0);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+}

tokio/src/runtime/metrics/mock.rs

@@ -6,6 +6,9 @@ pub(crate) struct WorkerMetrics {}
 
 pub(crate) struct MetricsBatch {}
 
+#[derive(Clone, Default)]
+pub(crate) struct HistogramBuilder {}
+
 impl SchedulerMetrics {
     pub(crate) fn new() -> Self {
         Self {}
@@ -20,19 +23,26 @@ impl WorkerMetrics {
         Self {}
     }
 
+    pub(crate) fn from_config(config: &crate::runtime::Config) -> Self {
+        // Prevent the dead-code warning from being triggered
+        let _ = &config.metrics_poll_count_histogram;
+        Self::new()
+    }
+
     pub(crate) fn set_queue_depth(&self, _len: usize) {}
 }
 
 impl MetricsBatch {
-    pub(crate) fn new() -> Self {
+    pub(crate) fn new(_: &WorkerMetrics) -> Self {
         Self {}
     }
 
     pub(crate) fn submit(&mut self, _to: &WorkerMetrics) {}
     pub(crate) fn about_to_park(&mut self) {}
     pub(crate) fn returned_from_park(&mut self) {}
-    pub(crate) fn incr_poll_count(&mut self) {}
     pub(crate) fn inc_local_schedule_count(&mut self) {}
+    pub(crate) fn start_poll(&mut self) {}
+    pub(crate) fn end_poll(&mut self) {}
 }
 
 cfg_rt_multi_thread! {

tokio/src/runtime/metrics/mod.rs

@@ -12,6 +12,11 @@ cfg_metrics! {
     mod batch;
     pub(crate) use batch::MetricsBatch;
 
+    mod histogram;
+    pub(crate) use histogram::{Histogram, HistogramBatch, HistogramBuilder};
+    #[allow(unreachable_pub)] // rust-lang/rust#57411
+    pub use histogram::HistogramScale;
+
     mod runtime;
     #[allow(unreachable_pub)] // rust-lang/rust#57411
     pub use runtime::RuntimeMetrics;
@@ -31,5 +36,5 @@ cfg_metrics! {
 cfg_not_metrics! {
     mod mock;
 
-    pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch};
+    pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch, HistogramBuilder};
 }

tokio/src/runtime/metrics/runtime.rs

@@ -1,5 +1,6 @@
 use crate::runtime::Handle;
 
+use std::ops::Range;
 use std::sync::atomic::Ordering::Relaxed;
 use std::time::Duration;
 
@@ -68,6 +69,25 @@ impl RuntimeMetrics {
         self.handle.inner.num_blocking_threads()
     }
 
+    /// Returns the number of active tasks in the runtime.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Handle;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///    let metrics = Handle::current().metrics();
+    ///
+    ///     let n = metrics.active_tasks_count();
+    ///     println!("Runtime has {} active tasks", n);
+    /// }
+    /// ```
+    pub fn active_tasks_count(&self) -> usize {
+        self.handle.inner.active_tasks_count()
+    }
+
     /// Returns the number of idle threads, which have spawned by the runtime
     /// for `spawn_blocking` calls.
     ///
@@ -559,6 +579,196 @@ impl RuntimeMetrics {
         self.handle.inner.worker_local_queue_depth(worker)
     }
 
+    /// Returns `true` if the runtime is tracking the distribution of task poll
+    /// times.
+    ///
+    /// Task poll times are not instrumented by default as doing so requires
+    /// calling [`Instant::now()`] twice per task poll. The feature is enabled
+    /// by calling [`enable_metrics_poll_count_histogram()`] when building the
+    /// runtime.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let enabled = metrics.poll_count_histogram_enabled();
+    ///
+    ///             println!("Tracking task poll time distribution: {:?}", enabled);
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`enable_metrics_poll_count_histogram()`]: crate::runtime::Builder::enable_metrics_poll_count_histogram
+    /// [`Instant::now()`]: std::time::Instant::now
+    pub fn poll_count_histogram_enabled(&self) -> bool {
+        self.handle
+            .inner
+            .worker_metrics(0)
+            .poll_count_histogram
+            .is_some()
+    }
+
+    /// Returns the number of histogram buckets tracking the distribution of
+    /// task poll times.
+    ///
+    /// This value is configured by calling
+    /// [`metrics_poll_count_histogram_buckets()`] when building the runtime.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let buckets = metrics.poll_count_histogram_num_buckets();
+    ///
+    ///             println!("Histogram buckets: {:?}", buckets);
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`metrics_poll_count_histogram_buckets()`]:
+    ///     crate::runtime::Builder::metrics_poll_count_histogram_buckets
+    pub fn poll_count_histogram_num_buckets(&self) -> usize {
+        self.handle
+            .inner
+            .worker_metrics(0)
+            .poll_count_histogram
+            .as_ref()
+            .map(|histogram| histogram.num_buckets())
+            .unwrap_or_default()
+    }
+
+    /// Returns the range of task poll times tracked by the given bucket.
+    ///
+    /// This value is configured by calling
+    /// [`metrics_poll_count_histogram_resolution()`] when building the runtime.
+    ///
+    /// # Panics
+    ///
+    /// The method panics if `bucket` represents an invalid bucket index, i.e.
+    /// is greater than or equal to `poll_count_histogram_num_buckets()`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let buckets = metrics.poll_count_histogram_num_buckets();
+    ///
+    ///             for i in 0..buckets {
+    ///                 let range = metrics.poll_count_histogram_bucket_range(i);
+    ///                 println!("Histogram bucket {} range: {:?}", i, range);
+    ///             }
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`metrics_poll_count_histogram_resolution()`]:
+    ///     crate::runtime::Builder::metrics_poll_count_histogram_resolution
+    #[track_caller]
+    pub fn poll_count_histogram_bucket_range(&self, bucket: usize) -> Range<Duration> {
+        self.handle
+            .inner
+            .worker_metrics(0)
+            .poll_count_histogram
+            .as_ref()
+            .map(|histogram| {
+                let range = histogram.bucket_range(bucket);
+                std::ops::Range {
+                    start: Duration::from_nanos(range.start),
+                    end: Duration::from_nanos(range.end),
+                }
+            })
+            .unwrap_or_default()
+    }
+
+    /// Returns the number of times the given worker polled tasks with a poll
+    /// duration within the given bucket's range.
+    ///
+    /// Each worker maintains its own histogram and the counts for each bucket
+    /// starts at zero when the runtime is created. Each time the worker polls a
+    /// task, it tracks the duration the task poll time took and increments the
+    /// associated bucket by 1.
+    ///
+    /// Each bucket is a monotonically increasing counter. It is never
+    /// decremented or reset to zero.
+    ///
+    /// # Arguments
+    ///
+    /// `worker` is the index of the worker being queried. The given value must
+    /// be between 0 and `num_workers()`. The index uniquely identifies a single
+    /// worker and will continue to identify the worker throughout the lifetime
+    /// of the runtime instance.
+    ///
+    /// `bucket` is the index of the bucket being queried. The bucket is scoped
+    /// to the worker. The range represented by the bucket can be queried by
+    /// calling [`poll_count_histogram_bucket_range()`]. Each worker maintains
+    /// identical bucket ranges.
+    ///
+    /// # Panics
+    ///
+    /// The method panics when `worker` represents an invalid worker, i.e. is
+    /// greater than or equal to `num_workers()` or if `bucket` represents an
+    /// invalid bucket.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let buckets = metrics.poll_count_histogram_num_buckets();
+    ///
+    ///             for worker in 0..metrics.num_workers() {
+    ///                 for i in 0..buckets {
+    ///                     let count = metrics.poll_count_histogram_bucket_count(worker, i);
+    ///                     println!("Poll count {}", count);
+    ///                 }
+    ///             }
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`poll_count_histogram_bucket_range()`]: crate::runtime::RuntimeMetrics::poll_count_histogram_bucket_range
+    #[track_caller]
+    pub fn poll_count_histogram_bucket_count(&self, worker: usize, bucket: usize) -> u64 {
+        self.handle
+            .inner
+            .worker_metrics(worker)
+            .poll_count_histogram
+            .as_ref()
+            .map(|histogram| histogram.get(bucket))
+            .unwrap_or_default()
+    }
+
     /// Returns the number of tasks currently scheduled in the blocking
     /// thread pool, spawned using `spawn_blocking`.
     ///

tokio/src/runtime/metrics/worker.rs

@@ -1,5 +1,7 @@
 use crate::loom::sync::atomic::Ordering::Relaxed;
 use crate::loom::sync::atomic::{AtomicU64, AtomicUsize};
+use crate::runtime::metrics::Histogram;
+use crate::runtime::Config;
 
 /// Retrieve runtime worker metrics.
 ///
@@ -38,9 +40,21 @@ pub(crate) struct WorkerMetrics {
     /// Number of tasks currently in the local queue. Used only by the
     /// current-thread scheduler.
     pub(crate) queue_depth: AtomicUsize,
+
+    /// If `Some`, tracks the the number of polls by duration range.
+    pub(super) poll_count_histogram: Option<Histogram>,
 }
 
 impl WorkerMetrics {
+    pub(crate) fn from_config(config: &Config) -> WorkerMetrics {
+        let mut worker_metrics = WorkerMetrics::new();
+        worker_metrics.poll_count_histogram = config
+            .metrics_poll_count_histogram
+            .as_ref()
+            .map(|histogram_builder| histogram_builder.build());
+        worker_metrics
+    }
+
     pub(crate) fn new() -> WorkerMetrics {
         WorkerMetrics {
             park_count: AtomicU64::new(0),
@@ -52,6 +66,7 @@ impl WorkerMetrics {
             busy_duration_total: AtomicU64::new(0),
             local_schedule_count: AtomicU64::new(0),
             queue_depth: AtomicUsize::new(0),
+            poll_count_histogram: None,
         }
     }
 

tokio/src/runtime/mod.rs

@@ -233,6 +233,11 @@ cfg_rt! {
     mod defer;
     pub(crate) use defer::Defer;
 
+    cfg_taskdump! {
+        pub mod dump;
+        pub use dump::Dump;
+    }
+
     mod handle;
     pub use handle::{EnterGuard, Handle, TryCurrentError};
 
@@ -244,9 +249,9 @@ cfg_rt! {
 
     cfg_metrics! {
         mod metrics;
-        pub use metrics::RuntimeMetrics;
+        pub use metrics::{RuntimeMetrics, HistogramScale};
 
-        pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
+        pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics, HistogramBuilder};
 
         cfg_net! {
         pub(crate) use metrics::IoDriverMetrics;
@@ -255,7 +260,7 @@ cfg_rt! {
 
     cfg_not_metrics! {
         pub(crate) mod metrics;
-        pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch};
+        pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch, HistogramBuilder};
     }
 
     /// After thread starts / before thread stops

tokio/src/runtime/runtime.rs

@@ -288,6 +288,15 @@ impl Runtime {
     /// [handle]: fn@Handle::block_on
     #[track_caller]
     pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        ))]
+        let future = super::task::trace::Trace::root(future);
+
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let future = crate::util::trace::task(
             future,

tokio/src/runtime/scheduler/current_thread.rs

@@ -1,8 +1,8 @@
 use crate::future::poll_fn;
 use crate::loom::sync::atomic::AtomicBool;
-use crate::loom::sync::{Arc, Mutex};
+use crate::loom::sync::Arc;
 use crate::runtime::driver::{self, Driver};
-use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task};
+use crate::runtime::task::{self, Inject, JoinHandle, OwnedTasks, Schedule, Task};
 use crate::runtime::{blocking, context, scheduler, Config};
 use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
 use crate::sync::notify::Notify;
@@ -46,7 +46,7 @@ pub(crate) struct Handle {
 /// a function that will perform the scheduling work and acts as a capability token.
 struct Core {
     /// Scheduler run queue
-    tasks: VecDeque<task::Notified<Arc<Handle>>>,
+    tasks: VecDeque<Notified>,
 
     /// Current tick
     tick: u32,
@@ -66,8 +66,8 @@ struct Core {
 
 /// Scheduler state shared between threads.
 struct Shared {
-    /// Remote run queue. None if the `Runtime` has been dropped.
-    queue: Mutex<Option<VecDeque<task::Notified<Arc<Handle>>>>>,
+    /// Remote run queue
+    inject: Inject<Arc<Handle>>,
 
     /// Collection of all active tasks spawned onto this executor.
     owned: OwnedTasks<Arc<Handle>>,
@@ -95,6 +95,8 @@ struct Context {
     core: RefCell<Option<Box<Core>>>,
 }
 
+type Notified = task::Notified<Arc<Handle>>;
+
 /// Initial queue capacity.
 const INITIAL_CAPACITY: usize = 64;
 
@@ -109,14 +111,16 @@ impl CurrentThread {
         seed_generator: RngSeedGenerator,
         config: Config,
     ) -> (CurrentThread, Arc<Handle>) {
+        let worker_metrics = WorkerMetrics::from_config(&config);
+
         let handle = Arc::new(Handle {
             shared: Shared {
-                queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))),
+                inject: Inject::new(),
                 owned: OwnedTasks::new(),
                 woken: AtomicBool::new(false),
                 config,
                 scheduler_metrics: SchedulerMetrics::new(),
-                worker_metrics: WorkerMetrics::new(),
+                worker_metrics,
             },
             driver: driver_handle,
             blocking_spawner,
@@ -127,7 +131,7 @@ impl CurrentThread {
             tasks: VecDeque::with_capacity(INITIAL_CAPACITY),
             tick: 0,
             driver: Some(driver),
-            metrics: MetricsBatch::new(),
+            metrics: MetricsBatch::new(&handle.shared.worker_metrics),
             unhandled_panic: false,
         })));
 
@@ -209,25 +213,22 @@ impl CurrentThread {
 
             // Drain local queue
             // We already shut down every task, so we just need to drop the task.
-            while let Some(task) = core.pop_task(handle) {
+            while let Some(task) = core.next_local_task(handle) {
                 drop(task);
             }
 
-            // Drain remote queue and set it to None
-            let remote_queue = handle.shared.queue.lock().take();
+            // Close the injection queue
+            handle.shared.inject.close();
 
-            // Using `Option::take` to replace the shared queue with `None`.
-            // We already shut down every task, so we just need to drop the task.
-            if let Some(remote_queue) = remote_queue {
-                for task in remote_queue {
-                    drop(task);
-                }
+            // Drain remote queue
+            while let Some(task) = handle.shared.inject.pop() {
+                drop(task);
             }
 
             assert!(handle.shared.owned.is_empty());
 
             // Submit metrics
-            core.metrics.submit(&handle.shared.worker_metrics);
+            core.submit_metrics(handle);
 
             // Shutdown the resource drivers
             if let Some(driver) = core.driver.as_mut() {
@@ -248,7 +249,23 @@ impl fmt::Debug for CurrentThread {
 // ===== impl Core =====
 
 impl Core {
-    fn pop_task(&mut self, handle: &Handle) -> Option<task::Notified<Arc<Handle>>> {
+    /// Get and increment the current tick
+    fn tick(&mut self) {
+        self.tick = self.tick.wrapping_add(1);
+    }
+
+    fn next_task(&mut self, handle: &Handle) -> Option<Notified> {
+        if self.tick % handle.shared.config.global_queue_interval == 0 {
+            handle
+                .next_remote_task()
+                .or_else(|| self.next_local_task(handle))
+        } else {
+            self.next_local_task(handle)
+                .or_else(|| handle.next_remote_task())
+        }
+    }
+
+    fn next_local_task(&mut self, handle: &Handle) -> Option<Notified> {
         let ret = self.tasks.pop_front();
         handle
             .shared
@@ -257,7 +274,7 @@ impl Core {
         ret
     }
 
-    fn push_task(&mut self, handle: &Handle, task: task::Notified<Arc<Handle>>) {
+    fn push_task(&mut self, handle: &Handle, task: Notified) {
         self.tasks.push_back(task);
         self.metrics.inc_local_schedule_count();
         handle
@@ -265,6 +282,10 @@ impl Core {
             .worker_metrics
             .set_queue_depth(self.tasks.len());
     }
+
+    fn submit_metrics(&mut self, handle: &Handle) {
+        self.metrics.submit(&handle.shared.worker_metrics);
+    }
 }
 
 fn did_defer_tasks() -> bool {
@@ -275,14 +296,26 @@ fn wake_deferred_tasks() {
     context::with_defer(|deferred| deferred.wake());
 }
 
+#[cfg(tokio_taskdump)]
+fn wake_deferred_tasks_and_free() {
+    let wakers = context::with_defer(|deferred| deferred.take_deferred());
+    if let Some(wakers) = wakers {
+        for waker in wakers {
+            waker.wake();
+        }
+    }
+}
+
 // ===== impl Context =====
 
 impl Context {
     /// Execute the closure with the given scheduler core stored in the
     /// thread-local context.
     fn run_task<R>(&self, mut core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) {
-        core.metrics.incr_poll_count();
-        self.enter(core, || crate::runtime::coop::budget(f))
+        core.metrics.start_poll();
+        let mut ret = self.enter(core, || crate::runtime::coop::budget(f));
+        ret.0.metrics.end_poll();
+        ret
     }
 
     /// Blocks the current thread until an event is received by the driver,
@@ -303,7 +336,7 @@ impl Context {
         if core.tasks.is_empty() {
             // Park until the thread is signaled
             core.metrics.about_to_park();
-            core.metrics.submit(&handle.shared.worker_metrics);
+            core.submit_metrics(handle);
 
             let (c, _) = self.enter(core, || {
                 driver.park(&handle.driver);
@@ -330,7 +363,8 @@ impl Context {
     fn park_yield(&self, mut core: Box<Core>, handle: &Handle) -> Box<Core> {
         let mut driver = core.driver.take().expect("driver missing");
 
-        core.metrics.submit(&handle.shared.worker_metrics);
+        core.submit_metrics(handle);
+
         let (mut core, _) = self.enter(core, || {
             driver.park_timeout(&handle.driver, Duration::from_millis(0));
             wake_deferred_tasks();
@@ -377,11 +411,55 @@ impl Handle {
         handle
     }
 
-    fn pop(&self) -> Option<task::Notified<Arc<Handle>>> {
-        match self.shared.queue.lock().as_mut() {
-            Some(queue) => queue.pop_front(),
-            None => None,
-        }
+    /// Capture a snapshot of this runtime's state.
+    #[cfg(all(
+        tokio_unstable,
+        tokio_taskdump,
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))]
+    pub(crate) fn dump(&self) -> crate::runtime::Dump {
+        use crate::runtime::dump;
+        use task::trace::trace_current_thread;
+
+        let mut traces = vec![];
+
+        // todo: how to make this work outside of a runtime context?
+        CURRENT.with(|maybe_context| {
+            // drain the local queue
+            let context = if let Some(context) = maybe_context {
+                context
+            } else {
+                return;
+            };
+            let mut maybe_core = context.core.borrow_mut();
+            let core = if let Some(core) = maybe_core.as_mut() {
+                core
+            } else {
+                return;
+            };
+            let local = &mut core.tasks;
+
+            if self.shared.inject.is_closed() {
+                return;
+            }
+
+            traces = trace_current_thread(&self.shared.owned, local, &self.shared.inject)
+                .into_iter()
+                .map(dump::Task::new)
+                .collect();
+        });
+
+        // Taking a taskdump could wakes every task, but we probably don't want
+        // the `yield_now` vector to be that large under normal circumstances.
+        // Therefore, we free its allocation.
+        wake_deferred_tasks_and_free();
+
+        dump::Dump::new(traces)
+    }
+
+    fn next_remote_task(&self) -> Option<Notified> {
+        self.shared.inject.pop()
     }
 
     fn waker_ref(me: &Arc<Self>) -> WakerRef<'_> {
@@ -404,14 +482,7 @@ cfg_metrics! {
         }
 
         pub(crate) fn injection_queue_depth(&self) -> usize {
-            // TODO: avoid having to lock. The multi-threaded injection queue
-            // could probably be used here.
-            self.shared
-                .queue
-                .lock()
-                .as_ref()
-                .map(|queue| queue.len())
-                .unwrap_or(0)
+            self.shared.inject.len()
         }
 
         pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
@@ -430,6 +501,10 @@ cfg_metrics! {
         pub(crate) fn blocking_queue_depth(&self) -> usize {
             self.blocking_spawner.queue_depth()
         }
+
+        pub(crate) fn active_tasks_count(&self) -> usize {
+            self.shared.owned.active_tasks_count()
+        }
     }
 }
 
@@ -461,14 +536,9 @@ impl Schedule for Arc<Handle> {
                 // Track that a task was scheduled from **outside** of the runtime.
                 self.shared.scheduler_metrics.inc_remote_schedule_count();
 
-                // If the queue is None, then the runtime has shut down. We
-                // don't need to do anything with the notification in that case.
-                let mut guard = self.shared.queue.lock();
-                if let Some(queue) = guard.as_mut() {
-                    queue.push_back(task);
-                    drop(guard);
-                    self.driver.unpark();
-                }
+                // Schedule the task
+                self.shared.inject.push(task);
+                self.driver.unpark();
             }
         });
     }
@@ -554,15 +624,9 @@ impl CoreGuard<'_> {
                         return (core, None);
                     }
 
-                    // Get and increment the current tick
-                    let tick = core.tick;
-                    core.tick = core.tick.wrapping_add(1);
+                    core.tick();
 
-                    let entry = if tick % handle.shared.config.global_queue_interval == 0 {
-                        handle.pop().or_else(|| core.tasks.pop_front())
-                    } else {
-                        core.tasks.pop_front().or_else(|| handle.pop())
-                    };
+                    let entry = core.next_task(handle);
 
                     let task = match entry {
                         Some(entry) => entry,

tokio/src/runtime/scheduler/mod.rs

@@ -135,6 +135,14 @@ cfg_rt! {
                 }
             }
 
+            pub(crate) fn active_tasks_count(&self) -> usize {
+                match self {
+                    Handle::CurrentThread(handle) => handle.active_tasks_count(),
+                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThread(handle) => handle.active_tasks_count(),
+                }
+            }
+
             pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
                 match self {
                     Handle::CurrentThread(handle) => handle.scheduler_metrics(),

tokio/src/runtime/scheduler/multi_thread/counters.rs

@@ -0,0 +1,62 @@
+#[cfg(tokio_internal_mt_counters)]
+mod imp {
+    use std::sync::atomic::AtomicUsize;
+    use std::sync::atomic::Ordering::Relaxed;
+
+    static NUM_MAINTENANCE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_NOTIFY_LOCAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_UNPARKS_LOCAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_SCHEDULES: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_CAPPED: AtomicUsize = AtomicUsize::new(0);
+
+    impl Drop for super::Counters {
+        fn drop(&mut self) {
+            let notifies_local = NUM_NOTIFY_LOCAL.load(Relaxed);
+            let unparks_local = NUM_UNPARKS_LOCAL.load(Relaxed);
+            let maintenance = NUM_MAINTENANCE.load(Relaxed);
+            let lifo_scheds = NUM_LIFO_SCHEDULES.load(Relaxed);
+            let lifo_capped = NUM_LIFO_CAPPED.load(Relaxed);
+
+            println!("---");
+            println!("notifies (local): {}", notifies_local);
+            println!(" unparks (local): {}", unparks_local);
+            println!("     maintenance: {}", maintenance);
+            println!("  LIFO schedules: {}", lifo_scheds);
+            println!("     LIFO capped: {}", lifo_capped);
+        }
+    }
+
+    pub(crate) fn inc_num_inc_notify_local() {
+        NUM_NOTIFY_LOCAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_unparks_local() {
+        NUM_UNPARKS_LOCAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_maintenance() {
+        NUM_MAINTENANCE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_lifo_schedules() {
+        NUM_LIFO_SCHEDULES.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_lifo_capped() {
+        NUM_LIFO_CAPPED.fetch_add(1, Relaxed);
+    }
+}
+
+#[cfg(not(tokio_internal_mt_counters))]
+mod imp {
+    pub(crate) fn inc_num_inc_notify_local() {}
+    pub(crate) fn inc_num_unparks_local() {}
+    pub(crate) fn inc_num_maintenance() {}
+    pub(crate) fn inc_lifo_schedules() {}
+    pub(crate) fn inc_lifo_capped() {}
+}
+
+#[derive(Debug)]
+pub(crate) struct Counters;
+
+pub(super) use imp::*;

tokio/src/runtime/scheduler/multi_thread/handle.rs

@@ -69,6 +69,10 @@ cfg_metrics! {
             self.blocking_spawner.num_idle_threads()
         }
 
+        pub(crate) fn active_tasks_count(&self) -> usize {
+            self.shared.owned.active_tasks_count()
+        }
+
         pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
             &self.shared.scheduler_metrics
         }

tokio/src/runtime/scheduler/multi_thread/mod.rs

@@ -1,5 +1,8 @@
 //! Multi-threaded runtime
 
+mod counters;
+use counters::Counters;
+
 mod handle;
 pub(crate) use handle::Handle;
 

tokio/src/runtime/scheduler/multi_thread/queue.rs

@@ -110,6 +110,15 @@ impl<T> Local<T> {
         !self.inner.is_empty()
     }
 
+    /// How many tasks can be pushed into the queue
+    pub(crate) fn remaining_slots(&self) -> usize {
+        self.inner.remaining_slots()
+    }
+
+    pub(crate) fn max_capacity(&self) -> usize {
+        LOCAL_QUEUE_CAPACITY
+    }
+
     /// Returns false if there are any entries in the queue
     ///
     /// Separate to is_stealable so that refactors of is_stealable to "protect"
@@ -118,8 +127,62 @@ impl<T> Local<T> {
         !self.inner.is_empty()
     }
 
-    /// Pushes a task to the back of the local queue, skipping the LIFO slot.
-    pub(crate) fn push_back(
+    /// Pushes a batch of tasks to the back of the queue. All tasks must fit in
+    /// the local queue.
+    ///
+    /// # Panics
+    ///
+    /// The method panics if there is not enough capacity to fit in the queue.
+    pub(crate) fn push_back(&mut self, tasks: impl ExactSizeIterator<Item = task::Notified<T>>) {
+        let len = tasks.len();
+        assert!(len <= LOCAL_QUEUE_CAPACITY);
+
+        if len == 0 {
+            // Nothing to do
+            return;
+        }
+
+        let head = self.inner.head.load(Acquire);
+        let (steal, _) = unpack(head);
+
+        // safety: this is the **only** thread that updates this cell.
+        let mut tail = unsafe { self.inner.tail.unsync_load() };
+
+        if tail.wrapping_sub(steal) <= (LOCAL_QUEUE_CAPACITY - len) as UnsignedShort {
+            // Yes, this if condition is structured a bit weird (first block
+            // does nothing, second returns an error). It is this way to match
+            // `push_back_or_overflow`.
+        } else {
+            panic!()
+        }
+
+        for task in tasks {
+            let idx = tail as usize & MASK;
+
+            self.inner.buffer[idx].with_mut(|ptr| {
+                // Write the task to the slot
+                //
+                // Safety: There is only one producer and the above `if`
+                // condition ensures we don't touch a cell if there is a
+                // value, thus no consumer.
+                unsafe {
+                    ptr::write((*ptr).as_mut_ptr(), task);
+                }
+            });
+
+            tail = tail.wrapping_add(1);
+        }
+
+        self.inner.tail.store(tail, Release);
+    }
+
+    /// Pushes a task to the back of the local queue, if there is not enough
+    /// capacity in the queue, this triggers the overflow operation.
+    ///
+    /// When the queue overflows, half of the curent contents of the queue is
+    /// moved to the given Injection queue. This frees up capacity for more
+    /// tasks to be pushed into the local queue.
+    pub(crate) fn push_back_or_overflow(
         &mut self,
         mut task: task::Notified<T>,
         inject: &Inject<T>,
@@ -153,6 +216,11 @@ impl<T> Local<T> {
             }
         };
 
+        self.push_back_finish(task, tail);
+    }
+
+    // Second half of `push_back`
+    fn push_back_finish(&self, task: task::Notified<T>, tail: UnsignedShort) {
         // Map the position to a slot index.
         let idx = tail as usize & MASK;
 
@@ -501,6 +569,13 @@ impl<T> Drop for Local<T> {
 }
 
 impl<T> Inner<T> {
+    fn remaining_slots(&self) -> usize {
+        let (steal, _) = unpack(self.head.load(Acquire));
+        let tail = self.tail.load(Acquire);
+
+        LOCAL_QUEUE_CAPACITY - (tail.wrapping_sub(steal) as usize)
+    }
+
     fn len(&self) -> UnsignedShort {
         let (_, head) = unpack(self.head.load(Acquire));
         let tail = self.tail.load(Acquire);

tokio/src/runtime/scheduler/multi_thread/worker.rs

@@ -59,7 +59,7 @@
 use crate::loom::sync::{Arc, Mutex};
 use crate::runtime;
 use crate::runtime::context;
-use crate::runtime::scheduler::multi_thread::{queue, Handle, Idle, Parker, Unparker};
+use crate::runtime::scheduler::multi_thread::{queue, Counters, Handle, Idle, Parker, Unparker};
 use crate::runtime::task::{Inject, OwnedTasks};
 use crate::runtime::{
     blocking, coop, driver, scheduler, task, Config, MetricsBatch, SchedulerMetrics, WorkerMetrics,
@@ -68,7 +68,7 @@ use crate::util::atomic_cell::AtomicCell;
 use crate::util::rand::{FastRand, RngSeedGenerator};
 
 use std::cell::RefCell;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 /// A scheduler worker
 pub(super) struct Worker {
@@ -90,10 +90,14 @@ struct Core {
     /// When a task is scheduled from a worker, it is stored in this slot. The
     /// worker will check this slot for a task **before** checking the run
     /// queue. This effectively results in the **last** scheduled task to be run
-    /// next (LIFO). This is an optimization for message passing patterns and
-    /// helps to reduce latency.
+    /// next (LIFO). This is an optimization for improving locality which
+    /// benefits message passing patterns and helps to reduce latency.
     lifo_slot: Option<Notified>,
 
+    /// When `true`, locally scheduled tasks go to the LIFO slot. When `false`,
+    /// they go to the back of the `run_queue`.
+    lifo_enabled: bool,
+
     /// The worker-local run queue.
     run_queue: queue::Local<Arc<Handle>>,
 
@@ -115,6 +119,24 @@ struct Core {
 
     /// Fast random number generator.
     rand: FastRand,
+
+    /// Instant at which the maintenance routine last ran.
+    last_tuning_at: Instant,
+
+    /// Number of tasks polled since last maintenance run.
+    ///
+    /// Note, LIFO polls are batched w/ the task that scheduled the task in the
+    /// LIFO slot.
+    tasks_polled_since_last_tuning: u32,
+
+    /// Mean task poll time (exponentially weighted moving average)
+    task_mean_poll_time: Duration,
+
+    /// How many ticks until the global queue should be checked
+    ticks_until_check_global_queue: u32,
+
+    /// How often to check the global queue
+    global_queue_interval: u32,
 }
 
 /// State shared across all workers
@@ -148,6 +170,12 @@ pub(super) struct Shared {
     pub(super) scheduler_metrics: SchedulerMetrics,
 
     pub(super) worker_metrics: Box<[WorkerMetrics]>,
+
+    /// Only held to trigger some code on drop. This is used to get internal
+    /// runtime metrics that can be useful when doing performance
+    /// investigations. This does nothing (empty struct, no drop impl) unless
+    /// the `tokio_internal_mt_counters` cfg flag is set.
+    _counters: Counters,
 }
 
 /// Used to communicate with a worker from other threads.
@@ -185,6 +213,24 @@ type Notified = task::Notified<Arc<Handle>>;
 // Tracks thread-local state
 scoped_thread_local!(static CURRENT: Context);
 
+/// Value picked out of thin-air. Running the LIFO slot a handful of times
+/// seemms sufficient to benefit from locality. More than 3 times probably is
+/// overweighing. The value can be tuned in the future with data that shows
+/// improvements.
+const MAX_LIFO_POLLS_PER_TICK: usize = 3;
+
+/// Target injection queue check interval
+const TARGET_INJECTION_QUEUE_INTERVAL: Duration = Duration::from_micros(500);
+
+/// Maximum number of tasks we poll before checking the injection queue.
+const MAX_TASKS_POLLED_PER_INJECTION_QUEUE_INTERVAL: u32 = 61;
+
+const INIT_MEAN_POLL_TIME: Duration = Duration::from_micros(
+    // Workaround no const division for duration.
+    TARGET_INJECTION_QUEUE_INTERVAL.as_micros() as u64
+        / MAX_TASKS_POLLED_PER_INJECTION_QUEUE_INTERVAL as u64,
+);
+
 pub(super) fn create(
     size: usize,
     park: Parker,
@@ -203,20 +249,27 @@ pub(super) fn create(
 
         let park = park.clone();
         let unpark = park.unpark();
+        let metrics = WorkerMetrics::from_config(&config);
 
         cores.push(Box::new(Core {
             tick: 0,
             lifo_slot: None,
+            lifo_enabled: !config.disable_lifo_slot,
             run_queue,
             is_searching: false,
             is_shutdown: false,
             park: Some(park),
-            metrics: MetricsBatch::new(),
+            metrics: MetricsBatch::new(&metrics),
             rand: FastRand::new(config.seed_generator.next_seed()),
+            last_tuning_at: Instant::now(),
+            tasks_polled_since_last_tuning: 0,
+            task_mean_poll_time: INIT_MEAN_POLL_TIME,
+            global_queue_interval: config.global_queue_interval,
+            ticks_until_check_global_queue: config.global_queue_interval,
         }));
 
         remotes.push(Remote { steal, unpark });
-        worker_metrics.push(WorkerMetrics::new());
+        worker_metrics.push(metrics);
     }
 
     let handle = Arc::new(Handle {
@@ -229,6 +282,7 @@ pub(super) fn create(
             config,
             scheduler_metrics: SchedulerMetrics::new(),
             worker_metrics: worker_metrics.into_boxed_slice(),
+            _counters: Counters,
         },
         driver: driver_handle,
         blocking_spawner,
@@ -414,7 +468,17 @@ fn run(worker: Arc<Worker>) {
 
 impl Context {
     fn run(&self, mut core: Box<Core>) -> RunResult {
+        // Initialize `last_maintenance_at here. This resets any values that may
+        // exist before the core was stolen.
+        core.reset_tuning();
+
+        // Reset `lifo_enabled` here in case the core was previously stolen from
+        // a task that had the LIFO slot disabled.
+        self.reset_lifo_enabled(&mut core);
+
         while !core.is_shutdown {
+            self.assert_lifo_enabled_is_correct(&core);
+
             // Increment the tick
             core.tick();
 
@@ -427,9 +491,15 @@ impl Context {
                 continue;
             }
 
+            // No immediately available work. We may spend time trying to find
+            // more. This is a good time to tune our heuristics.
+            core.tune();
+
             // There is no more **local** work to process, try to steal work
             // from other workers.
             if let Some(task) = core.steal_work(&self.worker) {
+                core.reset_tuning();
+
                 core = self.run_task(task, core)?;
             } else {
                 // Wait for work
@@ -438,6 +508,8 @@ impl Context {
                 } else {
                     self.park(core)
                 };
+
+                core.reset_tuning();
             }
         }
 
@@ -455,13 +527,18 @@ impl Context {
         // another idle worker to try to steal work.
         core.transition_from_searching(&self.worker);
 
+        self.assert_lifo_enabled_is_correct(&core);
+
+        core.tasks_polled_since_last_tuning += 1;
+        core.metrics.start_poll();
+
         // Make the core available to the runtime context
-        core.metrics.incr_poll_count();
         *self.core.borrow_mut() = Some(core);
 
         // Run the task
         coop::budget(|| {
             task.run();
+            let mut lifo_polls = 0;
 
             // As long as there is budget remaining and a task exists in the
             // `lifo_slot`, then keep running.
@@ -470,40 +547,96 @@ impl Context {
                 // by another worker.
                 let mut core = match self.core.borrow_mut().take() {
                     Some(core) => core,
-                    None => return Err(()),
+                    None => {
+                        // In this case, we cannot call `reset_lifo_enabled()`
+                        // because the core was stolen. The stealer will handle
+                        // that at the top of `Context::run`
+                        return Err(());
+                    }
                 };
 
+                // If task poll times is enabled, measure the poll time. Note
+                // that, if the `core` is stolen, this means `block_in_place`
+                // was called, turning the poll into a "blocking op". In this
+                // case, we don't want to measure the poll time as it doesn't
+                // really count as an async poll anymore.
+                core.metrics.end_poll();
+
                 // Check for a task in the LIFO slot
                 let task = match core.lifo_slot.take() {
                     Some(task) => task,
-                    None => return Ok(core),
+                    None => {
+                        self.reset_lifo_enabled(&mut core);
+                        return Ok(core);
+                    }
                 };
 
-                if coop::has_budget_remaining() {
-                    // Run the LIFO task, then loop
-                    core.metrics.incr_poll_count();
-                    *self.core.borrow_mut() = Some(core);
-                    let task = self.worker.handle.shared.owned.assert_owner(task);
-                    task.run();
-                } else {
+                if !coop::has_budget_remaining() {
                     // Not enough budget left to run the LIFO task, push it to
                     // the back of the queue and return.
-                    core.run_queue
-                        .push_back(task, self.worker.inject(), &mut core.metrics);
+                    core.run_queue.push_back_or_overflow(
+                        task,
+                        self.worker.inject(),
+                        &mut core.metrics,
+                    );
+                    // If we hit this point, the LIFO slot should be enabled.
+                    // There is no need to reset it.
+                    debug_assert!(core.lifo_enabled);
                     return Ok(core);
                 }
+
+                // Track that we are about to run a task from the LIFO slot.
+                lifo_polls += 1;
+                super::counters::inc_lifo_schedules();
+
+                // Disable the LIFO slot if we reach our limit
+                //
+                // In ping-ping style workloads where task A notifies task B,
+                // which notifies task A again, continuously prioritizing the
+                // LIFO slot can cause starvation as these two tasks will
+                // repeatedly schedule the other. To mitigate this, we limit the
+                // number of times the LIFO slot is prioritized.
+                if lifo_polls >= MAX_LIFO_POLLS_PER_TICK {
+                    core.lifo_enabled = false;
+                    super::counters::inc_lifo_capped();
+                }
+
+                // Run the LIFO task, then loop
+                core.metrics.start_poll();
+                *self.core.borrow_mut() = Some(core);
+                let task = self.worker.handle.shared.owned.assert_owner(task);
+                task.run();
             }
         })
     }
 
+    fn reset_lifo_enabled(&self, core: &mut Core) {
+        core.lifo_enabled = !self.worker.handle.shared.config.disable_lifo_slot;
+    }
+
+    fn assert_lifo_enabled_is_correct(&self, core: &Core) {
+        debug_assert_eq!(
+            core.lifo_enabled,
+            !self.worker.handle.shared.config.disable_lifo_slot
+        );
+    }
+
     fn maintenance(&self, mut core: Box<Core>) -> Box<Core> {
         if core.tick % self.worker.handle.shared.config.event_interval == 0 {
+            super::counters::inc_num_maintenance();
+
+            // Run tuning logic
+            core.tune();
+
             // Call `park` with a 0 timeout. This enables the I/O driver, timer, ...
             // to run without actually putting the thread to sleep.
             core = self.park_timeout(core, Some(Duration::from_millis(0)));
 
             // Run regularly scheduled maintenance
             core.maintenance(&self.worker);
+
+            // Reset tuning counters
+            core.reset_tuning();
         }
 
         core
@@ -547,6 +680,8 @@ impl Context {
     }
 
     fn park_timeout(&self, mut core: Box<Core>, duration: Option<Duration>) -> Box<Core> {
+        self.assert_lifo_enabled_is_correct(&core);
+
         // Take the parker out of core
         let mut park = core.park.take().expect("park missing");
 
@@ -571,7 +706,7 @@ impl Context {
         // If there are tasks available to steal, but this worker is not
         // looking for tasks to steal, notify another worker.
         if !core.is_searching && core.run_queue.is_stealable() {
-            self.worker.handle.notify_parked();
+            self.worker.handle.notify_parked_local();
         }
 
         core
@@ -582,14 +717,49 @@ impl Core {
     /// Increment the tick
     fn tick(&mut self) {
         self.tick = self.tick.wrapping_add(1);
+        self.ticks_until_check_global_queue = self.ticks_until_check_global_queue.saturating_sub(1);
     }
 
     /// Return the next notified task available to this worker.
     fn next_task(&mut self, worker: &Worker) -> Option<Notified> {
-        if self.tick % worker.handle.shared.config.global_queue_interval == 0 {
-            worker.inject().pop().or_else(|| self.next_local_task())
+        if self.ticks_until_check_global_queue == 0 {
+            self.next_global_task(worker)
+                .or_else(|| self.next_local_task())
         } else {
-            self.next_local_task().or_else(|| worker.inject().pop())
+            let maybe_task = self.next_local_task();
+
+            if maybe_task.is_some() {
+                return maybe_task;
+            }
+
+            self.ticks_until_check_global_queue = self.global_queue_interval;
+
+            // Other threads can only **remove** tasks from the current worker's
+            // `run_queue`. So, we can be confident that by the time we call
+            // `run_queue.push_back` below, there will be *at least* `cap`
+            // available slots in the queue.
+            let cap = usize::min(
+                self.run_queue.remaining_slots(),
+                self.run_queue.max_capacity() / 2,
+            );
+
+            // The worker is currently idle, pull a batch of work from the
+            // injection queue. We don't want to pull *all* the work so other
+            // workers can also get some.
+            let n = usize::min(
+                worker.inject().len() / worker.handle.shared.remotes.len() + 1,
+                cap,
+            );
+
+            let mut tasks = worker.inject().pop_n(n);
+
+            // Pop the first task to return immedietly
+            let ret = tasks.next();
+
+            // Push the rest of the on the run queue
+            self.run_queue.push_back(tasks);
+
+            ret
         }
     }
 
@@ -597,6 +767,11 @@ impl Core {
         self.lifo_slot.take().or_else(|| self.run_queue.pop())
     }
 
+    fn next_global_task(&mut self, worker: &Worker) -> Option<Notified> {
+        self.ticks_until_check_global_queue = self.global_queue_interval;
+        worker.inject().pop()
+    }
+
     /// Function responsible for stealing tasks from another worker
     ///
     /// Note: Only if less than half the workers are searching for tasks to steal
@@ -731,6 +906,50 @@ impl Core {
 
         park.shutdown(&handle.driver);
     }
+
+    fn reset_tuning(&mut self) {
+        // Initialize `last_maintenance_at here. This resets any values that may
+        // exist before the core was stolen.
+        self.last_tuning_at = Instant::now();
+        self.tasks_polled_since_last_tuning = 0;
+    }
+
+    fn tune(&mut self) {
+        // weighs newer measurements fairly heavily.
+        const ALPHA: f64 = 0.5;
+
+        if self.tasks_polled_since_last_tuning > 0 {
+            let now = Instant::now();
+            let elapsed = now - self.last_tuning_at;
+            let mean = (elapsed / self.tasks_polled_since_last_tuning).as_nanos() as f64;
+
+            let weighted = self.task_mean_poll_time.as_nanos() as f64;
+            let weighted = ALPHA * mean + (1.0 - ALPHA) * weighted;
+
+            self.task_mean_poll_time = Duration::from_nanos(weighted as u64);
+
+            let global_queue_interval = std::cmp::min(
+                (TARGET_INJECTION_QUEUE_INTERVAL.as_nanos() / self.task_mean_poll_time.as_nanos())
+                    as u32,
+                MAX_TASKS_POLLED_PER_INJECTION_QUEUE_INTERVAL,
+            );
+
+            if global_queue_interval > self.global_queue_interval {
+                self.ticks_until_check_global_queue +=
+                    global_queue_interval - self.global_queue_interval;
+                self.global_queue_interval = global_queue_interval;
+            } else if global_queue_interval < self.global_queue_interval {
+                // Remove extra ticks, but make sure not to wrap
+                self.ticks_until_check_global_queue = self
+                    .ticks_until_check_global_queue
+                    .saturating_sub(self.global_queue_interval - global_queue_interval);
+                self.global_queue_interval = global_queue_interval;
+            }
+
+            self.last_tuning_at = now;
+            self.tasks_polled_since_last_tuning = 0;
+        }
+    }
 }
 
 impl Worker {
@@ -772,7 +991,7 @@ impl Handle {
             // Otherwise, use the inject queue.
             self.shared.inject.push(task);
             self.shared.scheduler_metrics.inc_remote_schedule_count();
-            self.notify_parked();
+            self.notify_parked_remote();
         })
     }
 
@@ -783,9 +1002,9 @@ impl Handle {
         // task must always be pushed to the back of the queue, enabling other
         // tasks to be executed. If **not** a yield, then there is more
         // flexibility and the task may go to the front of the queue.
-        let should_notify = if is_yield || self.shared.config.disable_lifo_slot {
+        let should_notify = if is_yield || !core.lifo_enabled {
             core.run_queue
-                .push_back(task, &self.shared.inject, &mut core.metrics);
+                .push_back_or_overflow(task, &self.shared.inject, &mut core.metrics);
             true
         } else {
             // Push to the LIFO slot
@@ -794,7 +1013,7 @@ impl Handle {
 
             if let Some(prev) = prev {
                 core.run_queue
-                    .push_back(prev, &self.shared.inject, &mut core.metrics);
+                    .push_back_or_overflow(prev, &self.shared.inject, &mut core.metrics);
             }
 
             core.lifo_slot = Some(task);
@@ -806,7 +1025,7 @@ impl Handle {
         // scheduling is from a resource driver. As notifications often come in
         // batches, the notification is delayed until the park is complete.
         if should_notify && core.park.is_some() {
-            self.notify_parked();
+            self.notify_parked_local();
         }
     }
 
@@ -816,7 +1035,16 @@ impl Handle {
         }
     }
 
-    fn notify_parked(&self) {
+    fn notify_parked_local(&self) {
+        super::counters::inc_num_inc_notify_local();
+
+        if let Some(index) = self.shared.idle.worker_to_notify() {
+            super::counters::inc_num_unparks_local();
+            self.shared.remotes[index].unpark.unpark(&self.driver);
+        }
+    }
+
+    fn notify_parked_remote(&self) {
         if let Some(index) = self.shared.idle.worker_to_notify() {
             self.shared.remotes[index].unpark.unpark(&self.driver);
         }
@@ -831,13 +1059,13 @@ impl Handle {
     fn notify_if_work_pending(&self) {
         for remote in &self.shared.remotes[..] {
             if !remote.steal.is_empty() {
-                self.notify_parked();
+                self.notify_parked_local();
                 return;
             }
         }
 
         if !self.shared.inject.is_empty() {
-            self.notify_parked();
+            self.notify_parked_local();
         }
     }
 
@@ -845,7 +1073,7 @@ impl Handle {
         if self.shared.idle.transition_worker_from_searching() {
             // We are the final searching worker. Because work was found, we
             // need to notify another worker.
-            self.notify_parked();
+            self.notify_parked_local();
         }
     }
 

tokio/src/runtime/task/core.rs

@@ -278,15 +278,11 @@ impl<T: Future, S: Schedule> Core<T, S> {
     }
 }
 
-cfg_rt_multi_thread! {
-    impl Header {
-        pub(super) unsafe fn set_next(&self, next: Option<NonNull<Header>>) {
-            self.queue_next.with_mut(|ptr| *ptr = next);
-        }
-    }
-}
-
 impl Header {
+    pub(super) unsafe fn set_next(&self, next: Option<NonNull<Header>>) {
+        self.queue_next.with_mut(|ptr| *ptr = next);
+    }
+
     // safety: The caller must guarantee exclusive access to this field, and
     // must ensure that the id is either 0 or the id of the OwnedTasks
     // containing this task.

tokio/src/runtime/task/inject.rs

@@ -1,7 +1,7 @@
 //! Inject queue used to send wakeups to a work-stealing scheduler
 
 use crate::loom::sync::atomic::AtomicUsize;
-use crate::loom::sync::Mutex;
+use crate::loom::sync::{Mutex, MutexGuard};
 use crate::runtime::task;
 
 use std::marker::PhantomData;
@@ -32,6 +32,12 @@ struct Pointers {
     tail: Option<NonNull<task::Header>>,
 }
 
+pub(crate) struct Pop<'a, T: 'static> {
+    len: usize,
+    pointers: Option<MutexGuard<'a, Pointers>>,
+    _p: PhantomData<T>,
+}
+
 unsafe impl<T> Send for Inject<T> {}
 unsafe impl<T> Sync for Inject<T> {}
 
@@ -52,6 +58,12 @@ impl<T: 'static> Inject<T> {
         self.len() == 0
     }
 
+    // Kind of annoying to have to include the cfg here
+    #[cfg(any(tokio_taskdump, all(feature = "rt-multi-thread", not(tokio_wasi))))]
+    pub(crate) fn is_closed(&self) -> bool {
+        self.pointers.lock().is_closed
+    }
+
     /// Closes the injection queue, returns `true` if the queue is open when the
     /// transition is made.
     pub(crate) fn close(&self) -> bool {
@@ -65,10 +77,6 @@ impl<T: 'static> Inject<T> {
         true
     }
 
-    pub(crate) fn is_closed(&self) -> bool {
-        self.pointers.lock().is_closed
-    }
-
     pub(crate) fn len(&self) -> usize {
         self.len.load(Acquire)
     }
@@ -104,100 +112,108 @@ impl<T: 'static> Inject<T> {
         self.len.store(len + 1, Release);
     }
 
-    /// Pushes several values into the queue.
-    #[inline]
-    pub(crate) fn push_batch<I>(&self, mut iter: I)
-    where
-        I: Iterator<Item = task::Notified<T>>,
-    {
-        let first = match iter.next() {
-            Some(first) => first.into_raw(),
-            None => return,
-        };
-
-        // Link up all the tasks.
-        let mut prev = first;
-        let mut counter = 1;
-
-        // We are going to be called with an `std::iter::Chain`, and that
-        // iterator overrides `for_each` to something that is easier for the
-        // compiler to optimize than a loop.
-        iter.for_each(|next| {
-            let next = next.into_raw();
-
-            // safety: Holding the Notified for a task guarantees exclusive
-            // access to the `queue_next` field.
-            set_next(prev, Some(next));
-            prev = next;
-            counter += 1;
-        });
-
-        // Now that the tasks are linked together, insert them into the
-        // linked list.
-        self.push_batch_inner(first, prev, counter);
+    pub(crate) fn pop(&self) -> Option<task::Notified<T>> {
+        self.pop_n(1).next()
     }
 
-    /// Inserts several tasks that have been linked together into the queue.
-    ///
-    /// The provided head and tail may be be the same task. In this case, a
-    /// single task is inserted.
-    #[inline]
-    fn push_batch_inner(
-        &self,
-        batch_head: NonNull<task::Header>,
-        batch_tail: NonNull<task::Header>,
-        num: usize,
-    ) {
-        debug_assert!(get_next(batch_tail).is_none());
+    pub(crate) fn pop_n(&self, n: usize) -> Pop<'_, T> {
+        use std::cmp;
 
-        let mut p = self.pointers.lock();
-
-        if let Some(tail) = p.tail {
-            set_next(tail, Some(batch_head));
-        } else {
-            p.head = Some(batch_head);
+        // Fast path, if len == 0, then there are no values
+        if self.is_empty() {
+            return Pop {
+                len: 0,
+                pointers: None,
+                _p: PhantomData,
+            };
         }
 
-        p.tail = Some(batch_tail);
+        // Lock the queue
+        let p = self.pointers.lock();
 
-        // Increment the count.
-        //
         // safety: All updates to the len atomic are guarded by the mutex. As
         // such, a non-atomic load followed by a store is safe.
         let len = unsafe { self.len.unsync_load() };
 
-        self.len.store(len + num, Release);
-    }
-
-    pub(crate) fn pop(&self) -> Option<task::Notified<T>> {
-        // Fast path, if len == 0, then there are no values
-        if self.is_empty() {
-            return None;
-        }
-
-        let mut p = self.pointers.lock();
-
-        // It is possible to hit null here if another thread popped the last
-        // task between us checking `len` and acquiring the lock.
-        let task = p.head?;
-
-        p.head = get_next(task);
-
-        if p.head.is_none() {
-            p.tail = None;
-        }
-
-        set_next(task, None);
+        let n = cmp::min(n, len);
 
         // Decrement the count.
-        //
-        // safety: All updates to the len atomic are guarded by the mutex. As
-        // such, a non-atomic load followed by a store is safe.
-        self.len
-            .store(unsafe { self.len.unsync_load() } - 1, Release);
+        self.len.store(len - n, Release);
 
-        // safety: a `Notified` is pushed into the queue and now it is popped!
-        Some(unsafe { task::Notified::from_raw(task) })
+        Pop {
+            len: n,
+            pointers: Some(p),
+            _p: PhantomData,
+        }
+    }
+}
+
+cfg_rt_multi_thread! {
+    impl<T: 'static> Inject<T> {
+        /// Pushes several values into the queue.
+        #[inline]
+        pub(crate) fn push_batch<I>(&self, mut iter: I)
+        where
+            I: Iterator<Item = task::Notified<T>>,
+        {
+            let first = match iter.next() {
+                Some(first) => first.into_raw(),
+                None => return,
+            };
+
+            // Link up all the tasks.
+            let mut prev = first;
+            let mut counter = 1;
+
+            // We are going to be called with an `std::iter::Chain`, and that
+            // iterator overrides `for_each` to something that is easier for the
+            // compiler to optimize than a loop.
+            iter.for_each(|next| {
+                let next = next.into_raw();
+
+                // safety: Holding the Notified for a task guarantees exclusive
+                // access to the `queue_next` field.
+                set_next(prev, Some(next));
+                prev = next;
+                counter += 1;
+            });
+
+            // Now that the tasks are linked together, insert them into the
+            // linked list.
+            self.push_batch_inner(first, prev, counter);
+        }
+
+        /// Inserts several tasks that have been linked together into the queue.
+        ///
+        /// The provided head and tail may be be the same task. In this case, a
+        /// single task is inserted.
+        #[inline]
+        fn push_batch_inner(
+            &self,
+            batch_head: NonNull<task::Header>,
+            batch_tail: NonNull<task::Header>,
+            num: usize,
+        ) {
+            debug_assert!(get_next(batch_tail).is_none());
+
+            let mut p = self.pointers.lock();
+
+            if let Some(tail) = p.tail {
+                set_next(tail, Some(batch_head));
+            } else {
+                p.head = Some(batch_head);
+            }
+
+            p.tail = Some(batch_tail);
+
+            // Increment the count.
+            //
+            // safety: All updates to the len atomic are guarded by the mutex. As
+            // such, a non-atomic load followed by a store is safe.
+            let len = unsafe { self.len.unsync_load() };
+
+            self.len.store(len + num, Release);
+        }
     }
 }
 
@@ -209,6 +225,63 @@ impl<T: 'static> Drop for Inject<T> {
     }
 }
 
+impl<'a, T: 'static> Iterator for Pop<'a, T> {
+    type Item = task::Notified<T>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.len == 0 {
+            return None;
+        }
+
+        // `pointers` is always `Some` when `len() > 0`
+        let pointers = self.pointers.as_mut().unwrap();
+        let ret = pointers.pop();
+
+        debug_assert!(ret.is_some());
+
+        self.len -= 1;
+
+        if self.len == 0 {
+            self.pointers = None;
+        }
+
+        ret
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.len, Some(self.len))
+    }
+}
+
+impl<'a, T: 'static> ExactSizeIterator for Pop<'a, T> {
+    fn len(&self) -> usize {
+        self.len
+    }
+}
+
+impl<'a, T: 'static> Drop for Pop<'a, T> {
+    fn drop(&mut self) {
+        for _ in self.by_ref() {}
+    }
+}
+
+impl Pointers {
+    fn pop<T: 'static>(&mut self) -> Option<task::Notified<T>> {
+        let task = self.head?;
+
+        self.head = get_next(task);
+
+        if self.head.is_none() {
+            self.tail = None;
+        }
+
+        set_next(task, None);
+
+        // safety: a `Notified` is pushed into the queue and now it is popped!
+        Some(unsafe { task::Notified::from_raw(task) })
+    }
+}
+
 fn get_next(header: NonNull<task::Header>) -> Option<NonNull<task::Header>> {
     unsafe { header.as_ref().queue_next.with(|ptr| *ptr) }
 }

tokio/src/runtime/task/join.rs

@@ -313,6 +313,7 @@ impl<T> Future for JoinHandle<T> {
     type Output = super::Result<T>;
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        ready!(crate::trace::trace_leaf(cx));
         let mut ret = Poll::Pending;
 
         // Keep track of task budget

tokio/src/runtime/task/list.rs

@@ -10,7 +10,7 @@ use crate::future::Future;
 use crate::loom::cell::UnsafeCell;
 use crate::loom::sync::Mutex;
 use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task};
-use crate::util::linked_list::{Link, LinkedList};
+use crate::util::linked_list::{CountedLinkedList, Link, LinkedList};
 
 use std::marker::PhantomData;
 
@@ -54,9 +54,13 @@ cfg_not_has_atomic_u64! {
 }
 
 pub(crate) struct OwnedTasks<S: 'static> {
-    inner: Mutex<OwnedTasksInner<S>>,
+    inner: Mutex<CountedOwnedTasksInner<S>>,
     id: u64,
 }
+struct CountedOwnedTasksInner<S: 'static> {
+    list: CountedLinkedList<Task<S>, <Task<S> as Link>::Target>,
+    closed: bool,
+}
 pub(crate) struct LocalOwnedTasks<S: 'static> {
     inner: UnsafeCell<OwnedTasksInner<S>>,
     id: u64,
@@ -70,8 +74,8 @@ struct OwnedTasksInner<S: 'static> {
 impl<S: 'static> OwnedTasks<S> {
     pub(crate) fn new() -> Self {
         Self {
-            inner: Mutex::new(OwnedTasksInner {
-                list: LinkedList::new(),
+            inner: Mutex::new(CountedOwnedTasksInner {
+                list: CountedLinkedList::new(),
                 closed: false,
             }),
             id: get_next_id(),
@@ -153,6 +157,10 @@ impl<S: 'static> OwnedTasks<S> {
         }
     }
 
+    pub(crate) fn active_tasks_count(&self) -> usize {
+        self.inner.lock().list.count()
+    }
+
     pub(crate) fn remove(&self, task: &Task<S>) -> Option<Task<S>> {
         let task_id = task.header().get_owner_id();
         if task_id == 0 {
@@ -172,6 +180,18 @@ impl<S: 'static> OwnedTasks<S> {
     }
 }
 
+cfg_taskdump! {
+    impl<S: 'static> OwnedTasks<S> {
+        /// Locks the tasks, and calls `f` on an iterator over them.
+        pub(crate) fn for_each<F>(&self, f: F)
+        where
+            F: FnMut(&Task<S>)
+        {
+            self.inner.lock().list.for_each(f)
+        }
+    }
+}
+
 impl<S: 'static> LocalOwnedTasks<S> {
     pub(crate) fn new() -> Self {
         Self {

tokio/src/runtime/task/mod.rs

@@ -182,10 +182,8 @@ mod id;
 #[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
 pub use id::{id, try_id, Id};
 
-cfg_rt_multi_thread! {
-    mod inject;
-    pub(super) use self::inject::Inject;
-}
+mod inject;
+pub(super) use self::inject::Inject;
 
 #[cfg(feature = "rt")]
 mod abort;
@@ -207,6 +205,10 @@ use self::state::State;
 
 mod waker;
 
+cfg_taskdump! {
+    pub(crate) mod trace;
+}
+
 use crate::future::Future;
 use crate::util::linked_list;
 
@@ -340,6 +342,17 @@ impl<S: 'static> Task<S> {
         }
     }
 
+    #[cfg(all(
+        tokio_unstable,
+        tokio_taskdump,
+        feature = "rt",
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))]
+    pub(super) fn as_raw(&self) -> RawTask {
+        self.raw
+    }
+
     fn header(&self) -> &Header {
         self.raw.header()
     }
@@ -355,25 +368,23 @@ impl<S: 'static> Notified<S> {
     }
 }
 
-cfg_rt_multi_thread! {
-    impl<S: 'static> Notified<S> {
-        unsafe fn from_raw(ptr: NonNull<Header>) -> Notified<S> {
-            Notified(Task::from_raw(ptr))
-        }
+impl<S: 'static> Notified<S> {
+    unsafe fn from_raw(ptr: NonNull<Header>) -> Notified<S> {
+        Notified(Task::from_raw(ptr))
     }
+}
 
-    impl<S: 'static> Task<S> {
-        fn into_raw(self) -> NonNull<Header> {
-            let ret = self.raw.header_ptr();
-            mem::forget(self);
-            ret
-        }
+impl<S: 'static> Task<S> {
+    fn into_raw(self) -> NonNull<Header> {
+        let ret = self.raw.header_ptr();
+        mem::forget(self);
+        ret
     }
+}
 
-    impl<S: 'static> Notified<S> {
-        fn into_raw(self) -> NonNull<Header> {
-            self.0.into_raw()
-        }
+impl<S: 'static> Notified<S> {
+    fn into_raw(self) -> NonNull<Header> {
+        self.0.into_raw()
     }
 }
 

tokio/src/runtime/task/raw.rs

@@ -6,7 +6,7 @@ use std::ptr::NonNull;
 use std::task::{Poll, Waker};
 
 /// Raw task handle
-pub(super) struct RawTask {
+pub(in crate::runtime) struct RawTask {
     ptr: NonNull<Header>,
 }
 
@@ -195,7 +195,7 @@ impl RawTask {
     }
 
     /// Safety: mutual exclusion is required to call this function.
-    pub(super) fn poll(self) {
+    pub(crate) fn poll(self) {
         let vtable = self.header().vtable;
         unsafe { (vtable.poll)(self.ptr) }
     }

tokio/src/runtime/task/state.rs

@@ -88,7 +88,7 @@ pub(super) enum TransitionToNotifiedByVal {
 }
 
 #[must_use]
-pub(super) enum TransitionToNotifiedByRef {
+pub(crate) enum TransitionToNotifiedByRef {
     DoNothing,
     Submit,
 }
@@ -270,6 +270,22 @@ impl State {
         })
     }
 
+    /// Transitions the state to `NOTIFIED`, unconditionally increasing the ref count.
+    #[cfg(all(
+        tokio_unstable,
+        tokio_taskdump,
+        feature = "rt",
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))]
+    pub(super) fn transition_to_notified_for_tracing(&self) {
+        self.fetch_update_action(|mut snapshot| {
+            snapshot.set_notified();
+            snapshot.ref_inc();
+            ((), Some(snapshot))
+        });
+    }
+
     /// Sets the cancelled bit and transitions the state to `NOTIFIED` if idle.
     ///
     /// Returns `true` if the task needs to be submitted to the pool for

tokio/src/runtime/task/trace/mod.rs

@@ -0,0 +1,265 @@
+use crate::loom::sync::Arc;
+use crate::runtime::scheduler::current_thread;
+use crate::runtime::task::Inject;
+use backtrace::BacktraceFrame;
+use std::cell::Cell;
+use std::collections::VecDeque;
+use std::ffi::c_void;
+use std::fmt;
+use std::future::Future;
+use std::pin::Pin;
+use std::ptr::{self, NonNull};
+use std::task::{self, Poll};
+
+mod symbol;
+mod tree;
+
+use symbol::Symbol;
+use tree::Tree;
+
+use super::{Notified, OwnedTasks};
+
+type Backtrace = Vec<BacktraceFrame>;
+type SymbolTrace = Vec<Symbol>;
+
+/// The ambiant backtracing context.
+pub(crate) struct Context {
+    /// The address of [`Trace::root`] establishes an upper unwinding bound on
+    /// the backtraces in `Trace`.
+    active_frame: Cell<Option<NonNull<Frame>>>,
+    /// The place to stash backtraces.
+    collector: Cell<Option<Trace>>,
+}
+
+/// A [`Frame`] in an intrusive, doubly-linked tree of [`Frame`]s.
+struct Frame {
+    /// The location associated with this frame.
+    inner_addr: *const c_void,
+
+    /// The parent frame, if any.
+    parent: Option<NonNull<Frame>>,
+}
+
+/// An tree execution trace.
+///
+/// Traces are captured with [`Trace::capture`], rooted with [`Trace::root`]
+/// and leaved with [`trace_leaf`].
+#[derive(Clone, Debug)]
+pub(crate) struct Trace {
+    // The linear backtraces that comprise this trace. These linear traces can
+    // be re-knitted into a tree.
+    backtraces: Vec<Backtrace>,
+}
+
+pin_project_lite::pin_project! {
+    #[derive(Debug, Clone)]
+    #[must_use = "futures do nothing unless you `.await` or poll them"]
+    pub(crate) struct Root<T> {
+        #[pin]
+        future: T,
+    }
+}
+
+impl Context {
+    pub(crate) const fn new() -> Self {
+        Context {
+            active_frame: Cell::new(None),
+            collector: Cell::new(None),
+        }
+    }
+
+    /// SAFETY: Callers of this function must ensure that trace frames always
+    /// form a valid linked list.
+    unsafe fn with_current<F, R>(f: F) -> R
+    where
+        F: FnOnce(&Self) -> R,
+    {
+        crate::runtime::context::with_trace(f)
+    }
+
+    unsafe fn with_current_frame<F, R>(f: F) -> R
+    where
+        F: FnOnce(&Cell<Option<NonNull<Frame>>>) -> R,
+    {
+        Self::with_current(|context| f(&context.active_frame))
+    }
+
+    fn with_current_collector<F, R>(f: F) -> R
+    where
+        F: FnOnce(&Cell<Option<Trace>>) -> R,
+    {
+        unsafe { Self::with_current(|context| f(&context.collector)) }
+    }
+}
+
+impl Trace {
+    /// Invokes `f`, returning both its result and the collection of backtraces
+    /// captured at each sub-invocation of [`trace_leaf`].
+    #[inline(never)]
+    pub(crate) fn capture<F, R>(f: F) -> (R, Trace)
+    where
+        F: FnOnce() -> R,
+    {
+        let collector = Trace { backtraces: vec![] };
+
+        let previous = Context::with_current_collector(|current| current.replace(Some(collector)));
+
+        let result = f();
+
+        let collector =
+            Context::with_current_collector(|current| current.replace(previous)).unwrap();
+
+        (result, collector)
+    }
+
+    /// The root of a trace.
+    #[inline(never)]
+    pub(crate) fn root<F>(future: F) -> Root<F> {
+        Root { future }
+    }
+}
+
+/// If this is a sub-invocation of [`Trace::capture`], capture a backtrace.
+///
+/// The captured backtrace will be returned by [`Trace::capture`].
+///
+/// Invoking this function does nothing when it is not a sub-invocation
+/// [`Trace::capture`].
+// This function is marked `#[inline(never)]` to ensure that it gets a distinct `Frame` in the
+// backtrace, below which frames should not be included in the backtrace (since they reflect the
+// internal implementation details of this crate).
+#[inline(never)]
+pub(crate) fn trace_leaf(cx: &mut task::Context<'_>) -> Poll<()> {
+    // Safety: We don't manipulate the current context's active frame.
+    let did_trace = unsafe {
+        Context::with_current(|context_cell| {
+            if let Some(mut collector) = context_cell.collector.take() {
+                let mut frames = vec![];
+                let mut above_leaf = false;
+
+                if let Some(active_frame) = context_cell.active_frame.get() {
+                    let active_frame = active_frame.as_ref();
+
+                    backtrace::trace(|frame| {
+                        let below_root = !ptr::eq(frame.symbol_address(), active_frame.inner_addr);
+
+                        // only capture frames above `Trace::leaf` and below
+                        // `Trace::root`.
+                        if above_leaf && below_root {
+                            frames.push(frame.to_owned().into());
+                        }
+
+                        if ptr::eq(frame.symbol_address(), trace_leaf as *const _) {
+                            above_leaf = true;
+                        }
+
+                        // only continue unwinding if we're below `Trace::root`
+                        below_root
+                    });
+                }
+                collector.backtraces.push(frames);
+                context_cell.collector.set(Some(collector));
+                true
+            } else {
+                false
+            }
+        })
+    };
+
+    if did_trace {
+        // Use the same logic that `yield_now` uses to send out wakeups after
+        // the task yields.
+        let defer = crate::runtime::context::with_defer(|rt| {
+            rt.defer(cx.waker());
+        });
+        debug_assert!(defer.is_some());
+
+        Poll::Pending
+    } else {
+        Poll::Ready(())
+    }
+}
+
+impl fmt::Display for Trace {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        Tree::from_trace(self.clone()).fmt(f)
+    }
+}
+
+fn defer<F: FnOnce() -> R, R>(f: F) -> impl Drop {
+    use std::mem::ManuallyDrop;
+
+    struct Defer<F: FnOnce() -> R, R>(ManuallyDrop<F>);
+
+    impl<F: FnOnce() -> R, R> Drop for Defer<F, R> {
+        #[inline(always)]
+        fn drop(&mut self) {
+            unsafe {
+                ManuallyDrop::take(&mut self.0)();
+            }
+        }
+    }
+
+    Defer(ManuallyDrop::new(f))
+}
+
+impl<T: Future> Future for Root<T> {
+    type Output = T::Output;
+
+    #[inline(never)]
+    fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> {
+        // SAFETY: The context's current frame is restored to its original state
+        // before `frame` is dropped.
+        unsafe {
+            let mut frame = Frame {
+                inner_addr: Self::poll as *const c_void,
+                parent: None,
+            };
+
+            Context::with_current_frame(|current| {
+                frame.parent = current.take();
+                current.set(Some(NonNull::from(&frame)));
+            });
+
+            let _restore = defer(|| {
+                Context::with_current_frame(|current| {
+                    current.set(frame.parent);
+                });
+            });
+
+            let this = self.project();
+            this.future.poll(cx)
+        }
+    }
+}
+
+/// Trace and poll all tasks of the current_thread runtime.
+pub(in crate::runtime) fn trace_current_thread(
+    owned: &OwnedTasks<Arc<current_thread::Handle>>,
+    local: &mut VecDeque<Notified<Arc<current_thread::Handle>>>,
+    injection: &Inject<Arc<current_thread::Handle>>,
+) -> Vec<Trace> {
+    // clear the local and injection queues
+    local.clear();
+
+    while let Some(task) = injection.pop() {
+        drop(task);
+    }
+
+    // notify each task
+    let mut tasks = vec![];
+    owned.for_each(|task| {
+        // set the notified bit
+        task.as_raw().state().transition_to_notified_for_tracing();
+        // store the raw tasks into a vec
+        tasks.push(task.as_raw());
+    });
+
+    tasks
+        .into_iter()
+        .map(|task| {
+            let ((), trace) = Trace::capture(|| task.poll());
+            trace
+        })
+        .collect()
+}

tokio/src/runtime/task/trace/symbol.rs

@@ -0,0 +1,92 @@
+use backtrace::BacktraceSymbol;
+use std::fmt;
+use std::hash::{Hash, Hasher};
+use std::ptr;
+
+/// A symbol in a backtrace.
+///
+/// This wrapper type serves two purposes. The first is that it provides a
+/// representation of a symbol that can be inserted into hashmaps and hashsets;
+/// the [`backtrace`] crate does not define [`Hash`], [`PartialEq`], or [`Eq`]
+/// on [`BacktraceSymbol`], and recommends that users define their own wrapper
+/// which implements these traits.
+///
+/// Second, this wrapper includes a `parent_hash` field that uniquely
+/// identifies this symbol's position in its trace. Otherwise, e.g., our code
+/// would not be able to distinguish between recursive calls of a function at
+/// different depths.
+#[derive(Clone)]
+pub(super) struct Symbol {
+    pub(super) symbol: BacktraceSymbol,
+    pub(super) parent_hash: u64,
+}
+
+impl Hash for Symbol {
+    fn hash<H>(&self, state: &mut H)
+    where
+        H: Hasher,
+    {
+        if let Some(name) = self.symbol.name() {
+            name.as_bytes().hash(state);
+        }
+
+        if let Some(addr) = self.symbol.addr() {
+            ptr::hash(addr, state);
+        }
+
+        self.symbol.filename().hash(state);
+        self.symbol.lineno().hash(state);
+        self.symbol.colno().hash(state);
+        self.parent_hash.hash(state);
+    }
+}
+
+impl PartialEq for Symbol {
+    fn eq(&self, other: &Self) -> bool {
+        (self.parent_hash == other.parent_hash)
+            && match (self.symbol.name(), other.symbol.name()) {
+                (None, None) => true,
+                (Some(lhs_name), Some(rhs_name)) => lhs_name.as_bytes() == rhs_name.as_bytes(),
+                _ => false,
+            }
+            && match (self.symbol.addr(), other.symbol.addr()) {
+                (None, None) => true,
+                (Some(lhs_addr), Some(rhs_addr)) => ptr::eq(lhs_addr, rhs_addr),
+                _ => false,
+            }
+            && (self.symbol.filename() == other.symbol.filename())
+            && (self.symbol.lineno() == other.symbol.lineno())
+            && (self.symbol.colno() == other.symbol.colno())
+    }
+}
+
+impl Eq for Symbol {}
+
+impl fmt::Display for Symbol {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Some(name) = self.symbol.name() {
+            let name = name.to_string();
+            let name = if let Some((name, _)) = name.rsplit_once("::") {
+                name
+            } else {
+                &name
+            };
+            fmt::Display::fmt(&name, f)?;
+        }
+
+        if let Some(filename) = self.symbol.filename() {
+            f.write_str(" at ")?;
+            filename.to_string_lossy().fmt(f)?;
+            if let Some(lineno) = self.symbol.lineno() {
+                f.write_str(":")?;
+                fmt::Display::fmt(&lineno, f)?;
+                if let Some(colno) = self.symbol.colno() {
+                    f.write_str(":")?;
+                    fmt::Display::fmt(&colno, f)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+}

tokio/src/runtime/task/trace/tree.rs

@@ -0,0 +1,126 @@
+use std::collections::{hash_map::DefaultHasher, HashMap, HashSet};
+use std::fmt;
+use std::hash::{Hash, Hasher};
+
+use super::{Backtrace, Symbol, SymbolTrace, Trace};
+
+/// An adjacency list representation of an execution tree.
+///
+/// This tree provides a convenient intermediate representation for formatting
+/// [`Trace`] as a tree.
+pub(super) struct Tree {
+    /// The roots of the trees.
+    ///
+    /// There should only be one root, but the code is robust to multiple roots.
+    roots: HashSet<Symbol>,
+
+    /// The adjacency list of symbols in the execution tree(s).
+    edges: HashMap<Symbol, HashSet<Symbol>>,
+}
+
+impl Tree {
+    /// Constructs a [`Tree`] from [`Trace`]
+    pub(super) fn from_trace(trace: Trace) -> Self {
+        let mut roots: HashSet<Symbol> = HashSet::default();
+        let mut edges: HashMap<Symbol, HashSet<Symbol>> = HashMap::default();
+
+        for trace in trace.backtraces {
+            let trace = to_symboltrace(trace);
+
+            if let Some(first) = trace.first() {
+                roots.insert(first.to_owned());
+            }
+
+            let mut trace = trace.into_iter().peekable();
+            while let Some(frame) = trace.next() {
+                let subframes = edges.entry(frame).or_default();
+                if let Some(subframe) = trace.peek() {
+                    subframes.insert(subframe.clone());
+                }
+            }
+        }
+
+        Tree { roots, edges }
+    }
+
+    /// Produces the sub-symbols of a given symbol.
+    fn consequences(&self, frame: &Symbol) -> Option<impl ExactSizeIterator<Item = &Symbol>> {
+        Some(self.edges.get(frame)?.iter())
+    }
+
+    /// Format this [`Tree`] as a textual tree.
+    fn display<W: fmt::Write>(
+        &self,
+        f: &mut W,
+        root: &Symbol,
+        is_last: bool,
+        prefix: &str,
+    ) -> fmt::Result {
+        let root_fmt = format!("{}", root);
+
+        let current;
+        let next;
+
+        if is_last {
+            current = format!("{prefix}└╼\u{a0}{root_fmt}");
+            next = format!("{}\u{a0}\u{a0}\u{a0}", prefix);
+        } else {
+            current = format!("{prefix}├╼\u{a0}{root_fmt}");
+            next = format!("{}│\u{a0}\u{a0}", prefix);
+        }
+
+        write!(f, "{}", {
+            let mut current = current.chars();
+            current.next().unwrap();
+            current.next().unwrap();
+            &current.as_str()
+        })?;
+
+        if let Some(consequences) = self.consequences(root) {
+            let len = consequences.len();
+            for (i, consequence) in consequences.enumerate() {
+                let is_last = i == len - 1;
+                writeln!(f)?;
+                self.display(f, consequence, is_last, &next)?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl fmt::Display for Tree {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        for root in &self.roots {
+            self.display(f, root, true, " ")?;
+        }
+        Ok(())
+    }
+}
+
+/// Resolve a sequence of [`backtrace::BacktraceFrame`]s into a sequence of
+/// [`Symbol`]s.
+fn to_symboltrace(backtrace: Backtrace) -> SymbolTrace {
+    // Resolve the backtrace frames to symbols.
+    let backtrace: Backtrace = {
+        let mut backtrace = backtrace::Backtrace::from(backtrace);
+        backtrace.resolve();
+        backtrace.into()
+    };
+
+    // Accumulate the symbols in descending order into `symboltrace`.
+    let mut symboltrace: SymbolTrace = vec![];
+    let mut state = DefaultHasher::new();
+    for frame in backtrace.into_iter().rev() {
+        for symbol in frame.symbols().iter().rev() {
+            let symbol = Symbol {
+                symbol: symbol.clone(),
+                parent_hash: state.finish(),
+            };
+            symbol.hash(&mut state);
+            symboltrace.push(symbol);
+        }
+    }
+
+    symboltrace
+}

tokio/src/runtime/tests/inject.rs

@@ -0,0 +1,38 @@
+use crate::runtime::task::Inject;
+
+#[test]
+fn push_and_pop() {
+    let inject = Inject::new();
+
+    for _ in 0..10 {
+        let (task, _) = super::unowned(async {});
+        inject.push(task);
+    }
+
+    for _ in 0..10 {
+        assert!(inject.pop().is_some());
+    }
+
+    assert!(inject.pop().is_none());
+}
+
+#[test]
+fn push_batch_and_pop() {
+    let inject = Inject::new();
+
+    inject.push_batch((0..10).map(|_| super::unowned(async {}).0));
+
+    assert_eq!(5, inject.pop_n(5).count());
+    assert_eq!(5, inject.pop_n(5).count());
+    assert_eq!(0, inject.pop_n(5).count());
+}
+
+#[test]
+fn pop_n_drains_on_drop() {
+    let inject = Inject::new();
+
+    inject.push_batch((0..10).map(|_| super::unowned(async {}).0));
+    let _ = inject.pop_n(10);
+
+    assert_eq!(inject.len(), 0);
+}

tokio/src/runtime/tests/loom_queue.rs

@@ -5,15 +5,19 @@ use crate::runtime::MetricsBatch;
 
 use loom::thread;
 
+fn metrics_batch() -> MetricsBatch {
+    MetricsBatch::new(&crate::runtime::WorkerMetrics::new())
+}
+
 #[test]
 fn basic() {
     loom::model(|| {
         let (steal, mut local) = queue::local();
         let inject = Inject::new();
-        let mut metrics = MetricsBatch::new();
+        let mut metrics = metrics_batch();
 
         let th = thread::spawn(move || {
-            let mut metrics = MetricsBatch::new();
+            let mut metrics = metrics_batch();
             let (_, mut local) = queue::local();
             let mut n = 0;
 
@@ -35,7 +39,7 @@ fn basic() {
         for _ in 0..2 {
             for _ in 0..2 {
                 let (task, _) = super::unowned(async {});
-                local.push_back(task, &inject, &mut metrics);
+                local.push_back_or_overflow(task, &inject, &mut metrics);
             }
 
             if local.pop().is_some() {
@@ -44,7 +48,7 @@ fn basic() {
 
             // Push another task
             let (task, _) = super::unowned(async {});
-            local.push_back(task, &inject, &mut metrics);
+            local.push_back_or_overflow(task, &inject, &mut metrics);
 
             while local.pop().is_some() {
                 n += 1;
@@ -66,10 +70,10 @@ fn steal_overflow() {
     loom::model(|| {
         let (steal, mut local) = queue::local();
         let inject = Inject::new();
-        let mut metrics = MetricsBatch::new();
+        let mut metrics = metrics_batch();
 
         let th = thread::spawn(move || {
-            let mut metrics = MetricsBatch::new();
+            let mut metrics = metrics_batch();
             let (_, mut local) = queue::local();
             let mut n = 0;
 
@@ -88,7 +92,7 @@ fn steal_overflow() {
 
         // push a task, pop a task
         let (task, _) = super::unowned(async {});
-        local.push_back(task, &inject, &mut metrics);
+        local.push_back_or_overflow(task, &inject, &mut metrics);
 
         if local.pop().is_some() {
             n += 1;
@@ -96,7 +100,7 @@ fn steal_overflow() {
 
         for _ in 0..6 {
             let (task, _) = super::unowned(async {});
-            local.push_back(task, &inject, &mut metrics);
+            local.push_back_or_overflow(task, &inject, &mut metrics);
         }
 
         n += th.join().unwrap();
@@ -118,7 +122,7 @@ fn multi_stealer() {
     const NUM_TASKS: usize = 5;
 
     fn steal_tasks(steal: queue::Steal<NoopSchedule>) -> usize {
-        let mut metrics = MetricsBatch::new();
+        let mut metrics = metrics_batch();
         let (_, mut local) = queue::local();
 
         if steal.steal_into(&mut local, &mut metrics).is_none() {
@@ -137,12 +141,12 @@ fn multi_stealer() {
     loom::model(|| {
         let (steal, mut local) = queue::local();
         let inject = Inject::new();
-        let mut metrics = MetricsBatch::new();
+        let mut metrics = metrics_batch();
 
         // Push work
         for _ in 0..NUM_TASKS {
             let (task, _) = super::unowned(async {});
-            local.push_back(task, &inject, &mut metrics);
+            local.push_back_or_overflow(task, &inject, &mut metrics);
         }
 
         let th1 = {
@@ -172,7 +176,7 @@ fn multi_stealer() {
 #[test]
 fn chained_steal() {
     loom::model(|| {
-        let mut metrics = MetricsBatch::new();
+        let mut metrics = metrics_batch();
         let (s1, mut l1) = queue::local();
         let (s2, mut l2) = queue::local();
         let inject = Inject::new();
@@ -180,15 +184,15 @@ fn chained_steal() {
         // Load up some tasks
         for _ in 0..4 {
             let (task, _) = super::unowned(async {});
-            l1.push_back(task, &inject, &mut metrics);
+            l1.push_back_or_overflow(task, &inject, &mut metrics);
 
             let (task, _) = super::unowned(async {});
-            l2.push_back(task, &inject, &mut metrics);
+            l2.push_back_or_overflow(task, &inject, &mut metrics);
         }
 
         // Spawn a task to steal from **our** queue
         let th = thread::spawn(move || {
-            let mut metrics = MetricsBatch::new();
+            let mut metrics = metrics_batch();
             let (_, mut local) = queue::local();
             s1.steal_into(&mut local, &mut metrics);
 

tokio/src/runtime/tests/mod.rs

@@ -60,9 +60,14 @@ cfg_loom! {
     mod loom_shutdown_join;
     mod loom_join_set;
     mod loom_yield;
+
+    // Make sure debug assertions are enabled
+    #[cfg(not(debug_assertions))]
+    compiler_error!("these tests require debug assertions to be enabled");
 }
 
 cfg_not_loom! {
+    mod inject;
     mod queue;
 
     #[cfg(not(miri))]

tokio/src/runtime/tests/queue.rs

@@ -21,15 +21,20 @@ macro_rules! assert_metrics {
     }};
 }
 
+fn metrics_batch() -> MetricsBatch {
+    use crate::runtime::WorkerMetrics;
+    MetricsBatch::new(&WorkerMetrics::new())
+}
+
 #[test]
-fn fits_256() {
+fn fits_256_one_at_a_time() {
     let (_, mut local) = queue::local();
     let inject = Inject::new();
-    let mut metrics = MetricsBatch::new();
+    let mut metrics = metrics_batch();
 
     for _ in 0..256 {
         let (task, _) = super::unowned(async {});
-        local.push_back(task, &inject, &mut metrics);
+        local.push_back_or_overflow(task, &inject, &mut metrics);
     }
 
     cfg_metrics! {
@@ -41,15 +46,53 @@ fn fits_256() {
     while local.pop().is_some() {}
 }
 
+#[test]
+fn fits_256_all_at_once() {
+    let (_, mut local) = queue::local();
+
+    let mut tasks = (0..256)
+        .map(|_| super::unowned(async {}).0)
+        .collect::<Vec<_>>();
+    local.push_back(tasks.drain(..));
+
+    let mut i = 0;
+    while local.pop().is_some() {
+        i += 1;
+    }
+
+    assert_eq!(i, 256);
+}
+
+#[test]
+fn fits_256_all_in_chunks() {
+    let (_, mut local) = queue::local();
+
+    let mut tasks = (0..256)
+        .map(|_| super::unowned(async {}).0)
+        .collect::<Vec<_>>();
+
+    local.push_back(tasks.drain(..10));
+    local.push_back(tasks.drain(..100));
+    local.push_back(tasks.drain(..46));
+    local.push_back(tasks.drain(..100));
+
+    let mut i = 0;
+    while local.pop().is_some() {
+        i += 1;
+    }
+
+    assert_eq!(i, 256);
+}
+
 #[test]
 fn overflow() {
     let (_, mut local) = queue::local();
     let inject = Inject::new();
-    let mut metrics = MetricsBatch::new();
+    let mut metrics = metrics_batch();
 
     for _ in 0..257 {
         let (task, _) = super::unowned(async {});
-        local.push_back(task, &inject, &mut metrics);
+        local.push_back_or_overflow(task, &inject, &mut metrics);
     }
 
     cfg_metrics! {
@@ -71,7 +114,7 @@ fn overflow() {
 
 #[test]
 fn steal_batch() {
-    let mut metrics = MetricsBatch::new();
+    let mut metrics = metrics_batch();
 
     let (steal1, mut local1) = queue::local();
     let (_, mut local2) = queue::local();
@@ -79,7 +122,7 @@ fn steal_batch() {
 
     for _ in 0..4 {
         let (task, _) = super::unowned(async {});
-        local1.push_back(task, &inject, &mut metrics);
+        local1.push_back_or_overflow(task, &inject, &mut metrics);
     }
 
     assert!(steal1.steal_into(&mut local2, &mut metrics).is_some());
@@ -117,14 +160,14 @@ fn stress1() {
     const NUM_PUSH: usize = normal_or_miri(500, 10);
     const NUM_POP: usize = normal_or_miri(250, 10);
 
-    let mut metrics = MetricsBatch::new();
+    let mut metrics = metrics_batch();
 
     for _ in 0..NUM_ITER {
         let (steal, mut local) = queue::local();
         let inject = Inject::new();
 
         let th = thread::spawn(move || {
-            let mut metrics = MetricsBatch::new();
+            let mut metrics = metrics_batch();
             let (_, mut local) = queue::local();
             let mut n = 0;
 
@@ -152,7 +195,7 @@ fn stress1() {
         for _ in 0..NUM_LOCAL {
             for _ in 0..NUM_PUSH {
                 let (task, _) = super::unowned(async {});
-                local.push_back(task, &inject, &mut metrics);
+                local.push_back_or_overflow(task, &inject, &mut metrics);
             }
 
             for _ in 0..NUM_POP {
@@ -180,14 +223,14 @@ fn stress2() {
     const NUM_TASKS: usize = normal_or_miri(1_000_000, 50);
     const NUM_STEAL: usize = normal_or_miri(1_000, 10);
 
-    let mut metrics = MetricsBatch::new();
+    let mut metrics = metrics_batch();
 
     for _ in 0..NUM_ITER {
         let (steal, mut local) = queue::local();
         let inject = Inject::new();
 
         let th = thread::spawn(move || {
-            let mut stats = MetricsBatch::new();
+            let mut stats = metrics_batch();
             let (_, mut local) = queue::local();
             let mut n = 0;
 
@@ -210,7 +253,7 @@ fn stress2() {
 
         for i in 0..NUM_TASKS {
             let (task, _) = super::unowned(async {});
-            local.push_back(task, &inject, &mut metrics);
+            local.push_back_or_overflow(task, &inject, &mut metrics);
 
             if i % 128 == 0 && local.pop().is_some() {
                 num_pop += 1;

tokio/src/sync/broadcast.rs

@@ -1194,6 +1194,33 @@ impl<T: Clone> Receiver<T> {
         let guard = self.recv_ref(None)?;
         guard.clone_value().ok_or(TryRecvError::Closed)
     }
+
+    /// Blocking receive to call outside of asynchronous contexts.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if called within an asynchronous execution
+    /// context.
+    ///
+    /// # Examples
+    /// ```
+    /// use std::thread;
+    /// use tokio::sync::broadcast;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     let (tx, mut rx) = broadcast::channel(16);
+    ///
+    ///     let sync_code = thread::spawn(move || {
+    ///         assert_eq!(rx.blocking_recv(), Ok(10));
+    ///     });
+    ///
+    ///     let _ = tx.send(10);
+    ///     sync_code.join().unwrap();
+    /// }
+    pub fn blocking_recv(&mut self) -> Result<T, RecvError> {
+        crate::future::block_on(self.recv())
+    }
 }
 
 impl<T> Drop for Receiver<T> {

tokio/src/sync/mpsc/bounded.rs

@@ -18,7 +18,7 @@ use std::task::{Context, Poll};
 /// To convert the `Sender` into a `Sink` or use it in a poll function, you can
 /// use the [`PollSender`] utility.
 ///
-/// [`PollSender`]: https://docs.rs/tokio-util/0.6/tokio_util/sync/struct.PollSender.html
+/// [`PollSender`]: https://docs.rs/tokio-util/latest/tokio_util/sync/struct.PollSender.html
 pub struct Sender<T> {
     chan: chan::Tx<T, Semaphore>,
 }

tokio/src/sync/mpsc/error.rs

@@ -4,22 +4,28 @@ use std::error::Error;
 use std::fmt;
 
 /// Error returned by the `Sender`.
-#[derive(Debug)]
+#[derive(PartialEq, Eq, Clone, Copy)]
 pub struct SendError<T>(pub T);
 
+impl<T> fmt::Debug for SendError<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SendError").finish_non_exhaustive()
+    }
+}
+
 impl<T> fmt::Display for SendError<T> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(fmt, "channel closed")
     }
 }
 
-impl<T: fmt::Debug> std::error::Error for SendError<T> {}
+impl<T> std::error::Error for SendError<T> {}
 
 // ===== TrySendError =====
 
 /// This enumeration is the list of the possible error outcomes for the
 /// [try_send](super::Sender::try_send) method.
-#[derive(Debug, Eq, PartialEq)]
+#[derive(PartialEq, Eq, Clone, Copy)]
 pub enum TrySendError<T> {
     /// The data could not be sent on the channel because the channel is
     /// currently full and sending would require blocking.
@@ -30,7 +36,14 @@ pub enum TrySendError<T> {
     Closed(T),
 }
 
-impl<T: fmt::Debug> Error for TrySendError<T> {}
+impl<T> fmt::Debug for TrySendError<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match *self {
+            TrySendError::Full(..) => "Full(..)".fmt(f),
+            TrySendError::Closed(..) => "Closed(..)".fmt(f),
+        }
+    }
+}
 
 impl<T> fmt::Display for TrySendError<T> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -45,6 +58,8 @@ impl<T> fmt::Display for TrySendError<T> {
     }
 }
 
+impl<T> Error for TrySendError<T> {}
+
 impl<T> From<SendError<T>> for TrySendError<T> {
     fn from(src: SendError<T>) -> TrySendError<T> {
         TrySendError::Closed(src.0)
@@ -96,7 +111,7 @@ impl Error for RecvError {}
 cfg_time! {
     // ===== SendTimeoutError =====
 
-    #[derive(Debug, Eq, PartialEq)]
+    #[derive(PartialEq, Eq, Clone, Copy)]
     /// Error returned by [`Sender::send_timeout`](super::Sender::send_timeout)].
     pub enum SendTimeoutError<T> {
         /// The data could not be sent on the channel because the channel is
@@ -108,7 +123,14 @@ cfg_time! {
         Closed(T),
     }
 
-    impl<T: fmt::Debug> Error for SendTimeoutError<T> {}
+    impl<T> fmt::Debug for SendTimeoutError<T> {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            match *self {
+                SendTimeoutError::Timeout(..) => "Timeout(..)".fmt(f),
+                SendTimeoutError::Closed(..) => "Closed(..)".fmt(f),
+            }
+        }
+    }
 
     impl<T> fmt::Display for SendTimeoutError<T> {
         fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -122,4 +144,6 @@ cfg_time! {
             )
         }
     }
+
+    impl<T> Error for SendTimeoutError<T> {}
 }

tokio/src/sync/semaphore.rs

@@ -73,7 +73,7 @@ use std::sync::Arc;
 /// }
 /// ```
 ///
-/// [`PollSemaphore`]: https://docs.rs/tokio-util/0.6/tokio_util/sync/struct.PollSemaphore.html
+/// [`PollSemaphore`]: https://docs.rs/tokio-util/latest/tokio_util/sync/struct.PollSemaphore.html
 /// [`Semaphore::acquire_owned`]: crate::sync::Semaphore::acquire_owned
 #[derive(Debug)]
 pub struct Semaphore {

tokio/src/sync/watch.rs

@@ -208,18 +208,24 @@ pub mod error {
     use std::fmt;
 
     /// Error produced when sending a value fails.
-    #[derive(Debug)]
+    #[derive(PartialEq, Eq, Clone, Copy)]
     pub struct SendError<T>(pub T);
 
     // ===== impl SendError =====
 
-    impl<T: fmt::Debug> fmt::Display for SendError<T> {
+    impl<T> fmt::Debug for SendError<T> {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            f.debug_struct("SendError").finish_non_exhaustive()
+        }
+    }
+
+    impl<T> fmt::Display for SendError<T> {
         fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
             write!(fmt, "channel closed")
         }
     }
 
-    impl<T: fmt::Debug> std::error::Error for SendError<T> {}
+    impl<T> std::error::Error for SendError<T> {}
 
     /// Error produced when receiving a change notification.
     #[derive(Debug, Clone)]

tokio/src/task/blocking.rs

@@ -146,7 +146,7 @@ cfg_rt! {
     /// [blocking]: ../index.html#cpu-bound-tasks-and-blocking-code
     /// [rayon]: https://docs.rs/rayon
     /// [`mpsc channel`]: crate::sync::mpsc
-    /// [`SyncIoBridge`]: https://docs.rs/tokio-util/0.6/tokio_util/io/struct.SyncIoBridge.html
+    /// [`SyncIoBridge`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.SyncIoBridge.html
     /// [hyper]: https://docs.rs/hyper
     /// [`thread::spawn`]: fn@std::thread::spawn
     /// [`shutdown_timeout`]: fn@crate::runtime::Runtime::shutdown_timeout

tokio/src/task/builder.rs

@@ -80,7 +80,7 @@ impl<'a> Builder<'a> {
     ///
     /// This method panics if called outside of a Tokio runtime.
     ///
-    /// See [`task::spawn`](crate::task::spawn) for
+    /// See [`task::spawn`](crate::task::spawn()) for
     /// more details.
     #[track_caller]
     pub fn spawn<Fut>(self, future: Fut) -> io::Result<JoinHandle<Fut::Output>>

tokio/src/task/spawn.rs

@@ -179,6 +179,18 @@ cfg_rt! {
         T::Output: Send + 'static,
     {
         use crate::runtime::task;
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(
+                target_arch = "aarch64",
+                target_arch = "x86",
+                target_arch = "x86_64"
+            )
+        ))]
+        let future = task::trace::Trace::root(future);
         let id = task::Id::next();
         let task = crate::util::trace::task(future, "task", name, id.as_u64());
         let handle = Handle::current();

tokio/src/task/yield_now.rs

@@ -46,6 +46,8 @@ pub async fn yield_now() {
         type Output = ();
 
         fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
+            ready!(crate::trace::trace_leaf(cx));
+
             if self.yielded {
                 return Poll::Ready(());
             }
@@ -53,7 +55,7 @@ pub async fn yield_now() {
             self.yielded = true;
 
             let defer = context::with_defer(|rt| {
-                rt.defer(cx.waker().clone());
+                rt.defer(cx.waker());
             });
 
             if defer.is_none() {

tokio/src/time/interval.rs

@@ -387,7 +387,7 @@ impl Default for MissedTickBehavior {
 /// An `Interval` can be turned into a `Stream` with [`IntervalStream`].
 ///
 /// [`IntervalStream`]: https://docs.rs/tokio-stream/latest/tokio_stream/wrappers/struct.IntervalStream.html
-/// [`sleep`]: crate::time::sleep
+/// [`sleep`]: crate::time::sleep()
 #[derive(Debug)]
 pub struct Interval {
     /// Future that completes the next time the `Interval` yields a value.

tokio/src/util/idle_notified_set.rs

@@ -421,7 +421,7 @@ impl<T: 'static> Wake for ListEntry<T> {
             // We move ourself to the notified list.
             let me = unsafe {
                 // Safety: We just checked that we are in this particular list.
-                lock.idle.remove(NonNull::from(&**me)).unwrap()
+                lock.idle.remove(ListEntry::as_raw(me)).unwrap()
             };
             lock.notified.push_front(me);
 
@@ -460,3 +460,22 @@ unsafe impl<T> linked_list::Link for ListEntry<T> {
         ListEntry::addr_of_pointers(target)
     }
 }
+
+#[cfg(all(test, not(loom)))]
+mod tests {
+    use crate::runtime::Builder;
+    use crate::task::JoinSet;
+
+    // A test that runs under miri.
+    //
+    // https://github.com/tokio-rs/tokio/pull/5693
+    #[test]
+    fn join_set_test() {
+        let rt = Builder::new_current_thread().build().unwrap();
+
+        let mut set = JoinSet::new();
+        set.spawn_on(futures::future::ready(()), rt.handle());
+
+        rt.block_on(set.join_next()).unwrap().unwrap();
+    }
+}

tokio/src/util/linked_list.rs

@@ -228,6 +228,53 @@ impl<L: Link> fmt::Debug for LinkedList<L, L::Target> {
     }
 }
 
+// ===== impl CountedLinkedList ====
+
+// Delegates operations to the base LinkedList implementation, and adds a counter to the elements
+// in the list.
+pub(crate) struct CountedLinkedList<L: Link, T> {
+    list: LinkedList<L, T>,
+    count: usize,
+}
+
+impl<L: Link> CountedLinkedList<L, L::Target> {
+    pub(crate) fn new() -> CountedLinkedList<L, L::Target> {
+        CountedLinkedList {
+            list: LinkedList::new(),
+            count: 0,
+        }
+    }
+
+    pub(crate) fn push_front(&mut self, val: L::Handle) {
+        self.list.push_front(val);
+        self.count += 1;
+    }
+
+    pub(crate) fn pop_back(&mut self) -> Option<L::Handle> {
+        let val = self.list.pop_back();
+        if val.is_some() {
+            self.count -= 1;
+        }
+        val
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.list.is_empty()
+    }
+
+    pub(crate) unsafe fn remove(&mut self, node: NonNull<L::Target>) -> Option<L::Handle> {
+        let val = self.list.remove(node);
+        if val.is_some() {
+            self.count -= 1;
+        }
+        val
+    }
+
+    pub(crate) fn count(&self) -> usize {
+        self.count
+    }
+}
+
 #[cfg(any(
     feature = "fs",
     feature = "rt",
@@ -294,6 +341,36 @@ cfg_io_readiness! {
     }
 }
 
+cfg_taskdump! {
+    impl<T: Link> CountedLinkedList<T, T::Target> {
+        pub(crate) fn for_each<F>(&mut self, f: F)
+        where
+            F: FnMut(&T::Handle),
+        {
+            self.list.for_each(f)
+        }
+    }
+
+    impl<T: Link> LinkedList<T, T::Target> {
+        pub(crate) fn for_each<F>(&mut self, mut f: F)
+        where
+            F: FnMut(&T::Handle),
+        {
+            use std::mem::ManuallyDrop;
+
+            let mut next = self.head;
+
+            while let Some(curr) = next {
+                unsafe {
+                    let handle = ManuallyDrop::new(T::from_raw(curr));
+                    f(&handle);
+                    next = T::pointers(curr).as_ref().get_next();
+                }
+            }
+        }
+    }
+}
+
 // ===== impl GuardedLinkedList =====
 
 feature! {
@@ -719,6 +796,26 @@ pub(crate) mod tests {
         }
     }
 
+    #[test]
+    fn count() {
+        let mut list = CountedLinkedList::<&Entry, <&Entry as Link>::Target>::new();
+        assert_eq!(0, list.count());
+
+        let a = entry(5);
+        let b = entry(7);
+        list.push_front(a.as_ref());
+        list.push_front(b.as_ref());
+        assert_eq!(2, list.count());
+
+        list.pop_back();
+        assert_eq!(1, list.count());
+
+        unsafe {
+            list.remove(ptr(&b));
+        }
+        assert_eq!(0, list.count());
+    }
+
     /// This is a fuzz test. You run it by entering `cargo fuzz run fuzz_linked_list` in CLI in `/tokio/` module.
     #[cfg(fuzzing)]
     pub fn fuzz_linked_list(ops: &[u8]) {

tokio/tests/dump_current_thread.rs

@@ -0,0 +1,55 @@
+#![cfg(all(
+    tokio_unstable,
+    tokio_taskdump,
+    target_os = "linux",
+    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+))]
+
+use std::hint::black_box;
+use tokio::runtime;
+
+#[inline(never)]
+async fn a() {
+    black_box(b()).await
+}
+
+#[inline(never)]
+async fn b() {
+    black_box(c()).await
+}
+
+#[inline(never)]
+async fn c() {
+    black_box(tokio::task::yield_now()).await
+}
+
+#[test]
+fn test() {
+    let rt = runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    rt.spawn(a());
+
+    let handle = rt.handle();
+
+    assert_eq!(handle.dump().tasks().iter().count(), 0);
+
+    let dump = rt.block_on(async {
+        handle.spawn(a());
+        handle.dump()
+    });
+
+    let tasks: Vec<_> = dump.tasks().iter().collect();
+
+    assert_eq!(tasks.len(), 2);
+
+    for task in tasks {
+        let trace = task.trace().to_string();
+        assert!(trace.contains("dump_current_thread::a"));
+        assert!(trace.contains("dump_current_thread::b"));
+        assert!(trace.contains("dump_current_thread::c"));
+        assert!(trace.contains("tokio::task::yield_now"));
+    }
+}

tokio/tests/rt_common.rs

@@ -1317,4 +1317,39 @@ rt_test! {
             }
         });
     }
+
+    #[test]
+    #[cfg(not(target_os="wasi"))]
+    fn shutdown_concurrent_spawn() {
+        const NUM_TASKS: usize = 10_000;
+        for _ in 0..5 {
+            let (tx, rx) = std::sync::mpsc::channel();
+            let rt = rt();
+
+            let mut txs = vec![];
+
+            for _ in 0..NUM_TASKS {
+                let (tx, rx) = tokio::sync::oneshot::channel();
+                txs.push(tx);
+                rt.spawn(async move {
+                    rx.await.unwrap();
+                });
+            }
+
+            // Prime the tasks
+            rt.block_on(async { tokio::task::yield_now().await });
+
+            let th = std::thread::spawn(move || {
+                tx.send(()).unwrap();
+                for tx in txs.drain(..) {
+                    let _ = tx.send(());
+                }
+            });
+
+            rx.recv().unwrap();
+            drop(rt);
+
+            th.join().unwrap();
+        }
+    }
 }

tokio/tests/rt_metrics.rs

@@ -81,6 +81,23 @@ fn blocking_queue_depth() {
     assert_eq!(0, rt.metrics().blocking_queue_depth());
 }
 
+#[test]
+fn active_tasks_count() {
+    let rt = current_thread();
+    let metrics = rt.metrics();
+    assert_eq!(0, metrics.active_tasks_count());
+    rt.spawn(async move {
+        assert_eq!(1, metrics.active_tasks_count());
+    });
+
+    let rt = threaded();
+    let metrics = rt.metrics();
+    assert_eq!(0, metrics.active_tasks_count());
+    rt.spawn(async move {
+        assert_eq!(1, metrics.active_tasks_count());
+    });
+}
+
 #[test]
 fn remote_schedule_count() {
     use std::thread;
@@ -211,6 +228,12 @@ fn worker_poll_count() {
     drop(rt);
     assert_eq!(N, metrics.worker_poll_count(0));
 
+    // Does not populate the histogram
+    assert!(!metrics.poll_count_histogram_enabled());
+    for i in 0..10 {
+        assert_eq!(0, metrics.poll_count_histogram_bucket_count(0, i));
+    }
+
     let rt = threaded();
     let metrics = rt.metrics();
     rt.block_on(async {
@@ -225,6 +248,126 @@ fn worker_poll_count() {
         .sum();
 
     assert_eq!(N, n);
+
+    // Does not populate the histogram
+    assert!(!metrics.poll_count_histogram_enabled());
+    for n in 0..metrics.num_workers() {
+        for i in 0..10 {
+            assert_eq!(0, metrics.poll_count_histogram_bucket_count(n, i));
+        }
+    }
+}
+
+#[test]
+fn worker_poll_count_histogram() {
+    const N: u64 = 5;
+
+    let rts = [
+        tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .enable_metrics_poll_count_histogram()
+            .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear)
+            .metrics_poll_count_histogram_buckets(3)
+            .metrics_poll_count_histogram_resolution(Duration::from_millis(50))
+            .build()
+            .unwrap(),
+        tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .enable_all()
+            .enable_metrics_poll_count_histogram()
+            .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear)
+            .metrics_poll_count_histogram_buckets(3)
+            .metrics_poll_count_histogram_resolution(Duration::from_millis(50))
+            .build()
+            .unwrap(),
+    ];
+
+    for rt in rts {
+        let metrics = rt.metrics();
+        rt.block_on(async {
+            for _ in 0..N {
+                tokio::spawn(async {}).await.unwrap();
+            }
+        });
+        drop(rt);
+
+        let num_workers = metrics.num_workers();
+        let num_buckets = metrics.poll_count_histogram_num_buckets();
+
+        assert!(metrics.poll_count_histogram_enabled());
+        assert_eq!(num_buckets, 3);
+
+        let n = (0..num_workers)
+            .flat_map(|i| (0..num_buckets).map(move |j| (i, j)))
+            .map(|(worker, bucket)| metrics.poll_count_histogram_bucket_count(worker, bucket))
+            .sum();
+        assert_eq!(N, n);
+    }
+}
+
+#[test]
+fn worker_poll_count_histogram_range() {
+    let max = Duration::from_nanos(u64::MAX);
+
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .enable_metrics_poll_count_histogram()
+        .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear)
+        .metrics_poll_count_histogram_buckets(3)
+        .metrics_poll_count_histogram_resolution(us(50))
+        .build()
+        .unwrap();
+    let metrics = rt.metrics();
+
+    assert_eq!(metrics.poll_count_histogram_bucket_range(0), us(0)..us(50));
+    assert_eq!(
+        metrics.poll_count_histogram_bucket_range(1),
+        us(50)..us(100)
+    );
+    assert_eq!(metrics.poll_count_histogram_bucket_range(2), us(100)..max);
+
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .enable_metrics_poll_count_histogram()
+        .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Log)
+        .metrics_poll_count_histogram_buckets(3)
+        .metrics_poll_count_histogram_resolution(us(50))
+        .build()
+        .unwrap();
+    let metrics = rt.metrics();
+
+    let a = Duration::from_nanos(50000_u64.next_power_of_two());
+    let b = a * 2;
+
+    assert_eq!(metrics.poll_count_histogram_bucket_range(0), us(0)..a);
+    assert_eq!(metrics.poll_count_histogram_bucket_range(1), a..b);
+    assert_eq!(metrics.poll_count_histogram_bucket_range(2), b..max);
+}
+
+#[test]
+fn worker_poll_count_histogram_disabled_without_explicit_enable() {
+    let rts = [
+        tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear)
+            .metrics_poll_count_histogram_buckets(3)
+            .metrics_poll_count_histogram_resolution(Duration::from_millis(50))
+            .build()
+            .unwrap(),
+        tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .enable_all()
+            .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear)
+            .metrics_poll_count_histogram_buckets(3)
+            .metrics_poll_count_histogram_resolution(Duration::from_millis(50))
+            .build()
+            .unwrap(),
+    ];
+
+    for rt in rts {
+        let metrics = rt.metrics();
+        assert!(!metrics.poll_count_histogram_enabled());
+    }
 }
 
 #[test]
@@ -367,10 +510,20 @@ fn injection_queue_depth() {
     // First we need to block the runtime workers
     let (tx1, rx1) = std::sync::mpsc::channel();
     let (tx2, rx2) = std::sync::mpsc::channel();
+    let (tx3, rx3) = std::sync::mpsc::channel();
+    let rx3 = Arc::new(Mutex::new(rx3));
 
     rt.spawn(async move { rx1.recv().unwrap() });
     rt.spawn(async move { rx2.recv().unwrap() });
 
+    // Spawn some more to make sure there are items
+    for _ in 0..10 {
+        let rx = rx3.clone();
+        rt.spawn(async move {
+            rx.lock().unwrap().recv().unwrap();
+        });
+    }
+
     thread::spawn(move || {
         handle.spawn(async {});
     })
@@ -379,7 +532,11 @@ fn injection_queue_depth() {
 
     let n = metrics.injection_queue_depth();
     assert!(1 <= n, "{}", n);
-    assert!(3 >= n, "{}", n);
+    assert!(15 >= n, "{}", n);
+
+    for _ in 0..10 {
+        tx3.send(()).unwrap();
+    }
 
     tx1.send(()).unwrap();
     tx2.send(()).unwrap();
@@ -555,3 +712,7 @@ fn threaded() -> Runtime {
         .build()
         .unwrap()
 }
+
+fn us(n: u64) -> Duration {
+    Duration::from_micros(n)
+}

tokio/tests/rt_threaded.rs

@@ -30,7 +30,8 @@ fn single_thread() {
     let _ = runtime::Builder::new_multi_thread()
         .enable_all()
         .worker_threads(1)
-        .build();
+        .build()
+        .unwrap();
 }
 
 #[test]
@@ -160,6 +161,32 @@ fn many_multishot_futures() {
     }
 }
 
+#[test]
+fn lifo_slot_budget() {
+    async fn my_fn() {
+        spawn_another();
+    }
+
+    fn spawn_another() {
+        tokio::spawn(my_fn());
+    }
+
+    let rt = runtime::Builder::new_multi_thread()
+        .enable_all()
+        .worker_threads(1)
+        .build()
+        .unwrap();
+
+    let (send, recv) = oneshot::channel();
+
+    rt.spawn(async move {
+        tokio::spawn(my_fn());
+        let _ = send.send(());
+    });
+
+    let _ = rt.block_on(recv);
+}
+
 #[test]
 fn spawn_shutdown() {
     let rt = rt();