wip

This change pads the task struct to avoid false sharing. It is possible
for these structs to overlap cache lines without this alignment.

.cargo/config

@@ -1,2 +0,0 @@
-# [build]
-# rustflags = ["--cfg", "tokio_unstable"]

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ env:
   RUST_BACKTRACE: 1
   # Change to specific Rust release to pin
   rust_stable: stable
-  rust_nightly: nightly-2022-11-03
+  rust_nightly: nightly-2023-05-18
   rust_clippy: 1.65.0
   # When updating this, also update:
   # - README.md
@@ -42,7 +42,6 @@ jobs:
       - test-unstable
       - miri
       - asan
-      - semver
       - cross-check
       - cross-test
       - no-atomic-u64
@@ -56,9 +55,12 @@ jobs:
       - check-readme
       - test-hyper
       - x86_64-fortanix-unknown-sgx
+      - check-redox
       - wasm32-unknown-unknown
       - wasm32-wasi
       - check-external-types
+      - check-fuzzing
+      - check-unstable-mt-counters
     steps:
       - run: exit 0
 
@@ -186,10 +188,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os:
-          - windows-latest
-          - ubuntu-latest
-          - macos-latest
+        include:
+          - os: windows-latest
+          - os: ubuntu-latest
+          - os: macos-latest
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
@@ -207,6 +209,56 @@ jobs:
           # the unstable cfg to RustDoc
           RUSTDOCFLAGS: --cfg tokio_unstable
 
+  test-unstable-taskdump:
+    name: test tokio full --unstable --taskdump
+    needs: basics
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      # Run `tokio` with "unstable" and "taskdump" cfg flags.
+      - name: test tokio full --cfg unstable --cfg taskdump
+        run: cargo test --all-features
+        working-directory: tokio
+        env:
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
+          # in order to run doctests for unstable features, we must also pass
+          # the unstable cfg to RustDoc
+          RUSTDOCFLAGS: --cfg tokio_unstable --cfg tokio_taskdump
+
+  check-unstable-mt-counters:
+    name: check tokio full --internal-mt-counters
+    needs: basics
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      # Run `tokio` with "unstable" and "taskdump" cfg flags.
+      - name: check tokio full --cfg unstable --cfg internal-mt-counters
+        run: cargo test --all-features
+        working-directory: tokio
+        env:
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_internal_mt_counters -Dwarnings
+          # in order to run doctests for unstable features, we must also pass
+          # the unstable cfg to RustDoc
+          RUSTDOCFLAGS: --cfg tokio_unstable --cfg tokio_internal_mt_counters
+
   miri:
     name: miri
     needs: basics
@@ -242,23 +294,25 @@ jobs:
           toolchain: ${{ env.rust_nightly }}
       - uses: Swatinem/rust-cache@v2
       - name: asan
-        run: cargo test --workspace --all-features --target x86_64-unknown-linux-gnu --tests -- --test-threads 1
+        run: cargo test --workspace --all-features --target x86_64-unknown-linux-gnu --tests -- --test-threads 1 --nocapture
         env:
-          RUSTFLAGS: -Z sanitizer=address
+          RUSTFLAGS: -Z sanitizer=address --cfg tokio_no_tuning_tests
           # Ignore `trybuild` errors as they are irrelevant and flaky on nightly
           TRYBUILD: overwrite
 
-  semver:
-    name: semver
-    needs: basics
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check semver
-        uses: obi1kenobi/cargo-semver-checks-action@v2
-        with:
-          rust-toolchain: ${{ env.rust_stable }}
-          release-type: minor
+  # Re-enable this after the next release.
+  #
+  #semver:
+  #  name: semver
+  #  needs: basics
+  #  runs-on: ubuntu-latest
+  #  steps:
+  #    - uses: actions/checkout@v3
+  #    - name: Check semver
+  #      uses: obi1kenobi/cargo-semver-checks-action@v2
+  #      with:
+  #        rust-toolchain: ${{ env.rust_stable }}
+  #        release-type: minor
 
   cross-check:
     name: cross-check
@@ -293,9 +347,11 @@ jobs:
       matrix:
         include:
           - target: i686-unknown-linux-gnu
+            rustflags: --cfg tokio_taskdump
           - target: arm-unknown-linux-gnueabihf
           - target: armv7-unknown-linux-gnueabihf
           - target: aarch64-unknown-linux-gnu
+            rustflags: --cfg tokio_taskdump
 
           # Run a platform without AtomicU64 and no const Mutex::new
           - target: arm-unknown-linux-gnueabihf
@@ -312,14 +368,14 @@ jobs:
       # First run with all features (including parking_lot)
       - run: cross test -p tokio --all-features --target ${{ matrix.target }} --tests
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 ${{ matrix.rustflags }}
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_tuning_tests ${{ matrix.rustflags }}
       # Now run without parking_lot
       - name: Remove `parking_lot` from `full` feature
         run: sed -i '0,/parking_lot/{/parking_lot/d;}' tokio/Cargo.toml
       # The `tokio_no_parking_lot` cfg is here to ensure the `sed` above does not silently break.
       - run: cross test -p tokio --features full,test-util --target ${{ matrix.target }} --tests
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_parking_lot ${{ matrix.rustflags }}
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_parking_lot --cfg tokio_no_tuning_tests ${{ matrix.rustflags }}
 
   # See https://github.com/tokio-rs/tokio/issues/5187
   no-atomic-u64:
@@ -339,17 +395,17 @@ jobs:
       - uses: taiki-e/setup-cross-toolchain-action@v1
         with:
           target: i686-unknown-linux-gnu
-      - run: cargo test -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features
+      - run: cargo test -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features -- --test-threads 1 --nocapture
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64
       # https://github.com/tokio-rs/tokio/pull/5356
       # https://github.com/tokio-rs/tokio/issues/5373
-      - run: cargo hack build -p tokio --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
+      - run: cargo hack build -p tokio --feature-powerset --depth 2 --keep-going
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64 --cfg tokio_no_const_mutex_new
-      - run: cargo hack build -p tokio --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64 --cfg tokio_no_const_mutex_new
+      - run: cargo hack build -p tokio --feature-powerset --depth 2 --keep-going
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64
 
   features:
     name: features
@@ -366,12 +422,17 @@ jobs:
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
       - name: check --feature-powerset
-        run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
+        run: cargo hack check --all --feature-powerset --depth 2 --keep-going
       # Try with unstable feature flags
       - name: check --feature-powerset --unstable
-        run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going
+        run: cargo hack check --all --feature-powerset --depth 2 --keep-going
         env:
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+      # Try with unstable and taskdump feature flags
+      - name: check --feature-powerset --unstable --taskdump
+        run: cargo hack check --all --feature-powerset --depth 2 --keep-going
+        env:
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
 
   minrust:
     name: minrust
@@ -411,7 +472,7 @@ jobs:
           cargo hack check --all-features --ignore-private
       - name: "check --all-features --unstable -Z minimal-versions"
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
         run: |
           # Remove dev-dependencies from Cargo.toml to prevent the next `cargo update`
           # from determining minimal versions based on dev-dependencies.
@@ -435,8 +496,8 @@ jobs:
       - name: "rustfmt --check"
         # Workaround for rust-lang/cargo#7732
         run: |
-          if ! rustfmt --check --edition 2018 $(git ls-files '*.rs'); then
-            printf "Please run \`rustfmt --edition 2018 \$(git ls-files '*.rs')\` to fix rustfmt errors.\nSee CONTRIBUTING.md for more details.\n" >&2
+          if ! rustfmt --check --edition 2021 $(git ls-files '*.rs'); then
+            printf "Please run \`rustfmt --edition 2021 \$(git ls-files '*.rs')\` to fix rustfmt errors.\nSee CONTRIBUTING.md for more details.\n" >&2
             exit 1
           fi
 
@@ -468,8 +529,8 @@ jobs:
       - name: "doc --lib --all-features"
         run: cargo doc --lib --no-deps --all-features --document-private-items
         env:
-          RUSTFLAGS: --cfg docsrs --cfg tokio_unstable
-          RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable -Dwarnings
+          RUSTFLAGS: --cfg docsrs --cfg tokio_unstable --cfg tokio_taskdump
+          RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
 
   loom-compile:
     name: build loom tests
@@ -551,6 +612,21 @@ jobs:
         run: cargo build --target x86_64-fortanix-unknown-sgx --features rt,sync
         working-directory: tokio
 
+  check-redox:
+    name: build tokio for redox-os
+    needs: basics
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.rust_nightly }}
+          target: x86_64-unknown-redox
+      - name: check tokio on redox
+        run: cargo check --target x86_64-unknown-redox --all-features
+        working-directory: tokio
+
   wasm32-unknown-unknown:
     name: test tokio for wasm32-unknown-unknown
     needs: basics
@@ -617,7 +693,7 @@ jobs:
         working-directory: tests-integration
 
   check-external-types:
-    name: check-external-types
+    name: check-external-types (${{ matrix.os }})
     needs: basics
     runs-on: ${{ matrix.os }}
     strategy:
@@ -625,18 +701,41 @@ jobs:
         os:
           - windows-latest
           - ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install Rust nightly-2022-11-16
-        uses: dtolnay/rust-toolchain@master
-        with:
+        rust:
           # `check-external-types` requires a specific Rust nightly version. See
           # the README for details: https://github.com/awslabs/cargo-check-external-types
-          toolchain: nightly-2022-11-16
+          - nightly-2023-05-31
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ matrix.rust }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ matrix.rust }}
       - uses: Swatinem/rust-cache@v2
+      - name: Install cargo-check-external-types
+        uses: taiki-e/cache-cargo-install-action@v1
+        with:
+          tool: [email protected]
       - name: check-external-types
-        run: |
-          set -x
-          cargo install cargo-check-external-types --locked --version 0.1.6
-          cargo check-external-types --all-features --config external-types.toml
+        run: cargo check-external-types --all-features --config external-types.toml
         working-directory: tokio
+
+  check-fuzzing:
+    name: check-fuzzing
+    needs: basics
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.rust_nightly }}
+      - uses: Swatinem/rust-cache@v2
+      - name: Install cargo-fuzz
+        run: cargo install cargo-fuzz
+      - name: Check /tokio/
+        run: cargo fuzz check --all-features
+        working-directory: tokio
+      - name: Check /tokio-stream/
+        run: cargo fuzz check --all-features
+        working-directory: tokio-stream

.github/workflows/loom.yml

@@ -24,13 +24,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        scope:
-          - --skip loom_pool
-          - loom_pool::group_a
-          - loom_pool::group_b
-          - loom_pool::group_c
-          - loom_pool::group_d
-          - time::driver
+        include:
+          - scope: --skip loom_pool
+            max_preemptions: 2
+          - scope: loom_pool::group_a
+            max_preemptions: 2
+          - scope: loom_pool::group_b
+            max_preemptions: 2
+          - scope: loom_pool::group_c
+            max_preemptions: 2
+          - scope: loom_pool::group_d
+            max_preemptions: 2
+          - scope: time::driver
+            max_preemptions: 2
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
@@ -42,7 +48,7 @@ jobs:
         run: cargo test --lib --release --features full -- --nocapture $SCOPE
         working-directory: tokio
         env:
-          RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings
-          LOOM_MAX_PREEMPTIONS: 2
+          RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings -C debug-assertions
+          LOOM_MAX_PREEMPTIONS: ${{ matrix.max_preemptions }}
           LOOM_MAX_BRANCHES: 10000
           SCOPE: ${{ matrix.scope }}

.gitignore

@@ -2,3 +2,4 @@ target
 Cargo.lock
 
 .cargo/config.toml
+.cargo/config

CONTRIBUTING.md

@@ -173,10 +173,10 @@ command below instead:
 
 ```
 # Mac or Linux
-rustfmt --check --edition 2018 $(git ls-files '*.rs')
+rustfmt --check --edition 2021 $(git ls-files '*.rs')
 
 # Powershell
-Get-ChildItem . -Filter "*.rs" -Recurse | foreach { rustfmt --check --edition 2018 $_.FullName }
+Get-ChildItem . -Filter "*.rs" -Recurse | foreach { rustfmt --check --edition 2021 $_.FullName }
 ```
 The `--check` argument prints the things that need to be fixed. If you remove
 it, `rustfmt` will update your files locally instead.
@@ -230,7 +230,7 @@ integration tests in the crate and follow the style.
 
 Some of our crates include a set of fuzz tests, this will be marked by a
 directory `fuzz`. It is a good idea to run fuzz tests after each change.
-To get started with fuzz testing you'll need to install 
+To get started with fuzz testing you'll need to install
 [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz).
 
 `cargo install cargo-fuzz`
@@ -678,4 +678,4 @@ When releasing a new version of a crate, follow these steps:
 [unit-tests]: https://doc.rust-lang.org/rust-by-example/testing/unit_testing.html
 [integration-tests]: https://doc.rust-lang.org/rust-by-example/testing/integration_testing.html
 [documentation-tests]: https://doc.rust-lang.org/rust-by-example/testing/doc_testing.html
-[conditional-compilation]: https://doc.rust-lang.org/reference/conditional-compilation.html
+[conditional-compilation]: https://doc.rust-lang.org/reference/conditional-compilation.html

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-
+resolver = "2"
 members = [
   "tokio",
   "tokio-macros",
@@ -14,3 +14,6 @@ members = [
   "tests-build",
   "tests-integration",
 ]
+
+[patch.crates-io]
+loom = { git = "https://github.com/tokio-rs/loom", branch = "increase-max-threads" }

benches/Cargo.toml

@@ -2,7 +2,7 @@
 name = "benches"
 version = "0.0.0"
 publish = false
-edition = "2018"
+edition = "2021"
 
 [features]
 test-util = ["tokio/test-util"]
@@ -40,6 +40,11 @@ name = "sync_watch"
 path = "sync_watch.rs"
 harness = false
 
+[[bench]]
+name = "rt_current_thread"
+path = "rt_current_thread.rs"
+harness = false
+
 [[bench]]
 name = "rt_multi_threaded"
 path = "rt_multi_threaded.rs"

benches/rt_current_thread.rs

@@ -0,0 +1,83 @@
+//! Benchmark implementation details of the threaded scheduler. These benches are
+//! intended to be used as a form of regression testing and not as a general
+//! purpose benchmark demonstrating real-world performance.
+
+use tokio::runtime::{self, Runtime};
+
+use bencher::{benchmark_group, benchmark_main, Bencher};
+
+const NUM_SPAWN: usize = 1_000;
+
+fn spawn_many_local(b: &mut Bencher) {
+    let rt = rt();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    b.iter(|| {
+        rt.block_on(async {
+            for _ in 0..NUM_SPAWN {
+                handles.push(tokio::spawn(async move {}));
+            }
+
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn spawn_many_remote_idle(b: &mut Bencher) {
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn spawn_many_remote_busy(b: &mut Bencher) {
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    rt.spawn(async {
+        fn iter() {
+            tokio::spawn(async { iter() });
+        }
+
+        iter()
+    });
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+fn rt() -> Runtime {
+    runtime::Builder::new_current_thread().build().unwrap()
+}
+
+benchmark_group!(
+    scheduler,
+    spawn_many_local,
+    spawn_many_remote_idle,
+    spawn_many_remote_busy
+);
+
+benchmark_main!(scheduler);

benches/rt_multi_threaded.rs

@@ -6,13 +6,16 @@ use tokio::runtime::{self, Runtime};
 use tokio::sync::oneshot;
 
 use bencher::{benchmark_group, benchmark_main, Bencher};
-use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering::Relaxed;
+use std::sync::atomic::{AtomicBool, AtomicUsize};
 use std::sync::{mpsc, Arc};
+use std::time::{Duration, Instant};
 
-fn spawn_many(b: &mut Bencher) {
-    const NUM_SPAWN: usize = 10_000;
+const NUM_WORKERS: usize = 4;
+const NUM_SPAWN: usize = 10_000;
+const STALL_DUR: Duration = Duration::from_micros(10);
 
+fn spawn_many_local(b: &mut Bencher) {
     let rt = rt();
 
     let (tx, rx) = mpsc::sync_channel(1000);
@@ -38,6 +41,99 @@ fn spawn_many(b: &mut Bencher) {
     });
 }
 
+fn spawn_many_remote_idle(b: &mut Bencher) {
+    let rt = rt();
+
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+}
+
+// The runtime is busy with tasks that consume CPU time and yield. Yielding is a
+// lower notification priority than spawning / regular notification.
+fn spawn_many_remote_busy1(b: &mut Bencher) {
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+    let flag = Arc::new(AtomicBool::new(true));
+
+    // Spawn some tasks to keep the runtimes busy
+    for _ in 0..(2 * NUM_WORKERS) {
+        let flag = flag.clone();
+        rt.spawn(async move {
+            while flag.load(Relaxed) {
+                tokio::task::yield_now().await;
+                stall();
+            }
+        });
+    }
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+
+    flag.store(false, Relaxed);
+}
+
+// The runtime is busy with tasks that consume CPU time and spawn new high-CPU
+// tasks. Spawning goes via a higher notification priority than yielding.
+fn spawn_many_remote_busy2(b: &mut Bencher) {
+    const NUM_SPAWN: usize = 1_000;
+
+    let rt = rt();
+    let rt_handle = rt.handle();
+    let mut handles = Vec::with_capacity(NUM_SPAWN);
+    let flag = Arc::new(AtomicBool::new(true));
+
+    // Spawn some tasks to keep the runtimes busy
+    for _ in 0..(NUM_WORKERS) {
+        let flag = flag.clone();
+        fn iter(flag: Arc<AtomicBool>) {
+            tokio::spawn(async {
+                if flag.load(Relaxed) {
+                    stall();
+                    iter(flag);
+                }
+            });
+        }
+        rt.spawn(async {
+            iter(flag);
+        });
+    }
+
+    b.iter(|| {
+        for _ in 0..NUM_SPAWN {
+            handles.push(rt_handle.spawn(async {}));
+        }
+
+        rt.block_on(async {
+            for handle in handles.drain(..) {
+                handle.await.unwrap();
+            }
+        });
+    });
+
+    flag.store(false, Relaxed);
+}
+
 fn yield_many(b: &mut Bencher) {
     const NUM_YIELD: usize = 1_000;
     const TASKS: usize = 200;
@@ -140,12 +236,28 @@ fn chained_spawn(b: &mut Bencher) {
 
 fn rt() -> Runtime {
     runtime::Builder::new_multi_thread()
-        .worker_threads(4)
+        .worker_threads(NUM_WORKERS)
         .enable_all()
         .build()
         .unwrap()
 }
 
-benchmark_group!(scheduler, spawn_many, ping_pong, yield_many, chained_spawn,);
+fn stall() {
+    let now = Instant::now();
+    while now.elapsed() < STALL_DUR {
+        std::thread::yield_now();
+    }
+}
+
+benchmark_group!(
+    scheduler,
+    spawn_many_local,
+    spawn_many_remote_idle,
+    spawn_many_remote_busy1,
+    spawn_many_remote_busy2,
+    ping_pong,
+    yield_many,
+    chained_spawn,
+);
 
 benchmark_main!(scheduler);

examples/Cargo.toml

@@ -2,7 +2,7 @@
 name = "examples"
 version = "0.0.0"
 publish = false
-edition = "2018"
+edition = "2021"
 
 # If you copy one of the examples into a new project, you should be using
 # [dependencies] instead, and delete the **path**.
@@ -90,3 +90,7 @@ path = "named-pipe-ready.rs"
 [[example]]
 name = "named-pipe-multi-client"
 path = "named-pipe-multi-client.rs"
+
+[[example]]
+name = "dump"
+path = "dump.rs"

examples/dump.rs

@@ -0,0 +1,90 @@
+//! This example demonstrates tokio's experimental task dumping functionality.
+//! This application deadlocks. Input CTRL+C to display traces of each task, or
+//! input CTRL+C twice within 1 second to quit.
+
+#[cfg(all(
+    tokio_unstable,
+    tokio_taskdump,
+    target_os = "linux",
+    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+))]
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    use std::sync::Arc;
+    use tokio::sync::Barrier;
+
+    #[inline(never)]
+    async fn a(barrier: Arc<Barrier>) {
+        b(barrier).await
+    }
+
+    #[inline(never)]
+    async fn b(barrier: Arc<Barrier>) {
+        c(barrier).await
+    }
+
+    #[inline(never)]
+    async fn c(barrier: Arc<Barrier>) {
+        barrier.wait().await;
+    }
+
+    // Prints a task dump upon receipt of CTRL+C, or returns if CTRL+C is
+    // inputted twice within a second.
+    async fn dump_or_quit() {
+        use tokio::time::{timeout, Duration, Instant};
+        let handle = tokio::runtime::Handle::current();
+        let mut last_signal: Option<Instant> = None;
+        // wait for CTRL+C
+        while let Ok(_) = tokio::signal::ctrl_c().await {
+            // exit if a CTRL+C is inputted twice within 1 second
+            if let Some(time_since_last_signal) = last_signal.map(|i| i.elapsed()) {
+                if time_since_last_signal < Duration::from_secs(1) {
+                    return;
+                }
+            }
+            last_signal = Some(Instant::now());
+
+            // capture a dump, and print each trace
+            println!("{:-<80}", "");
+            if let Ok(dump) = timeout(Duration::from_secs(2), handle.dump()).await {
+                for (i, task) in dump.tasks().iter().enumerate() {
+                    let trace = task.trace();
+                    println!("TASK {i}:");
+                    println!("{trace}\n");
+                }
+            } else {
+                println!("Task dumping timed out. Use a native debugger (like gdb) to debug the deadlock.");
+            }
+            println!("{:-<80}", "");
+            println!("Input CTRL+C twice within 1 second to exit.");
+        }
+    }
+
+    println!("This program has a deadlock.");
+    println!("Input CTRL+C to print a task dump.");
+    println!("Input CTRL+C twice within 1 second to exit.");
+
+    // oops! this barrier waits for one more task than will ever come.
+    let barrier = Arc::new(Barrier::new(3));
+
+    let task_1 = tokio::spawn(a(barrier.clone()));
+    let task_2 = tokio::spawn(a(barrier));
+
+    tokio::select!(
+        _ = dump_or_quit() => {},
+        _ = task_1 => {},
+        _ = task_2 => {},
+    );
+
+    Ok(())
+}
+
+#[cfg(not(all(
+    tokio_unstable,
+    tokio_taskdump,
+    target_os = "linux",
+    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+)))]
+fn main() {
+    println!("task dumps are not available")
+}

examples/tinyhttp.rs

@@ -18,7 +18,7 @@ use futures::SinkExt;
 use http::{header::HeaderValue, Request, Response, StatusCode};
 #[macro_use]
 extern crate serde_derive;
-use std::{convert::TryFrom, env, error::Error, fmt, io};
+use std::{env, error::Error, fmt, io};
 use tokio::net::{TcpListener, TcpStream};
 use tokio_stream::StreamExt;
 use tokio_util::codec::{Decoder, Encoder, Framed};

stress-test/Cargo.toml

@@ -2,7 +2,7 @@
 name = "stress-test"
 version = "0.1.0"
 authors = ["Tokio Contributors <[email protected]>"]
-edition = "2018"
+edition = "2021"
 publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

tests-build/Cargo.toml

@@ -2,7 +2,7 @@
 name = "tests-build"
 version = "0.1.0"
 authors = ["Tokio Contributors <[email protected]>"]
-edition = "2018"
+edition = "2021"
 publish = false
 
 [features]

tests-integration/Cargo.toml

@@ -2,7 +2,7 @@
 name = "tests-integration"
 version = "0.1.0"
 authors = ["Tokio Contributors <[email protected]>"]
-edition = "2018"
+edition = "2021"
 publish = false
 
 [[bin]]

tests-integration/tests/process_stdio.rs

@@ -7,7 +7,6 @@ use tokio::process::{Child, Command};
 use tokio_test::assert_ok;
 
 use futures::future::{self, FutureExt};
-use std::convert::TryInto;
 use std::env;
 use std::io;
 use std::process::{ExitStatus, Stdio};

tokio-macros/Cargo.toml

@@ -5,7 +5,7 @@ name = "tokio-macros"
 # - Update CHANGELOG.md.
 # - Create "tokio-macros-1.x.y" git tag.
 version = "2.1.0"
-edition = "2018"
+edition = "2021"
 rust-version = "1.56"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"

tokio-macros/src/entry.rs

@@ -354,10 +354,10 @@ fn parse_knobs(mut input: ItemFn, is_test: bool, config: FinalConfig) -> TokenSt
         },
     };
     if let Some(v) = config.worker_threads {
-        rt = quote! { #rt.worker_threads(#v) };
+        rt = quote_spanned! {last_stmt_start_span=> #rt.worker_threads(#v) };
     }
     if let Some(v) = config.start_paused {
-        rt = quote! { #rt.start_paused(#v) };
+        rt = quote_spanned! {last_stmt_start_span=> #rt.start_paused(#v) };
     }
 
     let header = if is_test {

tokio-macros/src/lib.rs

@@ -209,7 +209,7 @@ pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
 
 /// Marks async function to be executed by selected runtime. This macro helps set up a `Runtime`
 /// without requiring the user to use [Runtime](../tokio/runtime/struct.Runtime.html) or
-/// [Builder](../tokio/runtime/struct.builder.html) directly.
+/// [Builder](../tokio/runtime/struct.Builder.html) directly.
 ///
 /// ## Function arguments:
 ///

tokio-stream/CHANGELOG.md

@@ -1,3 +1,22 @@
+# 0.1.14 (April 26th, 2023)
+
+This bugfix release bumps the minimum version of Tokio to 1.15, which is
+necessary for `timeout_repeating` to compile. ([#5657])
+
+[#5657]: https://github.com/tokio-rs/tokio/pull/5657
+
+# 0.1.13 (April 25th, 2023)
+
+This release bumps the MSRV of tokio-stream to 1.56.
+
+- stream: add "full" feature flag ([#5639])
+- stream: add `StreamExt::timeout_repeating` ([#5577])
+- stream: add `StreamNotifyClose` ([#4851])
+
+[#4851]: https://github.com/tokio-rs/tokio/pull/4851
+[#5577]: https://github.com/tokio-rs/tokio/pull/5577
+[#5639]: https://github.com/tokio-rs/tokio/pull/5639
+
 # 0.1.12 (January 20, 2023)
 
 - time: remove `Unpin` bound on `Throttle` methods ([#5105])

tokio-stream/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-stream"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-stream-0.1.x" git tag.
-version = "0.1.12"
+version = "0.1.14"
 edition = "2021"
 rust-version = "1.56"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -37,8 +37,8 @@ signal = ["tokio/signal"]
 
 [dependencies]
 futures-core = { version = "0.3.0" }
-pin-project-lite = "0.2.0"
-tokio = { version = "1.8.0", path = "../tokio", features = ["sync"] }
+pin-project-lite = "0.2.7"
+tokio = { version = "1.15.0", path = "../tokio", features = ["sync"] }
 tokio-util = { version = "0.7.0", path = "../tokio-util", optional = true }
 
 [dev-dependencies]

tokio-stream/fuzz/Cargo.toml

@@ -2,7 +2,7 @@
 name = "tokio-stream-fuzz"
 version = "0.0.0"
 publish = false
-edition = "2018"
+edition = "2021"
 
 [package.metadata]
 cargo-fuzz = true

tokio-stream/fuzz/fuzz_targets/fuzz_stream_map.rs

@@ -3,17 +3,8 @@
 use libfuzzer_sys::fuzz_target;
 use std::pin::Pin;
 
-use tokio_stream::{self as stream, pending, Stream, StreamExt, StreamMap};
-use tokio_test::{assert_ok, assert_pending, assert_ready, task};
-
-macro_rules! assert_ready_some {
-    ($($t:tt)*) => {
-        match assert_ready!($($t)*) {
-            Some(v) => v,
-            None => panic!("expected `Some`, got `None`"),
-        }
-    };
-}
+use tokio_stream::{self as stream, Stream, StreamMap};
+use tokio_test::{assert_pending, assert_ready, task};
 
 macro_rules! assert_ready_none {
     ($($t:tt)*) => {

tokio-stream/src/lib.rs

@@ -63,12 +63,12 @@
 //! [`tokio-util`] provides the [`StreamReader`] and [`ReaderStream`]
 //! types when the io feature is enabled.
 //!
-//! [`tokio-util`]: https://docs.rs/tokio-util/0.4/tokio_util/codec/index.html
-//! [`tokio::io`]: https://docs.rs/tokio/1.0/tokio/io/index.html
-//! [`AsyncRead`]: https://docs.rs/tokio/1.0/tokio/io/trait.AsyncRead.html
-//! [`AsyncWrite`]: https://docs.rs/tokio/1.0/tokio/io/trait.AsyncWrite.html
-//! [`ReaderStream`]: https://docs.rs/tokio-util/0.4/tokio_util/io/struct.ReaderStream.html
-//! [`StreamReader`]: https://docs.rs/tokio-util/0.4/tokio_util/io/struct.StreamReader.html
+//! [`tokio-util`]: https://docs.rs/tokio-util/latest/tokio_util/codec/index.html
+//! [`tokio::io`]: https://docs.rs/tokio/latest/tokio/io/index.html
+//! [`AsyncRead`]: https://docs.rs/tokio/latest/tokio/io/trait.AsyncRead.html
+//! [`AsyncWrite`]: https://docs.rs/tokio/latest/tokio/io/trait.AsyncWrite.html
+//! [`ReaderStream`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.ReaderStream.html
+//! [`StreamReader`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.StreamReader.html
 
 #[macro_use]
 mod macros;

tokio-util/CHANGELOG.md

@@ -1,3 +1,30 @@
+# 0.7.8 (April 25th, 2023)
+
+This release bumps the MSRV of tokio-util to 1.56.
+
+### Added
+
+- time: add `DelayQueue::peek` ([#5569])
+
+### Changed
+
+This release contains one performance improvement:
+
+- sync: try to lock the parent first in `CancellationToken` ([#5561])
+
+### Fixed
+
+- time: fix panic in `DelayQueue` ([#5630])
+
+### Documented
+
+- sync: improve `CancellationToken` doc on child tokens ([#5632])
+
+[#5561]: https://github.com/tokio-rs/tokio/pull/5561
+[#5569]: https://github.com/tokio-rs/tokio/pull/5569
+[#5630]: https://github.com/tokio-rs/tokio/pull/5630
+[#5632]: https://github.com/tokio-rs/tokio/pull/5632
+
 # 0.7.7 (February 12, 2023)
 
 This release reverts the removal of the `Encoder` bound on the `FramedParts`

tokio-util/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-util"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-util-0.7.x" git tag.
-version = "0.7.7"
+version = "0.7.8"
 edition = "2021"
 rust-version = "1.56"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -34,13 +34,13 @@ rt = ["tokio/rt", "tokio/sync", "futures-util", "hashbrown"]
 __docs_rs = ["futures-util"]
 
 [dependencies]
-tokio = { version = "1.22.0", path = "../tokio", features = ["sync"] }
+tokio = { version = "1.28.0", path = "../tokio", features = ["sync"] }
 bytes = "1.0.0"
 futures-core = "0.3.0"
 futures-sink = "0.3.0"
 futures-io = { version = "0.3.0", optional = true }
 futures-util = { version = "0.3.0", optional = true }
-pin-project-lite = "0.2.0"
+pin-project-lite = "0.2.7"
 slab = { version = "0.4.4", optional = true } # Backs `DelayQueue`
 tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true }
 
@@ -56,6 +56,7 @@ async-stream = "0.3.0"
 futures = "0.3.0"
 futures-test = "0.3.5"
 parking_lot = "0.12.0"
+tempfile = "3.1.0"
 
 [package.metadata.docs.rs]
 all-features = true

tokio-util/src/compat.rs

@@ -227,6 +227,8 @@ impl<T: tokio::io::AsyncSeek> futures_io::AsyncSeek for Compat<T> {
         pos: io::SeekFrom,
     ) -> Poll<io::Result<u64>> {
         if self.seek_pos != Some(pos) {
+            // Ensure previous seeks have finished before starting a new one
+            ready!(self.as_mut().project().inner.poll_complete(cx))?;
             self.as_mut().project().inner.start_seek(pos)?;
             *self.as_mut().project().seek_pos = Some(pos);
         }

tokio-util/src/io/inspect.rs

@@ -52,6 +52,42 @@ impl<R: AsyncRead, F: FnMut(&[u8])> AsyncRead for InspectReader<R, F> {
     }
 }
 
+impl<R: AsyncWrite, F> AsyncWrite for InspectReader<R, F> {
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<std::result::Result<usize, std::io::Error>> {
+        self.project().reader.poll_write(cx, buf)
+    }
+
+    fn poll_flush(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<std::result::Result<(), std::io::Error>> {
+        self.project().reader.poll_flush(cx)
+    }
+
+    fn poll_shutdown(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<std::result::Result<(), std::io::Error>> {
+        self.project().reader.poll_shutdown(cx)
+    }
+
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[IoSlice<'_>],
+    ) -> Poll<Result<usize>> {
+        self.project().reader.poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.reader.is_write_vectored()
+    }
+}
+
 pin_project! {
     /// An adapter that lets you inspect the data that's being written.
     ///
@@ -132,3 +168,13 @@ impl<W: AsyncWrite, F: FnMut(&[u8])> AsyncWrite for InspectWriter<W, F> {
         self.writer.is_write_vectored()
     }
 }
+
+impl<W: AsyncRead, F> AsyncRead for InspectWriter<W, F> {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        self.project().writer.poll_read(cx, buf)
+    }
+}

tokio-util/src/lib.rs

@@ -55,6 +55,8 @@ pub mod sync;
 
 pub mod either;
 
+pub use bytes;
+
 #[cfg(any(feature = "io", feature = "codec"))]
 mod util {
     use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};

tokio-util/src/sync/mpsc.rs

@@ -44,7 +44,7 @@ enum State<T> {
 pub struct PollSender<T> {
     sender: Option<Sender<T>>,
     state: State<T>,
-    acquire: ReusableBoxFuture<'static, Result<OwnedPermit<T>, PollSendError<T>>>,
+    acquire: PollSenderFuture<T>,
 }
 
 // Creates a future for acquiring a permit from the underlying channel.  This is used to ensure
@@ -64,13 +64,56 @@ async fn make_acquire_future<T>(
     }
 }
 
-impl<T: Send + 'static> PollSender<T> {
+type InnerFuture<'a, T> = ReusableBoxFuture<'a, Result<OwnedPermit<T>, PollSendError<T>>>;
+
+#[derive(Debug)]
+// TODO: This should be replace with a type_alias_impl_trait to eliminate `'static` and all the transmutes
+struct PollSenderFuture<T>(InnerFuture<'static, T>);
+
+impl<T> PollSenderFuture<T> {
+    /// Create with an empty inner future with no `Send` bound.
+    fn empty() -> Self {
+        // We don't use `make_acquire_future` here because our relaxed bounds on `T` are not
+        // compatible with the transitive bounds required by `Sender<T>`.
+        Self(ReusableBoxFuture::new(async { unreachable!() }))
+    }
+}
+
+impl<T: Send> PollSenderFuture<T> {
+    /// Create with an empty inner future.
+    fn new() -> Self {
+        let v = InnerFuture::new(make_acquire_future(None));
+        // This is safe because `make_acquire_future(None)` is actually `'static`
+        Self(unsafe { mem::transmute::<InnerFuture<'_, T>, InnerFuture<'static, T>>(v) })
+    }
+
+    /// Poll the inner future.
+    fn poll(&mut self, cx: &mut Context<'_>) -> Poll<Result<OwnedPermit<T>, PollSendError<T>>> {
+        self.0.poll(cx)
+    }
+
+    /// Replace the inner future.
+    fn set(&mut self, sender: Option<Sender<T>>) {
+        let inner: *mut InnerFuture<'static, T> = &mut self.0;
+        let inner: *mut InnerFuture<'_, T> = inner.cast();
+        // SAFETY: The `make_acquire_future(sender)` future must not exist after the type `T`
+        // becomes invalid, and this casts away the type-level lifetime check for that. However, the
+        // inner future is never moved out of this `PollSenderFuture<T>`, so the future will not
+        // live longer than the `PollSenderFuture<T>` lives. A `PollSenderFuture<T>` is guaranteed
+        // to not exist after the type `T` becomes invalid, because it is annotated with a `T`, so
+        // this is ok.
+        let inner = unsafe { &mut *inner };
+        inner.set(make_acquire_future(sender));
+    }
+}
+
+impl<T: Send> PollSender<T> {
     /// Creates a new `PollSender`.
     pub fn new(sender: Sender<T>) -> Self {
         Self {
             sender: Some(sender.clone()),
             state: State::Idle(sender),
-            acquire: ReusableBoxFuture::new(make_acquire_future(None)),
+            acquire: PollSenderFuture::new(),
         }
     }
 
@@ -97,7 +140,7 @@ impl<T: Send + 'static> PollSender<T> {
                 State::Idle(sender) => {
                     // Start trying to acquire a permit to reserve a slot for our send, and
                     // immediately loop back around to poll it the first time.
-                    self.acquire.set(make_acquire_future(Some(sender)));
+                    self.acquire.set(Some(sender));
                     (None, State::Acquiring)
                 }
                 State::Acquiring => match self.acquire.poll(cx) {
@@ -194,7 +237,7 @@ impl<T: Send + 'static> PollSender<T> {
         match self.state {
             State::Idle(_) => self.state = State::Closed,
             State::Acquiring => {
-                self.acquire.set(make_acquire_future(None));
+                self.acquire.set(None);
                 self.state = State::Closed;
             }
             _ => {}
@@ -215,7 +258,7 @@ impl<T: Send + 'static> PollSender<T> {
             // We're currently trying to reserve a slot to send into.
             State::Acquiring => {
                 // Replacing the future drops the in-flight one.
-                self.acquire.set(make_acquire_future(None));
+                self.acquire.set(None);
 
                 // If we haven't closed yet, we have to clone our stored sender since we have no way
                 // to get it back from the acquire future we just dropped.
@@ -255,9 +298,7 @@ impl<T> Clone for PollSender<T> {
         Self {
             sender,
             state,
-            // We don't use `make_acquire_future` here because our relaxed bounds on `T` are not
-            // compatible with the transitive bounds required by `Sender<T>`.
-            acquire: ReusableBoxFuture::new(async { unreachable!() }),
+            acquire: PollSenderFuture::empty(),
         }
     }
 }

tokio-util/src/task/join_map.rs

@@ -316,6 +316,60 @@ where
         self.insert(key, task);
     }
 
+    /// Spawn the blocking code on the blocking threadpool and store it in this `JoinMap` with the provided
+    /// key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// Note that blocking tasks cannot be cancelled after execution starts.
+    /// Replaced blocking tasks will still run to completion if the task has begun
+    /// to execute when it is replaced. A blocking task which is replaced before
+    /// it has been scheduled on a blocking worker thread will be cancelled.
+    ///
+    /// # Panics
+    ///
+    /// This method panics if called outside of a Tokio runtime.
+    ///
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
+    pub fn spawn_blocking<F>(&mut self, key: K, f: F)
+    where
+        F: FnOnce() -> V,
+        F: Send + 'static,
+        V: Send,
+    {
+        let task = self.tasks.spawn_blocking(f);
+        self.insert(key, task)
+    }
+
+    /// Spawn the blocking code on the blocking threadpool of the provided runtime and store it in this
+    /// `JoinMap` with the provided key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// Note that blocking tasks cannot be cancelled after execution starts.
+    /// Replaced blocking tasks will still run to completion if the task has begun
+    /// to execute when it is replaced. A blocking task which is replaced before
+    /// it has been scheduled on a blocking worker thread will be cancelled.
+    ///
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
+    pub fn spawn_blocking_on<F>(&mut self, key: K, f: F, handle: &Handle)
+    where
+        F: FnOnce() -> V,
+        F: Send + 'static,
+        V: Send,
+    {
+        let task = self.tasks.spawn_blocking_on(f, handle);
+        self.insert(key, task);
+    }
+
     /// Spawn the provided task on the current [`LocalSet`] and store it in this
     /// `JoinMap` with the provided key.
     ///

tokio-util/tests/compat.rs

@@ -0,0 +1,43 @@
+#![cfg(all(feature = "compat"))]
+#![cfg(not(target_os = "wasi"))] // WASI does not support all fs operations
+#![warn(rust_2018_idioms)]
+
+use futures_io::SeekFrom;
+use futures_util::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
+use tempfile::NamedTempFile;
+use tokio::fs::OpenOptions;
+use tokio_util::compat::TokioAsyncWriteCompatExt;
+
+#[tokio::test]
+async fn compat_file_seek() -> futures_util::io::Result<()> {
+    let temp_file = NamedTempFile::new()?;
+    let mut file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(temp_file)
+        .await?
+        .compat_write();
+
+    file.write_all(&[0, 1, 2, 3, 4, 5]).await?;
+    file.write_all(&[6, 7]).await?;
+
+    assert_eq!(file.stream_position().await?, 8);
+
+    // Modify elements at position 2.
+    assert_eq!(file.seek(SeekFrom::Start(2)).await?, 2);
+    file.write_all(&[8, 9]).await?;
+
+    file.flush().await?;
+
+    // Verify we still have 8 elements.
+    assert_eq!(file.seek(SeekFrom::End(0)).await?, 8);
+    // Seek back to the start of the file to read and verify contents.
+    file.seek(SeekFrom::Start(0)).await?;
+
+    let mut buf = Vec::new();
+    let num_bytes = file.read_to_end(&mut buf).await?;
+    assert_eq!(&buf[..num_bytes], &[0, 1, 8, 9, 4, 5, 6, 7]);
+
+    Ok(())
+}

tokio-util/tests/mpsc.rs

@@ -27,6 +27,29 @@ async fn simple() {
     send.send_item(42).unwrap();
 }
 
+#[tokio::test]
+async fn simple_ref() {
+    let v = vec![1, 2, 3i32];
+
+    let (send, mut recv) = channel(3);
+    let mut send = PollSender::new(send);
+
+    for vi in v.iter() {
+        let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+        assert_ready_ok!(reserve.poll());
+        send.send_item(vi).unwrap();
+    }
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+
+    assert_eq!(*recv.recv().await.unwrap(), 1);
+    assert!(reserve.is_woken());
+    assert_ready_ok!(reserve.poll());
+    drop(recv);
+    send.send_item(&42).unwrap();
+}
+
 #[tokio::test]
 async fn repeated_poll_reserve() {
     let (send, mut recv) = channel::<i32>(1);

tokio/Cargo.toml

@@ -99,11 +99,11 @@ autocfg = "1.1"
 [dependencies]
 tokio-macros = { version = "~2.1.0", path = "../tokio-macros", optional = true }
 
-pin-project-lite = "0.2.0"
+pin-project-lite = "0.2.7"
 
 # Everything else is optional...
 bytes = { version = "1.0.0", optional = true }
-mio = { version = "0.8.4", optional = true, default-features = false }
+mio = { version = "0.8.6", optional = true, default-features = false }
 num_cpus = { version = "1.8.0", optional = true }
 parking_lot = { version = "0.12.0", optional = true }
 
@@ -115,19 +115,24 @@ socket2 = { version = "0.4.9", optional = true, features = [ "all" ] }
 [target.'cfg(tokio_unstable)'.dependencies]
 tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true } # Not in full
 
+# Currently unstable. The API exposed by these features may be broken at any time.
+# Requires `--cfg tokio_unstable` to enable.
+[target.'cfg(tokio_taskdump)'.dependencies]
+backtrace = { version = "0.3.58" }
+
 [target.'cfg(unix)'.dependencies]
-libc = { version = "0.2.42", optional = true }
+libc = { version = "0.2.145", optional = true }
 signal-hook-registry = { version = "1.1.1", optional = true }
 
 [target.'cfg(unix)'.dev-dependencies]
-libc = { version = "0.2.42" }
+libc = { version = "0.2.145" }
 nix = { version = "0.26", default-features = false, features = ["fs", "socket"] }
 
 [target.'cfg(windows)'.dependencies.windows-sys]
 version = "0.48"
 optional = true
 
-[target.'cfg(docsrs)'.dependencies.windows-sys]
+[target.'cfg(windows)'.dev-dependencies.windows-sys]
 version = "0.48"
 features = [
     "Win32_Foundation",

tokio/fuzz/Cargo.toml

@@ -2,7 +2,7 @@
 name = "tokio-fuzz"
 version = "0.0.0"
 publish = false
-edition = "2018"
+edition = "2021"
 
 [package.metadata]
 cargo-fuzz = true

tokio/src/fs/file.rs

@@ -498,6 +498,7 @@ impl AsyncRead for File {
         cx: &mut Context<'_>,
         dst: &mut ReadBuf<'_>,
     ) -> Poll<io::Result<()>> {
+        ready!(crate::trace::trace_leaf(cx));
         let me = self.get_mut();
         let inner = me.inner.get_mut();
 
@@ -594,6 +595,7 @@ impl AsyncSeek for File {
     }
 
     fn poll_complete(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
+        ready!(crate::trace::trace_leaf(cx));
         let inner = self.inner.get_mut();
 
         loop {
@@ -629,6 +631,7 @@ impl AsyncWrite for File {
         cx: &mut Context<'_>,
         src: &[u8],
     ) -> Poll<io::Result<usize>> {
+        ready!(crate::trace::trace_leaf(cx));
         let me = self.get_mut();
         let inner = me.inner.get_mut();
 
@@ -695,11 +698,13 @@ impl AsyncWrite for File {
     }
 
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
+        ready!(crate::trace::trace_leaf(cx));
         let inner = self.inner.get_mut();
         inner.poll_flush(cx)
     }
 
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
+        ready!(crate::trace::trace_leaf(cx));
         self.poll_flush(cx)
     }
 }
@@ -774,8 +779,18 @@ impl Inner {
     async fn complete_inflight(&mut self) {
         use crate::future::poll_fn;
 
-        if let Err(e) = poll_fn(|cx| Pin::new(&mut *self).poll_flush(cx)).await {
-            self.last_write_err = Some(e.kind());
+        poll_fn(|cx| self.poll_complete_inflight(cx)).await
+    }
+
+    fn poll_complete_inflight(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+        ready!(crate::trace::trace_leaf(cx));
+        match self.poll_flush(cx) {
+            Poll::Ready(Err(e)) => {
+                self.last_write_err = Some(e.kind());
+                Poll::Ready(())
+            }
+            Poll::Ready(Ok(())) => Poll::Ready(()),
+            Poll::Pending => Poll::Pending,
         }
     }
 

tokio/src/fs/read_dir.rs

@@ -33,11 +33,11 @@ const CHUNK_SIZE: usize = 32;
 pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
     let path = path.as_ref().to_owned();
     asyncify(|| -> io::Result<ReadDir> {
-        let mut std = std::fs::read_dir(path)?.fuse();
+        let mut std = std::fs::read_dir(path)?;
         let mut buf = VecDeque::with_capacity(CHUNK_SIZE);
-        ReadDir::next_chunk(&mut buf, &mut std);
+        let remain = ReadDir::next_chunk(&mut buf, &mut std);
 
-        Ok(ReadDir(State::Idle(Some((buf, std)))))
+        Ok(ReadDir(State::Idle(Some((buf, std, remain)))))
     })
     .await
 }
@@ -64,12 +64,10 @@ pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
 #[must_use = "streams do nothing unless polled"]
 pub struct ReadDir(State);
 
-type StdReadDir = std::iter::Fuse<std::fs::ReadDir>;
-
 #[derive(Debug)]
 enum State {
-    Idle(Option<(VecDeque<io::Result<DirEntry>>, StdReadDir)>),
-    Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, StdReadDir)>),
+    Idle(Option<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir, bool)>),
+    Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir, bool)>),
 }
 
 impl ReadDir {
@@ -105,38 +103,35 @@ impl ReadDir {
         loop {
             match self.0 {
                 State::Idle(ref mut data) => {
-                    let (buf, _) = data.as_mut().unwrap();
+                    let (buf, _, ref remain) = data.as_mut().unwrap();
 
                     if let Some(ent) = buf.pop_front() {
                         return Poll::Ready(ent.map(Some));
-                    };
+                    } else if !remain {
+                        return Poll::Ready(Ok(None));
+                    }
 
-                    let (mut buf, mut std) = data.take().unwrap();
+                    let (mut buf, mut std, _) = data.take().unwrap();
 
                     self.0 = State::Pending(spawn_blocking(move || {
-                        ReadDir::next_chunk(&mut buf, &mut std);
-                        (buf, std)
+                        let remain = ReadDir::next_chunk(&mut buf, &mut std);
+                        (buf, std, remain)
                     }));
                 }
                 State::Pending(ref mut rx) => {
-                    let (mut buf, std) = ready!(Pin::new(rx).poll(cx))?;
-
-                    let ret = match buf.pop_front() {
-                        Some(Ok(x)) => Ok(Some(x)),
-                        Some(Err(e)) => Err(e),
-                        None => Ok(None),
-                    };
-
-                    self.0 = State::Idle(Some((buf, std)));
-
-                    return Poll::Ready(ret);
+                    self.0 = State::Idle(Some(ready!(Pin::new(rx).poll(cx))?));
                 }
             }
         }
     }
 
-    fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut StdReadDir) {
-        for ret in std.by_ref().take(CHUNK_SIZE) {
+    fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut std::fs::ReadDir) -> bool {
+        for _ in 0..CHUNK_SIZE {
+            let ret = match std.next() {
+                Some(ret) => ret,
+                None => return false,
+            };
+
             let success = ret.is_ok();
 
             buf.push_back(ret.map(|std| DirEntry {
@@ -144,7 +139,9 @@ impl ReadDir {
                     target_os = "solaris",
                     target_os = "illumos",
                     target_os = "haiku",
-                    target_os = "vxworks"
+                    target_os = "vxworks",
+                    target_os = "nto",
+                    target_os = "vita",
                 )))]
                 file_type: std.file_type().ok(),
                 std: Arc::new(std),
@@ -154,6 +151,8 @@ impl ReadDir {
                 break;
             }
         }
+
+        true
     }
 }
 
@@ -203,7 +202,9 @@ pub struct DirEntry {
         target_os = "solaris",
         target_os = "illumos",
         target_os = "haiku",
-        target_os = "vxworks"
+        target_os = "vxworks",
+        target_os = "nto",
+        target_os = "vita",
     )))]
     file_type: Option<FileType>,
     std: Arc<std::fs::DirEntry>,
@@ -334,7 +335,9 @@ impl DirEntry {
             target_os = "solaris",
             target_os = "illumos",
             target_os = "haiku",
-            target_os = "vxworks"
+            target_os = "vxworks",
+            target_os = "nto",
+            target_os = "vita",
         )))]
         if let Some(file_type) = self.file_type {
             return Ok(file_type);

tokio/src/io/async_fd.rs

@@ -1,4 +1,4 @@
-use crate::io::Interest;
+use crate::io::{Interest, Ready};
 use crate::runtime::io::{ReadyEvent, Registration};
 use crate::runtime::scheduler;
 
@@ -201,13 +201,14 @@ pub struct AsyncFdReadyMutGuard<'a, T: AsRawFd> {
     event: Option<ReadyEvent>,
 }
 
-const ALL_INTEREST: Interest = Interest::READABLE.add(Interest::WRITABLE);
-
 impl<T: AsRawFd> AsyncFd<T> {
     /// Creates an AsyncFd backed by (and taking ownership of) an object
     /// implementing [`AsRawFd`]. The backing file descriptor is cached at the
     /// time of creation.
     ///
+    /// Only configures the [`Interest::READABLE`] and [`Interest::WRITABLE`] interests. For more
+    /// control, use [`AsyncFd::with_interest`].
+    ///
     /// This method must be called in the context of a tokio runtime.
     ///
     /// # Panics
@@ -220,11 +221,12 @@ impl<T: AsRawFd> AsyncFd<T> {
     where
         T: AsRawFd,
     {
-        Self::with_interest(inner, ALL_INTEREST)
+        Self::with_interest(inner, Interest::READABLE | Interest::WRITABLE)
     }
 
-    /// Creates new instance as `new` with additional ability to customize interest,
-    /// allowing to specify whether file descriptor will be polled for read, write or both.
+    /// Creates an AsyncFd backed by (and taking ownership of) an object
+    /// implementing [`AsRawFd`], with a specific [`Interest`]. The backing
+    /// file descriptor is cached at the time of creation.
     ///
     /// # Panics
     ///
@@ -440,7 +442,96 @@ impl<T: AsRawFd> AsyncFd<T> {
         .into()
     }
 
-    async fn readiness(&self, interest: Interest) -> io::Result<AsyncFdReadyGuard<'_, T>> {
+    /// Waits for any of the requested ready states, returning a
+    /// [`AsyncFdReadyGuard`] that must be dropped to resume
+    /// polling for the requested ready states.
+    ///
+    /// The function may complete without the file descriptor being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
+    /// When an IO operation does return `io::ErrorKind::WouldBlock`, the readiness must be cleared.
+    /// When a combined interest is used, it is important to clear only the readiness
+    /// that is actually observed to block. For instance when the combined
+    /// interest `Interest::READABLE | Interest::WRITABLE` is used, and a read blocks, only
+    /// read readiness should be cleared using the [`AsyncFdReadyGuard::clear_ready_matching`] method:
+    /// `guard.clear_ready_matching(Ready::READABLE)`.
+    /// Also clearing the write readiness in this case would be incorrect. The [`AsyncFdReadyGuard::clear_ready`]
+    /// method clears all readiness flags.
+    ///
+    /// This method takes `&self`, so it is possible to call this method
+    /// concurrently with other methods on this struct. This method only
+    /// provides shared access to the inner IO resource when handling the
+    /// [`AsyncFdReadyGuard`].
+    ///
+    /// # Examples
+    ///
+    /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without
+    /// splitting.
+    ///
+    /// ```no_run
+    /// use std::error::Error;
+    /// use std::io;
+    /// use std::io::{Read, Write};
+    /// use std::net::TcpStream;
+    /// use tokio::io::unix::AsyncFd;
+    /// use tokio::io::{Interest, Ready};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn Error>> {
+    ///     let stream = TcpStream::connect("127.0.0.1:8080")?;
+    ///     stream.set_nonblocking(true)?;
+    ///     let stream = AsyncFd::new(stream)?;
+    ///
+    ///     loop {
+    ///         let mut guard = stream
+    ///             .ready(Interest::READABLE | Interest::WRITABLE)
+    ///             .await?;
+    ///
+    ///         if guard.ready().is_readable() {
+    ///             let mut data = vec![0; 1024];
+    ///             // Try to read data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match stream.get_ref().read(&mut data) {
+    ///                 Ok(n) => {
+    ///                     println!("read {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a read has blocked, but a write might still succeed.
+    ///                     // clear only the read readiness.
+    ///                     guard.clear_ready_matching(Ready::READABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///
+    ///         if guard.ready().is_writable() {
+    ///             // Try to write data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match stream.get_ref().write(b"hello world") {
+    ///                 Ok(n) => {
+    ///                     println!("write {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a write has blocked, but a read might still succeed.
+    ///                     // clear only the write readiness.
+    ///                     guard.clear_ready_matching(Ready::WRITABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    pub async fn ready(&self, interest: Interest) -> io::Result<AsyncFdReadyGuard<'_, T>> {
         let event = self.registration.readiness(interest).await?;
 
         Ok(AsyncFdReadyGuard {
@@ -449,7 +540,94 @@ impl<T: AsRawFd> AsyncFd<T> {
         })
     }
 
-    async fn readiness_mut(
+    /// Waits for any of the requested ready states, returning a
+    /// [`AsyncFdReadyMutGuard`] that must be dropped to resume
+    /// polling for the requested ready states.
+    ///
+    /// The function may complete without the file descriptor being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
+    /// When an IO operation does return `io::ErrorKind::WouldBlock`, the readiness must be cleared.
+    /// When a combined interest is used, it is important to clear only the readiness
+    /// that is actually observed to block. For instance when the combined
+    /// interest `Interest::READABLE | Interest::WRITABLE` is used, and a read blocks, only
+    /// read readiness should be cleared using the [`AsyncFdReadyMutGuard::clear_ready_matching`] method:
+    /// `guard.clear_ready_matching(Ready::READABLE)`.
+    /// Also clearing the write readiness in this case would be incorrect.
+    /// The [`AsyncFdReadyMutGuard::clear_ready`] method clears all readiness flags.
+    ///
+    /// This method takes `&mut self`, so it is possible to access the inner IO
+    /// resource mutably when handling the [`AsyncFdReadyMutGuard`].
+    ///
+    /// # Examples
+    ///
+    /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without
+    /// splitting.
+    ///
+    /// ```no_run
+    /// use std::error::Error;
+    /// use std::io;
+    /// use std::io::{Read, Write};
+    /// use std::net::TcpStream;
+    /// use tokio::io::unix::AsyncFd;
+    /// use tokio::io::{Interest, Ready};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn Error>> {
+    ///     let stream = TcpStream::connect("127.0.0.1:8080")?;
+    ///     stream.set_nonblocking(true)?;
+    ///     let mut stream = AsyncFd::new(stream)?;
+    ///
+    ///     loop {
+    ///         let mut guard = stream
+    ///             .ready_mut(Interest::READABLE | Interest::WRITABLE)
+    ///             .await?;
+    ///
+    ///         if guard.ready().is_readable() {
+    ///             let mut data = vec![0; 1024];
+    ///             // Try to read data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match guard.get_inner_mut().read(&mut data) {
+    ///                 Ok(n) => {
+    ///                     println!("read {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a read has blocked, but a write might still succeed.
+    ///                     // clear only the read readiness.
+    ///                     guard.clear_ready_matching(Ready::READABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///
+    ///         if guard.ready().is_writable() {
+    ///             // Try to write data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match guard.get_inner_mut().write(b"hello world") {
+    ///                 Ok(n) => {
+    ///                     println!("write {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a write has blocked, but a read might still succeed.
+    ///                     // clear only the write readiness.
+    ///                     guard.clear_ready_matching(Ready::WRITABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    pub async fn ready_mut(
         &mut self,
         interest: Interest,
     ) -> io::Result<AsyncFdReadyMutGuard<'_, T>> {
@@ -471,7 +649,7 @@ impl<T: AsRawFd> AsyncFd<T> {
     /// [`AsyncFdReadyGuard`].
     #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering.
     pub async fn readable<'a>(&'a self) -> io::Result<AsyncFdReadyGuard<'a, T>> {
-        self.readiness(Interest::READABLE).await
+        self.ready(Interest::READABLE).await
     }
 
     /// Waits for the file descriptor to become readable, returning a
@@ -482,7 +660,7 @@ impl<T: AsRawFd> AsyncFd<T> {
     /// resource mutably when handling the [`AsyncFdReadyMutGuard`].
     #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering.
     pub async fn readable_mut<'a>(&'a mut self) -> io::Result<AsyncFdReadyMutGuard<'a, T>> {
-        self.readiness_mut(Interest::READABLE).await
+        self.ready_mut(Interest::READABLE).await
     }
 
     /// Waits for the file descriptor to become writable, returning a
@@ -495,7 +673,7 @@ impl<T: AsRawFd> AsyncFd<T> {
     /// [`AsyncFdReadyGuard`].
     #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering.
     pub async fn writable<'a>(&'a self) -> io::Result<AsyncFdReadyGuard<'a, T>> {
-        self.readiness(Interest::WRITABLE).await
+        self.ready(Interest::WRITABLE).await
     }
 
     /// Waits for the file descriptor to become writable, returning a
@@ -506,7 +684,7 @@ impl<T: AsRawFd> AsyncFd<T> {
     /// resource mutably when handling the [`AsyncFdReadyMutGuard`].
     #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering.
     pub async fn writable_mut<'a>(&'a mut self) -> io::Result<AsyncFdReadyMutGuard<'a, T>> {
-        self.readiness_mut(Interest::WRITABLE).await
+        self.ready_mut(Interest::WRITABLE).await
     }
 
     /// Reads or writes from the file descriptor using a user-provided IO operation.
@@ -641,22 +819,117 @@ impl<T: AsRawFd> Drop for AsyncFd<T> {
 }
 
 impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> {
-    /// Indicates to tokio that the file descriptor is no longer ready. The
-    /// internal readiness flag will be cleared, and tokio will wait for the
+    /// Indicates to tokio that the file descriptor is no longer ready. All
+    /// internal readiness flags will be cleared, and tokio will wait for the
     /// next edge-triggered readiness notification from the OS.
     ///
+    /// This function is commonly used with guards returned by [`AsyncFd::readable`] and
+    /// [`AsyncFd::writable`].
+    ///
     /// It is critical that this function not be called unless your code
     /// _actually observes_ that the file descriptor is _not_ ready. Do not call
     /// it simply because, for example, a read succeeded; it should be called
     /// when a read is observed to block.
-    ///
-    /// [`drop`]: method@std::mem::drop
     pub fn clear_ready(&mut self) {
         if let Some(event) = self.event.take() {
             self.async_fd.registration.clear_readiness(event);
         }
     }
 
+    /// Indicates to tokio that the file descriptor no longer has a specific readiness.
+    /// The internal readiness flag will be cleared, and tokio will wait for the
+    /// next edge-triggered readiness notification from the OS.
+    ///
+    /// This function is useful in combination with the [`AsyncFd::ready`] method when a
+    /// combined interest like `Interest::READABLE | Interest::WRITABLE` is used.
+    ///
+    /// It is critical that this function not be called unless your code
+    /// _actually observes_ that the file descriptor is _not_ ready for the provided `Ready`.
+    /// Do not call it simply because, for example, a read succeeded; it should be called
+    /// when a read is observed to block. Only clear the specific readiness that is observed to
+    /// block. For example when a read blocks when using a combined interest,
+    /// only clear `Ready::READABLE`.
+    ///
+    /// # Examples
+    ///
+    /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without
+    /// splitting.
+    ///
+    /// ```no_run
+    /// use std::error::Error;
+    /// use std::io;
+    /// use std::io::{Read, Write};
+    /// use std::net::TcpStream;
+    /// use tokio::io::unix::AsyncFd;
+    /// use tokio::io::{Interest, Ready};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn Error>> {
+    ///     let stream = TcpStream::connect("127.0.0.1:8080")?;
+    ///     stream.set_nonblocking(true)?;
+    ///     let stream = AsyncFd::new(stream)?;
+    ///
+    ///     loop {
+    ///         let mut guard = stream
+    ///             .ready(Interest::READABLE | Interest::WRITABLE)
+    ///             .await?;
+    ///
+    ///         if guard.ready().is_readable() {
+    ///             let mut data = vec![0; 1024];
+    ///             // Try to read data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match stream.get_ref().read(&mut data) {
+    ///                 Ok(n) => {
+    ///                     println!("read {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a read has blocked, but a write might still succeed.
+    ///                     // clear only the read readiness.
+    ///                     guard.clear_ready_matching(Ready::READABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///
+    ///         if guard.ready().is_writable() {
+    ///             // Try to write data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match stream.get_ref().write(b"hello world") {
+    ///                 Ok(n) => {
+    ///                     println!("write {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a write has blocked, but a read might still succeed.
+    ///                     // clear only the write readiness.
+    ///                     guard.clear_ready_matching(Ready::WRITABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    pub fn clear_ready_matching(&mut self, ready: Ready) {
+        if let Some(mut event) = self.event.take() {
+            self.async_fd
+                .registration
+                .clear_readiness(event.with_ready(ready));
+
+            // the event is no longer ready for the readiness that was just cleared
+            event.ready = event.ready - ready;
+
+            if !event.ready.is_empty() {
+                self.event = Some(event);
+            }
+        }
+    }
+
     /// This method should be invoked when you intentionally want to keep the
     /// ready flag asserted.
     ///
@@ -666,6 +939,20 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> {
         // no-op
     }
 
+    /// Get the [`Ready`] value associated with this guard.
+    ///
+    /// This method will return the empty readiness state if
+    /// [`AsyncFdReadyGuard::clear_ready`] has been called on
+    /// the guard.
+    ///
+    /// [`Ready`]: crate::io::Ready
+    pub fn ready(&self) -> Ready {
+        match &self.event {
+            Some(event) => event.ready,
+            None => Ready::EMPTY,
+        }
+    }
+
     /// Performs the provided IO operation.
     ///
     /// If `f` returns a [`WouldBlock`] error, the readiness state associated
@@ -751,22 +1038,117 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> {
 }
 
 impl<'a, Inner: AsRawFd> AsyncFdReadyMutGuard<'a, Inner> {
-    /// Indicates to tokio that the file descriptor is no longer ready. The
-    /// internal readiness flag will be cleared, and tokio will wait for the
+    /// Indicates to tokio that the file descriptor is no longer ready. All
+    /// internal readiness flags will be cleared, and tokio will wait for the
     /// next edge-triggered readiness notification from the OS.
     ///
+    /// This function is commonly used with guards returned by [`AsyncFd::readable_mut`] and
+    /// [`AsyncFd::writable_mut`].
+    ///
     /// It is critical that this function not be called unless your code
     /// _actually observes_ that the file descriptor is _not_ ready. Do not call
     /// it simply because, for example, a read succeeded; it should be called
     /// when a read is observed to block.
-    ///
-    /// [`drop`]: method@std::mem::drop
     pub fn clear_ready(&mut self) {
         if let Some(event) = self.event.take() {
             self.async_fd.registration.clear_readiness(event);
         }
     }
 
+    /// Indicates to tokio that the file descriptor no longer has a specific readiness.
+    /// The internal readiness flag will be cleared, and tokio will wait for the
+    /// next edge-triggered readiness notification from the OS.
+    ///
+    /// This function is useful in combination with the [`AsyncFd::ready_mut`] method when a
+    /// combined interest like `Interest::READABLE | Interest::WRITABLE` is used.
+    ///
+    /// It is critical that this function not be called unless your code
+    /// _actually observes_ that the file descriptor is _not_ ready for the provided `Ready`.
+    /// Do not call it simply because, for example, a read succeeded; it should be called
+    /// when a read is observed to block. Only clear the specific readiness that is observed to
+    /// block. For example when a read blocks when using a combined interest,
+    /// only clear `Ready::READABLE`.
+    ///
+    /// # Examples
+    ///
+    /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without
+    /// splitting.
+    ///
+    /// ```no_run
+    /// use std::error::Error;
+    /// use std::io;
+    /// use std::io::{Read, Write};
+    /// use std::net::TcpStream;
+    /// use tokio::io::unix::AsyncFd;
+    /// use tokio::io::{Interest, Ready};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn Error>> {
+    ///     let stream = TcpStream::connect("127.0.0.1:8080")?;
+    ///     stream.set_nonblocking(true)?;
+    ///     let mut stream = AsyncFd::new(stream)?;
+    ///
+    ///     loop {
+    ///         let mut guard = stream
+    ///             .ready_mut(Interest::READABLE | Interest::WRITABLE)
+    ///             .await?;
+    ///
+    ///         if guard.ready().is_readable() {
+    ///             let mut data = vec![0; 1024];
+    ///             // Try to read data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match guard.get_inner_mut().read(&mut data) {
+    ///                 Ok(n) => {
+    ///                     println!("read {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a read has blocked, but a write might still succeed.
+    ///                     // clear only the read readiness.
+    ///                     guard.clear_ready_matching(Ready::READABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///
+    ///         if guard.ready().is_writable() {
+    ///             // Try to write data, this may still fail with `WouldBlock`
+    ///             // if the readiness event is a false positive.
+    ///             match guard.get_inner_mut().write(b"hello world") {
+    ///                 Ok(n) => {
+    ///                     println!("write {} bytes", n);
+    ///                 }
+    ///                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+    ///                     // a write has blocked, but a read might still succeed.
+    ///                     // clear only the write readiness.
+    ///                     guard.clear_ready_matching(Ready::WRITABLE);
+    ///                     continue;
+    ///                 }
+    ///                 Err(e) => {
+    ///                     return Err(e.into());
+    ///                 }
+    ///             }
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    pub fn clear_ready_matching(&mut self, ready: Ready) {
+        if let Some(mut event) = self.event.take() {
+            self.async_fd
+                .registration
+                .clear_readiness(event.with_ready(ready));
+
+            // the event is no longer ready for the readiness that was just cleared
+            event.ready = event.ready - ready;
+
+            if !event.ready.is_empty() {
+                self.event = Some(event);
+            }
+        }
+    }
+
     /// This method should be invoked when you intentionally want to keep the
     /// ready flag asserted.
     ///
@@ -776,6 +1158,20 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyMutGuard<'a, Inner> {
         // no-op
     }
 
+    /// Get the [`Ready`] value associated with this guard.
+    ///
+    /// This method will return the empty readiness state if
+    /// [`AsyncFdReadyGuard::clear_ready`] has been called on
+    /// the guard.
+    ///
+    /// [`Ready`]: super::Ready
+    pub fn ready(&self) -> Ready {
+        match &self.event {
+            Some(event) => event.ready,
+            None => Ready::EMPTY,
+        }
+    }
+
     /// Performs the provided IO operation.
     ///
     /// If `f` returns a [`WouldBlock`] error, the readiness state associated

tokio/src/io/interest.rs

@@ -44,6 +44,11 @@ impl Interest {
     /// Writable interest includes write-closed events.
     pub const WRITABLE: Interest = Interest(mio::Interest::WRITABLE);
 
+    /// Returns a `Interest` set representing priority completion interests.
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
+    pub const PRIORITY: Interest = Interest(mio::Interest::PRIORITY);
+
     /// Returns true if the value includes readable interest.
     ///
     /// # Examples
@@ -78,6 +83,25 @@ impl Interest {
         self.0.is_writable()
     }
 
+    /// Returns true if the value includes priority interest.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::io::Interest;
+    ///
+    /// assert!(!Interest::READABLE.is_priority());
+    /// assert!(Interest::PRIORITY.is_priority());
+    ///
+    /// let both = Interest::READABLE | Interest::PRIORITY;
+    /// assert!(both.is_priority());
+    /// ```
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
+    pub const fn is_priority(self) -> bool {
+        self.0.is_priority()
+    }
+
     /// Add together two `Interest` values.
     ///
     /// This function works from a `const` context.
@@ -104,6 +128,8 @@ impl Interest {
         match self {
             Interest::READABLE => Ready::READABLE | Ready::READ_CLOSED,
             Interest::WRITABLE => Ready::WRITABLE | Ready::WRITE_CLOSED,
+            #[cfg(any(target_os = "linux", target_os = "android"))]
+            Interest::PRIORITY => Ready::PRIORITY | Ready::READ_CLOSED,
             _ => Ready::EMPTY,
         }
     }

tokio/src/io/poll_evented.rs

@@ -73,6 +73,9 @@ cfg_io_driver! {
 impl<E: Source> PollEvented<E> {
     /// Creates a new `PollEvented` associated with the default reactor.
     ///
+    /// The returned `PollEvented` has readable and writable interests. For more control, use
+    /// [`Self::new_with_interest`].
+    ///
     /// # Panics
     ///
     /// This function panics if thread-local runtime is not set.

tokio/src/io/ready.rs

@@ -7,6 +7,8 @@ const READABLE: usize = 0b0_01;
 const WRITABLE: usize = 0b0_10;
 const READ_CLOSED: usize = 0b0_0100;
 const WRITE_CLOSED: usize = 0b0_1000;
+#[cfg(any(target_os = "linux", target_os = "android"))]
+const PRIORITY: usize = 0b1_0000;
 
 /// Describes the readiness state of an I/O resources.
 ///
@@ -31,7 +33,17 @@ impl Ready {
     /// Returns a `Ready` representing write closed readiness.
     pub const WRITE_CLOSED: Ready = Ready(WRITE_CLOSED);
 
+    /// Returns a `Ready` representing priority readiness.
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
+    pub const PRIORITY: Ready = Ready(PRIORITY);
+
     /// Returns a `Ready` representing readiness for all operations.
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED | PRIORITY);
+
+    /// Returns a `Ready` representing readiness for all operations.
+    #[cfg(not(any(target_os = "linux", target_os = "android")))]
     pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED);
 
     // Must remain crate-private to avoid adding a public dependency on Mio.
@@ -65,6 +77,13 @@ impl Ready {
             ready |= Ready::WRITE_CLOSED;
         }
 
+        #[cfg(any(target_os = "linux", target_os = "android"))]
+        {
+            if event.is_priority() {
+                ready |= Ready::PRIORITY;
+            }
+        }
+
         ready
     }
 
@@ -144,6 +163,23 @@ impl Ready {
         self.contains(Ready::WRITE_CLOSED)
     }
 
+    /// Returns `true` if the value includes priority `readiness`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::io::Ready;
+    ///
+    /// assert!(!Ready::EMPTY.is_priority());
+    /// assert!(!Ready::WRITABLE.is_priority());
+    /// assert!(Ready::PRIORITY.is_priority());
+    /// ```
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
+    pub fn is_priority(self) -> bool {
+        self.contains(Ready::PRIORITY)
+    }
+
     /// Returns true if `self` is a superset of `other`.
     ///
     /// `other` may represent more than one readiness operations, in which case
@@ -191,6 +227,12 @@ cfg_io_readiness! {
                 ready |= Ready::WRITE_CLOSED;
             }
 
+            #[cfg(any(target_os = "linux", target_os = "android"))]
+            if interest.is_priority() {
+                ready |= Ready::PRIORITY;
+                ready |= Ready::READ_CLOSED;
+            }
+
             ready
         }
 
@@ -240,11 +282,16 @@ impl ops::Sub<Ready> for Ready {
 
 impl fmt::Debug for Ready {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt.debug_struct("Ready")
-            .field("is_readable", &self.is_readable())
+        let mut fmt = fmt.debug_struct("Ready");
+
+        fmt.field("is_readable", &self.is_readable())
             .field("is_writable", &self.is_writable())
             .field("is_read_closed", &self.is_read_closed())
-            .field("is_write_closed", &self.is_write_closed())
-            .finish()
+            .field("is_write_closed", &self.is_write_closed());
+
+        #[cfg(any(target_os = "linux", target_os = "android"))]
+        fmt.field("is_priority", &self.is_priority());
+
+        fmt.finish()
     }
 }

tokio/src/io/util/async_buf_read_ext.rs

@@ -146,7 +146,7 @@ cfg_io_util! {
         ///    [`next_line`] method.
         ///  * Use [`tokio_util::codec::LinesCodec`][LinesCodec].
         ///
-        /// [LinesCodec]: https://docs.rs/tokio-util/0.6/tokio_util/codec/struct.LinesCodec.html
+        /// [LinesCodec]: https://docs.rs/tokio-util/latest/tokio_util/codec/struct.LinesCodec.html
         /// [`read_until`]: Self::read_until
         /// [`lines`]: Self::lines
         /// [`next_line`]: crate::io::Lines::next_line

tokio/src/io/util/empty.rs

@@ -53,6 +53,7 @@ impl AsyncRead for Empty {
         cx: &mut Context<'_>,
         _: &mut ReadBuf<'_>,
     ) -> Poll<io::Result<()>> {
+        ready!(crate::trace::trace_leaf(cx));
         ready!(poll_proceed_and_make_progress(cx));
         Poll::Ready(Ok(()))
     }
@@ -61,6 +62,7 @@ impl AsyncRead for Empty {
 impl AsyncBufRead for Empty {
     #[inline]
     fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
+        ready!(crate::trace::trace_leaf(cx));
         ready!(poll_proceed_and_make_progress(cx));
         Poll::Ready(Ok(&[]))
     }

tokio/src/io/util/mem.rs

@@ -233,6 +233,7 @@ impl AsyncRead for Pipe {
             cx: &mut task::Context<'_>,
             buf: &mut ReadBuf<'_>,
         ) -> Poll<std::io::Result<()>> {
+            ready!(crate::trace::trace_leaf(cx));
             let coop = ready!(crate::runtime::coop::poll_proceed(cx));
 
             let ret = self.poll_read_internal(cx, buf);
@@ -249,6 +250,7 @@ impl AsyncRead for Pipe {
             cx: &mut task::Context<'_>,
             buf: &mut ReadBuf<'_>,
         ) -> Poll<std::io::Result<()>> {
+            ready!(crate::trace::trace_leaf(cx));
             self.poll_read_internal(cx, buf)
         }
     }
@@ -261,6 +263,7 @@ impl AsyncWrite for Pipe {
             cx: &mut task::Context<'_>,
             buf: &[u8],
         ) -> Poll<std::io::Result<usize>> {
+            ready!(crate::trace::trace_leaf(cx));
             let coop = ready!(crate::runtime::coop::poll_proceed(cx));
 
             let ret = self.poll_write_internal(cx, buf);
@@ -277,6 +280,7 @@ impl AsyncWrite for Pipe {
             cx: &mut task::Context<'_>,
             buf: &[u8],
         ) -> Poll<std::io::Result<usize>> {
+            ready!(crate::trace::trace_leaf(cx));
             self.poll_write_internal(cx, buf)
         }
     }

tokio/src/lib.rs

@@ -487,6 +487,21 @@ compile_error!("Tokio's build script has incorrectly detected wasm.");
 ))]
 compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm.");
 
+#[cfg(all(not(tokio_unstable), tokio_taskdump))]
+compile_error!("The `tokio_taskdump` feature requires `--cfg tokio_unstable`.");
+
+#[cfg(all(
+    tokio_taskdump,
+    not(all(
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))
+))]
+compile_error!(
+    "The `tokio_taskdump` feature is only currently supported on \
+linux, on `aarch64`, `x86` and `x86_64`."
+);
+
 // Includes re-exports used by macros.
 //
 // This module is not intended to be part of the public API. In general, any
@@ -552,6 +567,40 @@ cfg_time! {
     pub mod time;
 }
 
+mod trace {
+    use std::future::Future;
+    use std::pin::Pin;
+    use std::task::{Context, Poll};
+
+    cfg_taskdump! {
+        pub(crate) use crate::runtime::task::trace::trace_leaf;
+    }
+
+    cfg_not_taskdump! {
+        #[inline(always)]
+        #[allow(dead_code)]
+        pub(crate) fn trace_leaf(_: &mut std::task::Context<'_>) -> std::task::Poll<()> {
+            std::task::Poll::Ready(())
+        }
+    }
+
+    #[cfg_attr(not(feature = "sync"), allow(dead_code))]
+    pub(crate) fn async_trace_leaf() -> impl Future<Output = ()> {
+        struct Trace;
+
+        impl Future for Trace {
+            type Output = ();
+
+            #[inline(always)]
+            fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
+                trace_leaf(cx)
+            }
+        }
+
+        Trace
+    }
+}
+
 mod util;
 
 /// Due to the `Stream` trait's inclusion in `std` landing later than Tokio's 1.0

tokio/src/loom/std/barrier.rs

@@ -0,0 +1,217 @@
+//! A `Barrier` that provides `wait_timeout`.
+//!
+//! This implementation mirrors that of the Rust standard library.
+
+use crate::loom::sync::{Condvar, Mutex};
+use std::fmt;
+use std::time::{Duration, Instant};
+
+/// A barrier enables multiple threads to synchronize the beginning
+/// of some computation.
+///
+/// # Examples
+///
+/// ```
+/// use std::sync::{Arc, Barrier};
+/// use std::thread;
+///
+/// let mut handles = Vec::with_capacity(10);
+/// let barrier = Arc::new(Barrier::new(10));
+/// for _ in 0..10 {
+///     let c = Arc::clone(&barrier);
+///     // The same messages will be printed together.
+///     // You will NOT see any interleaving.
+///     handles.push(thread::spawn(move|| {
+///         println!("before wait");
+///         c.wait();
+///         println!("after wait");
+///     }));
+/// }
+/// // Wait for other threads to finish.
+/// for handle in handles {
+///     handle.join().unwrap();
+/// }
+/// ```
+pub(crate) struct Barrier {
+    lock: Mutex<BarrierState>,
+    cvar: Condvar,
+    num_threads: usize,
+}
+
+// The inner state of a double barrier
+struct BarrierState {
+    count: usize,
+    generation_id: usize,
+}
+
+/// A `BarrierWaitResult` is returned by [`Barrier::wait()`] when all threads
+/// in the [`Barrier`] have rendezvoused.
+///
+/// # Examples
+///
+/// ```
+/// use std::sync::Barrier;
+///
+/// let barrier = Barrier::new(1);
+/// let barrier_wait_result = barrier.wait();
+/// ```
+pub(crate) struct BarrierWaitResult(bool);
+
+impl fmt::Debug for Barrier {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Barrier").finish_non_exhaustive()
+    }
+}
+
+impl Barrier {
+    /// Creates a new barrier that can block a given number of threads.
+    ///
+    /// A barrier will block `n`-1 threads which call [`wait()`] and then wake
+    /// up all threads at once when the `n`th thread calls [`wait()`].
+    ///
+    /// [`wait()`]: Barrier::wait
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::sync::Barrier;
+    ///
+    /// let barrier = Barrier::new(10);
+    /// ```
+    #[must_use]
+    pub(crate) fn new(n: usize) -> Barrier {
+        Barrier {
+            lock: Mutex::new(BarrierState {
+                count: 0,
+                generation_id: 0,
+            }),
+            cvar: Condvar::new(),
+            num_threads: n,
+        }
+    }
+
+    /// Blocks the current thread until all threads have rendezvoused here.
+    ///
+    /// Barriers are re-usable after all threads have rendezvoused once, and can
+    /// be used continuously.
+    ///
+    /// A single (arbitrary) thread will receive a [`BarrierWaitResult`] that
+    /// returns `true` from [`BarrierWaitResult::is_leader()`] when returning
+    /// from this function, and all other threads will receive a result that
+    /// will return `false` from [`BarrierWaitResult::is_leader()`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::sync::{Arc, Barrier};
+    /// use std::thread;
+    ///
+    /// let mut handles = Vec::with_capacity(10);
+    /// let barrier = Arc::new(Barrier::new(10));
+    /// for _ in 0..10 {
+    ///     let c = Arc::clone(&barrier);
+    ///     // The same messages will be printed together.
+    ///     // You will NOT see any interleaving.
+    ///     handles.push(thread::spawn(move|| {
+    ///         println!("before wait");
+    ///         c.wait();
+    ///         println!("after wait");
+    ///     }));
+    /// }
+    /// // Wait for other threads to finish.
+    /// for handle in handles {
+    ///     handle.join().unwrap();
+    /// }
+    /// ```
+    pub(crate) fn wait(&self) -> BarrierWaitResult {
+        let mut lock = self.lock.lock();
+        let local_gen = lock.generation_id;
+        lock.count += 1;
+        if lock.count < self.num_threads {
+            // We need a while loop to guard against spurious wakeups.
+            // https://en.wikipedia.org/wiki/Spurious_wakeup
+            while local_gen == lock.generation_id {
+                lock = self.cvar.wait(lock).unwrap();
+            }
+            BarrierWaitResult(false)
+        } else {
+            lock.count = 0;
+            lock.generation_id = lock.generation_id.wrapping_add(1);
+            self.cvar.notify_all();
+            BarrierWaitResult(true)
+        }
+    }
+
+    /// Blocks the current thread until all threads have rendezvoused here for
+    /// at most `timeout` duration.
+    pub(crate) fn wait_timeout(&self, timeout: Duration) -> Option<BarrierWaitResult> {
+        // This implementation mirrors `wait`, but with each blocking operation
+        // replaced by a timeout-amenable alternative.
+
+        let deadline = Instant::now() + timeout;
+
+        // Acquire `self.lock` with at most `timeout` duration.
+        let mut lock = loop {
+            if let Some(guard) = self.lock.try_lock() {
+                break guard;
+            } else if Instant::now() > deadline {
+                return None;
+            } else {
+                std::thread::yield_now();
+            }
+        };
+
+        // Shrink the `timeout` to account for the time taken to acquire `lock`.
+        let timeout = deadline.saturating_duration_since(Instant::now());
+
+        let local_gen = lock.generation_id;
+        lock.count += 1;
+        if lock.count < self.num_threads {
+            // We need a while loop to guard against spurious wakeups.
+            // https://en.wikipedia.org/wiki/Spurious_wakeup
+            while local_gen == lock.generation_id {
+                let (guard, timeout_result) = self.cvar.wait_timeout(lock, timeout).unwrap();
+                lock = guard;
+                if timeout_result.timed_out() {
+                    return None;
+                }
+            }
+            Some(BarrierWaitResult(false))
+        } else {
+            lock.count = 0;
+            lock.generation_id = lock.generation_id.wrapping_add(1);
+            self.cvar.notify_all();
+            Some(BarrierWaitResult(true))
+        }
+    }
+}
+
+impl fmt::Debug for BarrierWaitResult {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("BarrierWaitResult")
+            .field("is_leader", &self.is_leader())
+            .finish()
+    }
+}
+
+impl BarrierWaitResult {
+    /// Returns `true` if this thread is the "leader thread" for the call to
+    /// [`Barrier::wait()`].
+    ///
+    /// Only one thread will have `true` returned from their result, all other
+    /// threads will have `false` returned.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::sync::Barrier;
+    ///
+    /// let barrier = Barrier::new(1);
+    /// let barrier_wait_result = barrier.wait();
+    /// println!("{:?}", barrier_wait_result.is_leader());
+    /// ```
+    #[must_use]
+    pub(crate) fn is_leader(&self) -> bool {
+        self.0
+    }
+}

tokio/src/loom/std/mod.rs

@@ -4,6 +4,7 @@ mod atomic_u16;
 mod atomic_u32;
 mod atomic_u64;
 mod atomic_usize;
+mod barrier;
 mod mutex;
 #[cfg(feature = "parking_lot")]
 mod parking_lot;
@@ -76,6 +77,8 @@ pub(crate) mod sync {
 
         pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering};
     }
+
+    pub(crate) use super::barrier::Barrier;
 }
 
 pub(crate) mod sys {

tokio/src/loom/std/unsafe_cell.rs

@@ -6,10 +6,12 @@ impl<T> UnsafeCell<T> {
         UnsafeCell(std::cell::UnsafeCell::new(data))
     }
 
+    #[inline(always)]
     pub(crate) fn with<R>(&self, f: impl FnOnce(*const T) -> R) -> R {
         f(self.0.get())
     }
 
+    #[inline(always)]
     pub(crate) fn with_mut<R>(&self, f: impl FnOnce(*mut T) -> R) -> R {
         f(self.0.get())
     }

tokio/src/macros/cfg.rs

@@ -373,6 +373,44 @@ macro_rules! cfg_not_rt_multi_thread {
     }
 }
 
+macro_rules! cfg_taskdump {
+    ($($item:item)*) => {
+        $(
+            #[cfg(all(
+                tokio_unstable,
+                tokio_taskdump,
+                feature = "rt",
+                target_os = "linux",
+                any(
+                    target_arch = "aarch64",
+                    target_arch = "x86",
+                    target_arch = "x86_64"
+                )
+            ))]
+            $item
+        )*
+    };
+}
+
+macro_rules! cfg_not_taskdump {
+    ($($item:item)*) => {
+        $(
+            #[cfg(not(all(
+                tokio_unstable,
+                tokio_taskdump,
+                feature = "rt",
+                target_os = "linux",
+                any(
+                    target_arch = "aarch64",
+                    target_arch = "x86",
+                    target_arch = "x86_64"
+                )
+            )))]
+            $item
+        )*
+    };
+}
+
 macro_rules! cfg_test_util {
     ($($item:item)*) => {
         $(

tokio/src/macros/loom.rs

@@ -1,11 +1,7 @@
 macro_rules! if_loom {
     ($($t:tt)*) => {{
         #[cfg(loom)]
-        const LOOM: bool = true;
-        #[cfg(not(loom))]
-        const LOOM: bool = false;
-
-        if LOOM {
+        {
             $($t)*
         }
     }}

tokio/src/macros/mod.rs

@@ -23,10 +23,6 @@ cfg_trace! {
     mod trace;
 }
 
-#[macro_use]
-#[cfg(feature = "rt")]
-pub(crate) mod scoped_tls;
-
 cfg_macros! {
     #[macro_use]
     mod select;

tokio/src/macros/scoped_tls.rs

@@ -1,77 +0,0 @@
-use crate::loom::thread::LocalKey;
-
-use std::cell::Cell;
-use std::marker;
-
-/// Sets a reference as a thread-local.
-macro_rules! scoped_thread_local {
-    ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty) => (
-        $(#[$attrs])*
-        $vis static $name: $crate::macros::scoped_tls::ScopedKey<$ty>
-            = $crate::macros::scoped_tls::ScopedKey {
-                inner: {
-                    tokio_thread_local!(static FOO: ::std::cell::Cell<*const ()> = const {
-                        std::cell::Cell::new(::std::ptr::null())
-                    });
-                    &FOO
-                },
-                _marker: ::std::marker::PhantomData,
-            };
-    )
-}
-
-/// Type representing a thread local storage key corresponding to a reference
-/// to the type parameter `T`.
-pub(crate) struct ScopedKey<T> {
-    pub(crate) inner: &'static LocalKey<Cell<*const ()>>,
-    pub(crate) _marker: marker::PhantomData<T>,
-}
-
-unsafe impl<T> Sync for ScopedKey<T> {}
-
-impl<T> ScopedKey<T> {
-    /// Inserts a value into this scoped thread local storage slot for a
-    /// duration of a closure.
-    pub(crate) fn set<F, R>(&'static self, t: &T, f: F) -> R
-    where
-        F: FnOnce() -> R,
-    {
-        struct Reset {
-            key: &'static LocalKey<Cell<*const ()>>,
-            val: *const (),
-        }
-
-        impl Drop for Reset {
-            fn drop(&mut self) {
-                self.key.with(|c| c.set(self.val));
-            }
-        }
-
-        let prev = self.inner.with(|c| {
-            let prev = c.get();
-            c.set(t as *const _ as *const ());
-            prev
-        });
-
-        let _reset = Reset {
-            key: self.inner,
-            val: prev,
-        };
-
-        f()
-    }
-
-    /// Gets a value out of this scoped variable.
-    pub(crate) fn with<F, R>(&'static self, f: F) -> R
-    where
-        F: FnOnce(Option<&T>) -> R,
-    {
-        let val = self.inner.with(|c| c.get());
-
-        if val.is_null() {
-            f(None)
-        } else {
-            unsafe { f(Some(&*(val as *const T))) }
-        }
-    }
-}

tokio/src/net/tcp/socket.rs

@@ -404,6 +404,51 @@ impl TcpSocket {
         self.inner.linger()
     }
 
+    /// Sets the value of the `TCP_NODELAY` option on this socket.
+    ///
+    /// If set, this option disables the Nagle algorithm. This means that segments are always
+    /// sent as soon as possible, even if there is only a small amount of data. When not set,
+    /// data is buffered until there is a sufficient amount to send out, thereby avoiding
+    /// the frequent sending of small packets.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use tokio::net::TcpSocket;
+    ///
+    /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
+    /// let socket = TcpSocket::new_v4()?;
+    ///
+    /// println!("{:?}", socket.nodelay()?);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> {
+        self.inner.set_nodelay(nodelay)
+    }
+
+    /// Gets the value of the `TCP_NODELAY` option on this socket.
+    ///
+    /// For more information about this option, see [`set_nodelay`].
+    ///
+    /// [`set_nodelay`]: TcpSocket::set_nodelay
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use tokio::net::TcpSocket;
+    ///
+    /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
+    /// let stream = TcpSocket::new_v4()?;
+    ///
+    /// stream.set_nodelay(true)?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn nodelay(&self) -> io::Result<bool> {
+        self.inner.nodelay()
+    }
+
     /// Gets the value of the `IP_TOS` option for this socket.
     ///
     /// For more information about this option, see [`set_tos`].

tokio/src/net/unix/datagram/socket.rs

@@ -91,6 +91,7 @@ cfg_net_unix! {
     /// # Ok(())
     /// # }
     /// ```
+    #[cfg_attr(docsrs, doc(alias = "uds"))]
     pub struct UnixDatagram {
         io: PollEvented<mio::net::UnixDatagram>,
     }

tokio/src/net/unix/listener.rs

@@ -45,6 +45,7 @@ cfg_net_unix! {
     ///     }
     /// }
     /// ```
+    #[cfg_attr(docsrs, doc(alias = "uds"))]
     pub struct UnixListener {
         io: PollEvented<mio::net::UnixListener>,
     }

tokio/src/net/unix/stream.rs

@@ -34,6 +34,7 @@ cfg_net_unix! {
     ///
     /// [`shutdown()`]: fn@crate::io::AsyncWriteExt::shutdown
     /// [`UnixListener::accept`]: crate::net::UnixListener::accept
+    #[cfg_attr(docsrs, doc(alias = "uds"))]
     pub struct UnixStream {
         io: PollEvented<mio::net::UnixStream>,
     }

tokio/src/net/unix/ucred.rs

@@ -31,7 +31,12 @@ impl UCred {
     }
 }
 
-#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))]
+#[cfg(any(
+    target_os = "linux",
+    target_os = "redox",
+    target_os = "android",
+    target_os = "openbsd"
+))]
 pub(crate) use self::impl_linux::get_peer_cred;
 
 #[cfg(any(target_os = "netbsd"))]
@@ -49,7 +54,12 @@ pub(crate) use self::impl_solaris::get_peer_cred;
 #[cfg(target_os = "aix")]
 pub(crate) use self::impl_aix::get_peer_cred;
 
-#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))]
+#[cfg(any(
+    target_os = "linux",
+    target_os = "redox",
+    target_os = "android",
+    target_os = "openbsd"
+))]
 pub(crate) mod impl_linux {
     use crate::net::unix::{self, UnixStream};
 
@@ -58,7 +68,7 @@ pub(crate) mod impl_linux {
 
     #[cfg(target_os = "openbsd")]
     use libc::sockpeercred as ucred;
-    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[cfg(any(target_os = "linux", target_os = "redox", target_os = "android"))]
     use libc::ucred;
 
     pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result<super::UCred> {

tokio/src/process/mod.rs

@@ -400,6 +400,22 @@ impl Command {
         self
     }
 
+    /// Append literal text to the command line without any quoting or escaping.
+    ///
+    /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow
+    /// `CommandLineToArgvW` escaping rules.
+    ///
+    /// **Note**: This is an [unstable API][unstable] but will be stabilised once
+    /// tokio's MSRV is sufficiently new. See [the documentation on
+    /// unstable features][unstable] for details about using unstable features.
+    #[cfg(windows)]
+    #[cfg(tokio_unstable)]
+    #[cfg_attr(docsrs, doc(cfg(all(windows, tokio_unstable))))]
+    pub fn raw_arg<S: AsRef<OsStr>>(&mut self, text_to_append_as_is: S) -> &mut Command {
+        self.std.raw_arg(text_to_append_as_is);
+        self
+    }
+
     /// Inserts or updates an environment variable mapping.
     ///
     /// Note that environment variable names are case-insensitive (but case-preserving) on Windows,
@@ -995,6 +1011,7 @@ where
     type Output = Result<T, E>;
 
     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        ready!(crate::trace::trace_leaf(cx));
         // Keep track of task budget
         let coop = ready!(crate::runtime::coop::poll_proceed(cx));
 

tokio/src/runtime/blocking/schedule.rs

@@ -25,6 +25,8 @@ impl BlockingSchedule {
                 }
                 #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                 scheduler::Handle::MultiThread(_) => {}
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThreadAlt(_) => {}
             }
         }
         BlockingSchedule {
@@ -45,6 +47,8 @@ impl task::Schedule for BlockingSchedule {
                 }
                 #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                 scheduler::Handle::MultiThread(_) => {}
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThreadAlt(_) => {}
             }
         }
         None

tokio/src/runtime/builder.rs

@@ -1,5 +1,5 @@
 use crate::runtime::handle::Handle;
-use crate::runtime::{blocking, driver, Callback, Runtime};
+use crate::runtime::{blocking, driver, Callback, HistogramBuilder, Runtime};
 use crate::util::rand::{RngSeed, RngSeedGenerator};
 
 use std::fmt;
@@ -82,7 +82,13 @@ pub struct Builder {
     pub(super) keep_alive: Option<Duration>,
 
     /// How many ticks before pulling a task from the global/remote queue?
-    pub(super) global_queue_interval: u32,
+    ///
+    /// When `None`, the value is unspecified and behavior details are left to
+    /// the scheduler. Each scheduler flavor could choose to either pick its own
+    /// default value or use some other strategy to decide when to poll from the
+    /// global queue. For example, the multi-threaded scheduler uses a
+    /// self-tuning strategy based on mean task poll times.
+    pub(super) global_queue_interval: Option<u32>,
 
     /// How many ticks before yielding to the driver for timer and I/O events?
     pub(super) event_interval: u32,
@@ -95,6 +101,12 @@ pub struct Builder {
     /// Specify a random number generator seed to provide deterministic results
     pub(super) seed_generator: RngSeedGenerator,
 
+    /// When true, enables task poll count histogram instrumentation.
+    pub(super) metrics_poll_count_histogram_enable: bool,
+
+    /// Configures the task poll count histogram
+    pub(super) metrics_poll_count_histogram: HistogramBuilder,
+
     #[cfg(tokio_unstable)]
     pub(super) unhandled_panic: UnhandledPanic,
 }
@@ -187,6 +199,8 @@ pub(crate) enum Kind {
     CurrentThread,
     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
     MultiThread,
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+    MultiThreadAlt,
 }
 
 impl Builder {
@@ -205,7 +219,7 @@ impl Builder {
         #[cfg(not(loom))]
         const EVENT_INTERVAL: u32 = 61;
 
-        Builder::new(Kind::CurrentThread, 31, EVENT_INTERVAL)
+        Builder::new(Kind::CurrentThread, EVENT_INTERVAL)
     }
 
     cfg_not_wasi! {
@@ -216,7 +230,27 @@ impl Builder {
         #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
         pub fn new_multi_thread() -> Builder {
             // The number `61` is fairly arbitrary. I believe this value was copied from golang.
-            Builder::new(Kind::MultiThread, 61, 61)
+            Builder::new(Kind::MultiThread, 61)
+        }
+
+        cfg_unstable! {
+            /// Returns a new builder with the alternate multi thread scheduler
+            /// selected.
+            ///
+            /// The alternate multi threaded scheduler is an in-progress
+            /// candidate to replace the existing multi threaded scheduler. It
+            /// currently does not scale as well to 16+ processors.
+            ///
+            /// This runtime flavor is currently **not considered production
+            /// ready**.
+            ///
+            /// Configuration methods can be chained on the return value.
+            #[cfg(feature = "rt-multi-thread")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
+            pub fn new_multi_thread_alt() -> Builder {
+                // The number `61` is fairly arbitrary. I believe this value was copied from golang.
+                Builder::new(Kind::MultiThreadAlt, 61)
+            }
         }
     }
 
@@ -224,7 +258,7 @@ impl Builder {
     /// values.
     ///
     /// Configuration methods can be chained on the return value.
-    pub(crate) fn new(kind: Kind, global_queue_interval: u32, event_interval: u32) -> Builder {
+    pub(crate) fn new(kind: Kind, event_interval: u32) -> Builder {
         Builder {
             kind,
 
@@ -260,7 +294,7 @@ impl Builder {
 
             // Defaults for these values depend on the scheduler kind, so we get them
             // as parameters.
-            global_queue_interval,
+            global_queue_interval: None,
             event_interval,
 
             seed_generator: RngSeedGenerator::new(RngSeed::new()),
@@ -268,6 +302,10 @@ impl Builder {
             #[cfg(tokio_unstable)]
             unhandled_panic: UnhandledPanic::Ignore,
 
+            metrics_poll_count_histogram_enable: false,
+
+            metrics_poll_count_histogram: Default::default(),
+
             disable_lifo_slot: false,
         }
     }
@@ -640,6 +678,8 @@ impl Builder {
             Kind::CurrentThread => self.build_current_thread_runtime(),
             #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
             Kind::MultiThread => self.build_threaded_runtime(),
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+            Kind::MultiThreadAlt => self.build_alt_threaded_runtime(),
         }
     }
 
@@ -649,6 +689,8 @@ impl Builder {
                 Kind::CurrentThread => true,
                 #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                 Kind::MultiThread => false,
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                Kind::MultiThreadAlt => false,
             },
             enable_io: self.enable_io,
             enable_time: self.enable_time,
@@ -706,7 +748,7 @@ impl Builder {
     /// # }
     /// ```
     pub fn global_queue_interval(&mut self, val: u32) -> &mut Self {
-        self.global_queue_interval = val;
+        self.global_queue_interval = Some(val);
         self
     }
 
@@ -877,6 +919,133 @@ impl Builder {
         }
     }
 
+    cfg_metrics! {
+        /// Enables tracking the distribution of task poll times.
+        ///
+        /// Task poll times are not instrumented by default as doing so requires
+        /// calling [`Instant::now()`] twice per task poll, which could add
+        /// measurable overhead. Use the [`Handle::metrics()`] to access the
+        /// metrics data.
+        ///
+        /// The histogram uses fixed bucket sizes. In other words, the histogram
+        /// buckets are not dynamic based on input values. Use the
+        /// `metrics_poll_count_histogram_` builder methods to configure the
+        /// histogram details.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .build()
+        ///     .unwrap();
+        /// # // Test default values here
+        /// # fn us(n: u64) -> std::time::Duration { std::time::Duration::from_micros(n) }
+        /// # let m = rt.handle().metrics();
+        /// # assert_eq!(m.poll_count_histogram_num_buckets(), 10);
+        /// # assert_eq!(m.poll_count_histogram_bucket_range(0), us(0)..us(100));
+        /// # assert_eq!(m.poll_count_histogram_bucket_range(1), us(100)..us(200));
+        /// ```
+        ///
+        /// [`Handle::metrics()`]: crate::runtime::Handle::metrics
+        /// [`Instant::now()`]: std::time::Instant::now
+        pub fn enable_metrics_poll_count_histogram(&mut self) -> &mut Self {
+            self.metrics_poll_count_histogram_enable = true;
+            self
+        }
+
+        /// Sets the histogram scale for tracking the distribution of task poll
+        /// times.
+        ///
+        /// Tracking the distribution of task poll times can be done using a
+        /// linear or log scale. When using linear scale, each histogram bucket
+        /// will represent the same range of poll times. When using log scale,
+        /// each histogram bucket will cover a range twice as big as the
+        /// previous bucket.
+        ///
+        /// **Default:** linear scale.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::{self, HistogramScale};
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .metrics_poll_count_histogram_scale(HistogramScale::Log)
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn metrics_poll_count_histogram_scale(&mut self, histogram_scale: crate::runtime::HistogramScale) -> &mut Self {
+            self.metrics_poll_count_histogram.scale = histogram_scale;
+            self
+        }
+
+        /// Sets the histogram resolution for tracking the distribution of task
+        /// poll times.
+        ///
+        /// The resolution is the histogram's first bucket's range. When using a
+        /// linear histogram scale, each bucket will cover the same range. When
+        /// using a log scale, each bucket will cover a range twice as big as
+        /// the previous bucket. In the log case, the resolution represents the
+        /// smallest bucket range.
+        ///
+        /// Note that, when using log scale, the resolution is rounded up to the
+        /// nearest power of 2 in nanoseconds.
+        ///
+        /// **Default:** 100 microseconds.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        /// use std::time::Duration;
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .metrics_poll_count_histogram_resolution(Duration::from_micros(100))
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn metrics_poll_count_histogram_resolution(&mut self, resolution: Duration) -> &mut Self {
+            assert!(resolution > Duration::from_secs(0));
+            // Sanity check the argument and also make the cast below safe.
+            assert!(resolution <= Duration::from_secs(1));
+
+            let resolution = resolution.as_nanos() as u64;
+            self.metrics_poll_count_histogram.resolution = resolution;
+            self
+        }
+
+        /// Sets the number of buckets for the histogram tracking the
+        /// distribution of task poll times.
+        ///
+        /// The last bucket tracks all greater values that fall out of other
+        /// ranges. So, configuring the histogram using a linear scale,
+        /// resolution of 50ms, and 10 buckets, the 10th bucket will track task
+        /// polls that take more than 450ms to complete.
+        ///
+        /// **Default:** 10
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        ///
+        /// let rt = runtime::Builder::new_multi_thread()
+        ///     .enable_metrics_poll_count_histogram()
+        ///     .metrics_poll_count_histogram_buckets(15)
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn metrics_poll_count_histogram_buckets(&mut self, buckets: usize) -> &mut Self {
+            self.metrics_poll_count_histogram.num_buckets = buckets;
+            self
+        }
+    }
+
     fn build_current_thread_runtime(&mut self) -> io::Result<Runtime> {
         use crate::runtime::scheduler::{self, CurrentThread};
         use crate::runtime::{runtime::Scheduler, Config};
@@ -909,6 +1078,7 @@ impl Builder {
                 unhandled_panic: self.unhandled_panic.clone(),
                 disable_lifo_slot: self.disable_lifo_slot,
                 seed_generator: seed_generator_1,
+                metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
             },
         );
 
@@ -922,6 +1092,14 @@ impl Builder {
             blocking_pool,
         ))
     }
+
+    fn metrics_poll_count_histogram_builder(&self) -> Option<HistogramBuilder> {
+        if self.metrics_poll_count_histogram_enable {
+            Some(self.metrics_poll_count_histogram.clone())
+        } else {
+            None
+        }
+    }
 }
 
 cfg_io_driver! {
@@ -1050,6 +1228,7 @@ cfg_rt_multi_thread! {
                     unhandled_panic: self.unhandled_panic.clone(),
                     disable_lifo_slot: self.disable_lifo_slot,
                     seed_generator: seed_generator_1,
+                    metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
                 },
             );
 
@@ -1061,6 +1240,48 @@ cfg_rt_multi_thread! {
 
             Ok(Runtime::from_parts(Scheduler::MultiThread(scheduler), handle, blocking_pool))
         }
+
+        cfg_unstable! {
+            fn build_alt_threaded_runtime(&mut self) -> io::Result<Runtime> {
+                use crate::loom::sys::num_cpus;
+                use crate::runtime::{Config, runtime::Scheduler};
+                use crate::runtime::scheduler::MultiThreadAlt;
+
+                let core_threads = self.worker_threads.unwrap_or_else(num_cpus);
+
+                let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?;
+
+                // Create the blocking pool
+                let blocking_pool =
+                    blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads);
+                let blocking_spawner = blocking_pool.spawner().clone();
+
+                // Generate a rng seed for this runtime.
+                let seed_generator_1 = self.seed_generator.next_generator();
+                let seed_generator_2 = self.seed_generator.next_generator();
+
+                let (scheduler, handle) = MultiThreadAlt::new(
+                    core_threads,
+                    driver,
+                    driver_handle,
+                    blocking_spawner,
+                    seed_generator_2,
+                    Config {
+                        before_park: self.before_park.clone(),
+                        after_unpark: self.after_unpark.clone(),
+                        global_queue_interval: self.global_queue_interval,
+                        event_interval: self.event_interval,
+                        #[cfg(tokio_unstable)]
+                        unhandled_panic: self.unhandled_panic.clone(),
+                        disable_lifo_slot: self.disable_lifo_slot,
+                        seed_generator: seed_generator_1,
+                        metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
+                    },
+                );
+
+                Ok(Runtime::from_parts(Scheduler::MultiThreadAlt(scheduler), handle, blocking_pool))
+            }
+        }
     }
 }
 

tokio/src/runtime/config.rs

@@ -4,7 +4,7 @@ use crate::util::RngSeedGenerator;
 
 pub(crate) struct Config {
     /// How many ticks before pulling a task from the global/remote queue?
-    pub(crate) global_queue_interval: u32,
+    pub(crate) global_queue_interval: Option<u32>,
 
     /// How many ticks before yielding to the driver for timer and I/O events?
     pub(crate) event_interval: u32,
@@ -28,6 +28,9 @@ pub(crate) struct Config {
     /// deterministic way.
     pub(crate) seed_generator: RngSeedGenerator,
 
+    /// How to build poll time histograms
+    pub(crate) metrics_poll_count_histogram: Option<crate::runtime::HistogramBuilder>,
+
     #[cfg(tokio_unstable)]
     /// How to respond to unhandled task panics.
     pub(crate) unhandled_panic: crate::runtime::UnhandledPanic,

tokio/src/runtime/context.rs

@@ -4,14 +4,33 @@ use crate::runtime::coop;
 use std::cell::Cell;
 
 #[cfg(any(feature = "rt", feature = "macros"))]
-use crate::util::rand::{FastRand, RngSeed};
+use crate::util::rand::FastRand;
 
 cfg_rt! {
-    use crate::runtime::{scheduler, task::Id, Defer};
+    mod blocking;
+    pub(crate) use blocking::{disallow_block_in_place, try_enter_blocking_region, BlockingRegionGuard};
 
-    use std::cell::RefCell;
-    use std::marker::PhantomData;
-    use std::time::Duration;
+    mod current;
+    pub(crate) use current::{with_current, try_set_current, SetCurrentGuard};
+
+    mod runtime;
+    pub(crate) use runtime::{EnterRuntime, enter_runtime};
+
+    mod scoped;
+    use scoped::Scoped;
+
+    use crate::runtime::{scheduler, task::Id};
+
+    use std::task::Waker;
+
+    cfg_taskdump! {
+        use crate::runtime::task::trace;
+    }
+}
+
+cfg_rt_multi_thread! {
+    mod runtime_mt;
+    pub(crate) use runtime_mt::{current_enter_context, exit_runtime};
 }
 
 struct Context {
@@ -21,7 +40,11 @@ struct Context {
 
     /// Handle to the runtime scheduler running on the current thread.
     #[cfg(feature = "rt")]
-    handle: RefCell<Option<scheduler::Handle>>,
+    current: current::HandleCell,
+
+    /// Handle to the scheduler's internal "context"
+    #[cfg(feature = "rt")]
+    scheduler: Scoped<scheduler::Context>,
 
     #[cfg(feature = "rt")]
     current_task_id: Cell<Option<Id>>,
@@ -34,21 +57,25 @@ struct Context {
     #[cfg(feature = "rt")]
     runtime: Cell<EnterRuntime>,
 
-    /// Yielded task wakers are stored here and notified after resource drivers
-    /// are polled.
-    #[cfg(feature = "rt")]
-    defer: RefCell<Option<Defer>>,
-
     #[cfg(any(feature = "rt", feature = "macros"))]
-    rng: FastRand,
+    rng: Cell<Option<FastRand>>,
 
     /// Tracks the amount of "work" a task may still do before yielding back to
     /// the sheduler
     budget: Cell<coop::Budget>,
+
+    #[cfg(all(
+        tokio_unstable,
+        tokio_taskdump,
+        feature = "rt",
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))]
+    trace: trace::Context,
 }
 
 tokio_thread_local! {
-    static CONTEXT: Context = {
+    static CONTEXT: Context = const {
         Context {
             #[cfg(feature = "rt")]
             thread_id: Cell::new(None),
@@ -56,7 +83,12 @@ tokio_thread_local! {
             /// Tracks the current runtime handle to use when spawning,
             /// accessing drivers, etc...
             #[cfg(feature = "rt")]
-            handle: RefCell::new(None),
+            current: current::HandleCell::new(),
+
+            /// Tracks the current scheduler internal context
+            #[cfg(feature = "rt")]
+            scheduler: Scoped::new(),
+
             #[cfg(feature = "rt")]
             current_task_id: Cell::new(None),
 
@@ -68,20 +100,35 @@ tokio_thread_local! {
             #[cfg(feature = "rt")]
             runtime: Cell::new(EnterRuntime::NotEntered),
 
-            #[cfg(feature = "rt")]
-            defer: RefCell::new(None),
-
             #[cfg(any(feature = "rt", feature = "macros"))]
-            rng: FastRand::new(RngSeed::new()),
+            rng: Cell::new(None),
 
             budget: Cell::new(coop::Budget::unconstrained()),
+
+            #[cfg(all(
+                tokio_unstable,
+                tokio_taskdump,
+                feature = "rt",
+                target_os = "linux",
+                any(
+                    target_arch = "aarch64",
+                    target_arch = "x86",
+                    target_arch = "x86_64"
+                )
+            ))]
+            trace: trace::Context::new(),
         }
     }
 }
 
 #[cfg(any(feature = "macros", all(feature = "sync", feature = "rt")))]
 pub(crate) fn thread_rng_n(n: u32) -> u32 {
-    CONTEXT.with(|ctx| ctx.rng.fastrand_n(n))
+    CONTEXT.with(|ctx| {
+        let mut rng = ctx.rng.get().unwrap_or_else(FastRand::new);
+        let ret = rng.fastrand_n(n);
+        ctx.rng.set(Some(rng));
+        ret
+    })
 }
 
 pub(super) fn budget<R>(f: impl FnOnce(&Cell<coop::Budget>) -> R) -> Result<R, AccessError> {
@@ -89,9 +136,7 @@ pub(super) fn budget<R>(f: impl FnOnce(&Cell<coop::Budget>) -> R) -> Result<R, A
 }
 
 cfg_rt! {
-    use crate::runtime::{ThreadId, TryCurrentError};
-
-    use std::fmt;
+    use crate::runtime::ThreadId;
 
     pub(crate) fn thread_id() -> Result<ThreadId, AccessError> {
         CONTEXT.try_with(|ctx| {
@@ -106,48 +151,6 @@ cfg_rt! {
         })
     }
 
-    #[derive(Debug, Clone, Copy)]
-    #[must_use]
-    pub(crate) enum EnterRuntime {
-        /// Currently in a runtime context.
-        #[cfg_attr(not(feature = "rt"), allow(dead_code))]
-        Entered { allow_block_in_place: bool },
-
-        /// Not in a runtime context **or** a blocking region.
-        NotEntered,
-    }
-
-    #[derive(Debug)]
-    #[must_use]
-    pub(crate) struct SetCurrentGuard {
-        old_handle: Option<scheduler::Handle>,
-        old_seed: RngSeed,
-    }
-
-    /// Guard tracking that a caller has entered a runtime context.
-    #[must_use]
-    pub(crate) struct EnterRuntimeGuard {
-        /// Tracks that the current thread has entered a blocking function call.
-        pub(crate) blocking: BlockingRegionGuard,
-
-        #[allow(dead_code)] // Only tracking the guard.
-        pub(crate) handle: SetCurrentGuard,
-
-        /// If true, then this is the root runtime guard. It is possible to nest
-        /// runtime guards by using `block_in_place` between the calls. We need
-        /// to track the root guard as this is the guard responsible for freeing
-        /// the deferred task queue.
-        is_root: bool,
-    }
-
-    /// Guard tracking that a caller has entered a blocking region.
-    #[must_use]
-    pub(crate) struct BlockingRegionGuard {
-        _p: PhantomData<RefCell<()>>,
-    }
-
-    pub(crate) struct DisallowBlockInPlaceGuard(bool);
-
     pub(crate) fn set_current_task_id(id: Option<Id>) -> Option<Id> {
         CONTEXT.try_with(|ctx| ctx.current_task_id.replace(id)).unwrap_or(None)
     }
@@ -156,265 +159,33 @@ cfg_rt! {
         CONTEXT.try_with(|ctx| ctx.current_task_id.get()).unwrap_or(None)
     }
 
-    pub(crate) fn try_current() -> Result<scheduler::Handle, TryCurrentError> {
-        match CONTEXT.try_with(|ctx| ctx.handle.borrow().clone()) {
-            Ok(Some(handle)) => Ok(handle),
-            Ok(None) => Err(TryCurrentError::new_no_context()),
-            Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()),
-        }
-    }
-
-    /// Sets this [`Handle`] as the current active [`Handle`].
-    ///
-    /// [`Handle`]: crate::runtime::scheduler::Handle
-    pub(crate) fn try_set_current(handle: &scheduler::Handle) -> Option<SetCurrentGuard> {
-        CONTEXT.try_with(|ctx| ctx.set_current(handle)).ok()
-    }
-
-
-    /// Marks the current thread as being within the dynamic extent of an
-    /// executor.
     #[track_caller]
-    pub(crate) fn enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool) -> EnterRuntimeGuard {
-        if let Some(enter) = try_enter_runtime(handle, allow_block_in_place) {
-            return enter;
-        }
-
-        panic!(
-            "Cannot start a runtime from within a runtime. This happens \
-            because a function (like `block_on`) attempted to block the \
-            current thread while the thread is being used to drive \
-            asynchronous tasks."
-        );
-    }
-
-    /// Tries to enter a runtime context, returns `None` if already in a runtime
-    /// context.
-    fn try_enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool) -> Option<EnterRuntimeGuard> {
-        CONTEXT.with(|c| {
-            if c.runtime.get().is_entered() {
-                None
+    pub(crate) fn defer(waker: &Waker) {
+        with_scheduler(|maybe_scheduler| {
+            if let Some(scheduler) = maybe_scheduler {
+                scheduler.defer(waker);
             } else {
-                // Set the entered flag
-                c.runtime.set(EnterRuntime::Entered { allow_block_in_place });
-
-                // Initialize queue to track yielded tasks
-                let mut defer = c.defer.borrow_mut();
-
-                let is_root = if defer.is_none() {
-                    *defer = Some(Defer::new());
-                    true
-                } else {
-                    false
-                };
-
-                Some(EnterRuntimeGuard {
-                    blocking: BlockingRegionGuard::new(),
-                    handle: c.set_current(handle),
-                    is_root,
-                })
-            }
-        })
-    }
-
-    pub(crate) fn try_enter_blocking_region() -> Option<BlockingRegionGuard> {
-        CONTEXT.try_with(|c| {
-            if c.runtime.get().is_entered() {
-                None
-            } else {
-                Some(BlockingRegionGuard::new())
-            }
-            // If accessing the thread-local fails, the thread is terminating
-            // and thread-locals are being destroyed. Because we don't know if
-            // we are currently in a runtime or not, we default to being
-            // permissive.
-        }).unwrap_or_else(|_| Some(BlockingRegionGuard::new()))
-    }
-
-    /// Disallows blocking in the current runtime context until the guard is dropped.
-    pub(crate) fn disallow_block_in_place() -> DisallowBlockInPlaceGuard {
-        let reset = CONTEXT.with(|c| {
-            if let EnterRuntime::Entered {
-                allow_block_in_place: true,
-            } = c.runtime.get()
-            {
-                c.runtime.set(EnterRuntime::Entered {
-                    allow_block_in_place: false,
-                });
-                true
-            } else {
-                false
+                // Called from outside of the runtime, immediately wake the
+                // task.
+                waker.wake_by_ref();
             }
         });
-
-        DisallowBlockInPlaceGuard(reset)
     }
 
-    pub(crate) fn with_defer<R>(f: impl FnOnce(&mut Defer) -> R) -> Option<R> {
-        CONTEXT.with(|c| {
-            let mut defer = c.defer.borrow_mut();
-            defer.as_mut().map(f)
-        })
+    pub(super) fn set_scheduler<R>(v: &scheduler::Context, f: impl FnOnce() -> R) -> R {
+        CONTEXT.with(|c| c.scheduler.set(v, f))
     }
 
-    impl Context {
-        fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard {
-            let rng_seed = handle.seed_generator().next_seed();
-
-            let old_handle = self.handle.borrow_mut().replace(handle.clone());
-            let old_seed = self.rng.replace_seed(rng_seed);
-
-            SetCurrentGuard {
-                old_handle,
-                old_seed,
-            }
-        }
+    #[track_caller]
+    pub(super) fn with_scheduler<R>(f: impl FnOnce(Option<&scheduler::Context>) -> R) -> R {
+        CONTEXT.with(|c| c.scheduler.with(f))
     }
 
-    impl Drop for SetCurrentGuard {
-        fn drop(&mut self) {
-            CONTEXT.with(|ctx| {
-                *ctx.handle.borrow_mut() = self.old_handle.take();
-                ctx.rng.replace_seed(self.old_seed.clone());
-            });
-        }
-    }
-
-    impl fmt::Debug for EnterRuntimeGuard {
-        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-            f.debug_struct("Enter").finish()
-        }
-    }
-
-    impl Drop for EnterRuntimeGuard {
-        fn drop(&mut self) {
-            CONTEXT.with(|c| {
-                assert!(c.runtime.get().is_entered());
-                c.runtime.set(EnterRuntime::NotEntered);
-
-                if self.is_root {
-                    *c.defer.borrow_mut() = None;
-                }
-            });
-        }
-    }
-
-    impl BlockingRegionGuard {
-        fn new() -> BlockingRegionGuard {
-            BlockingRegionGuard { _p: PhantomData }
-        }
-        /// Blocks the thread on the specified future, returning the value with
-        /// which that future completes.
-        pub(crate) fn block_on<F>(&mut self, f: F) -> Result<F::Output, AccessError>
-        where
-            F: std::future::Future,
-        {
-            use crate::runtime::park::CachedParkThread;
-
-            let mut park = CachedParkThread::new();
-            park.block_on(f)
-        }
-
-        /// Blocks the thread on the specified future for **at most** `timeout`
-        ///
-        /// If the future completes before `timeout`, the result is returned. If
-        /// `timeout` elapses, then `Err` is returned.
-        pub(crate) fn block_on_timeout<F>(&mut self, f: F, timeout: Duration) -> Result<F::Output, ()>
-        where
-            F: std::future::Future,
-        {
-            use crate::runtime::park::CachedParkThread;
-            use std::task::Context;
-            use std::task::Poll::Ready;
-            use std::time::Instant;
-
-            let mut park = CachedParkThread::new();
-            let waker = park.waker().map_err(|_| ())?;
-            let mut cx = Context::from_waker(&waker);
-
-            pin!(f);
-            let when = Instant::now() + timeout;
-
-            loop {
-                if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) {
-                    return Ok(v);
-                }
-
-                let now = Instant::now();
-
-                if now >= when {
-                    return Err(());
-                }
-
-                // Wake any yielded tasks before parking in order to avoid
-                // blocking.
-                with_defer(|defer| defer.wake());
-
-                park.park_timeout(when - now);
-            }
-        }
-    }
-
-    impl Drop for DisallowBlockInPlaceGuard {
-        fn drop(&mut self) {
-            if self.0 {
-                // XXX: Do we want some kind of assertion here, or is "best effort" okay?
-                CONTEXT.with(|c| {
-                    if let EnterRuntime::Entered {
-                        allow_block_in_place: false,
-                    } = c.runtime.get()
-                    {
-                        c.runtime.set(EnterRuntime::Entered {
-                            allow_block_in_place: true,
-                        });
-                    }
-                })
-            }
-        }
-    }
-
-    impl EnterRuntime {
-        pub(crate) fn is_entered(self) -> bool {
-            matches!(self, EnterRuntime::Entered { .. })
+    cfg_taskdump! {
+        /// SAFETY: Callers of this function must ensure that trace frames always
+        /// form a valid linked list.
+        pub(crate) unsafe fn with_trace<R>(f: impl FnOnce(&trace::Context) -> R) -> Option<R> {
+            CONTEXT.try_with(|c| f(&c.trace)).ok()
         }
     }
 }
-
-// Forces the current "entered" state to be cleared while the closure
-// is executed.
-//
-// # Warning
-//
-// This is hidden for a reason. Do not use without fully understanding
-// executors. Misusing can easily cause your program to deadlock.
-cfg_rt_multi_thread! {
-    /// Returns true if in a runtime context.
-    pub(crate) fn current_enter_context() -> EnterRuntime {
-        CONTEXT.with(|c| c.runtime.get())
-    }
-
-    pub(crate) fn exit_runtime<F: FnOnce() -> R, R>(f: F) -> R {
-        // Reset in case the closure panics
-        struct Reset(EnterRuntime);
-
-        impl Drop for Reset {
-            fn drop(&mut self) {
-                CONTEXT.with(|c| {
-                    assert!(!c.runtime.get().is_entered(), "closure claimed permanent executor");
-                    c.runtime.set(self.0);
-                });
-            }
-        }
-
-        let was = CONTEXT.with(|c| {
-            let e = c.runtime.get();
-            assert!(e.is_entered(), "asked to exit when not entered");
-            c.runtime.set(EnterRuntime::NotEntered);
-            e
-        });
-
-        let _reset = Reset(was);
-        // dropping _reset after f() will reset ENTERED
-        f()
-    }
-}

tokio/src/runtime/context/blocking.rs

@@ -0,0 +1,121 @@
+use super::{EnterRuntime, CONTEXT};
+
+use crate::loom::thread::AccessError;
+use crate::util::markers::NotSendOrSync;
+
+use std::marker::PhantomData;
+use std::time::Duration;
+
+/// Guard tracking that a caller has entered a blocking region.
+#[must_use]
+pub(crate) struct BlockingRegionGuard {
+    _p: PhantomData<NotSendOrSync>,
+}
+
+pub(crate) struct DisallowBlockInPlaceGuard(bool);
+
+pub(crate) fn try_enter_blocking_region() -> Option<BlockingRegionGuard> {
+    CONTEXT
+        .try_with(|c| {
+            if c.runtime.get().is_entered() {
+                None
+            } else {
+                Some(BlockingRegionGuard::new())
+            }
+            // If accessing the thread-local fails, the thread is terminating
+            // and thread-locals are being destroyed. Because we don't know if
+            // we are currently in a runtime or not, we default to being
+            // permissive.
+        })
+        .unwrap_or_else(|_| Some(BlockingRegionGuard::new()))
+}
+
+/// Disallows blocking in the current runtime context until the guard is dropped.
+pub(crate) fn disallow_block_in_place() -> DisallowBlockInPlaceGuard {
+    let reset = CONTEXT.with(|c| {
+        if let EnterRuntime::Entered {
+            allow_block_in_place: true,
+        } = c.runtime.get()
+        {
+            c.runtime.set(EnterRuntime::Entered {
+                allow_block_in_place: false,
+            });
+            true
+        } else {
+            false
+        }
+    });
+
+    DisallowBlockInPlaceGuard(reset)
+}
+
+impl BlockingRegionGuard {
+    pub(super) fn new() -> BlockingRegionGuard {
+        BlockingRegionGuard { _p: PhantomData }
+    }
+
+    /// Blocks the thread on the specified future, returning the value with
+    /// which that future completes.
+    pub(crate) fn block_on<F>(&mut self, f: F) -> Result<F::Output, AccessError>
+    where
+        F: std::future::Future,
+    {
+        use crate::runtime::park::CachedParkThread;
+
+        let mut park = CachedParkThread::new();
+        park.block_on(f)
+    }
+
+    /// Blocks the thread on the specified future for **at most** `timeout`
+    ///
+    /// If the future completes before `timeout`, the result is returned. If
+    /// `timeout` elapses, then `Err` is returned.
+    pub(crate) fn block_on_timeout<F>(&mut self, f: F, timeout: Duration) -> Result<F::Output, ()>
+    where
+        F: std::future::Future,
+    {
+        use crate::runtime::park::CachedParkThread;
+        use std::task::Context;
+        use std::task::Poll::Ready;
+        use std::time::Instant;
+
+        let mut park = CachedParkThread::new();
+        let waker = park.waker().map_err(|_| ())?;
+        let mut cx = Context::from_waker(&waker);
+
+        pin!(f);
+        let when = Instant::now() + timeout;
+
+        loop {
+            if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) {
+                return Ok(v);
+            }
+
+            let now = Instant::now();
+
+            if now >= when {
+                return Err(());
+            }
+
+            park.park_timeout(when - now);
+        }
+    }
+}
+
+impl Drop for DisallowBlockInPlaceGuard {
+    fn drop(&mut self) {
+        if self.0 {
+            // XXX: Do we want some kind of assertion here, or is "best effort" okay?
+            CONTEXT.with(|c| {
+                if let EnterRuntime::Entered {
+                    allow_block_in_place: false,
+                } = c.runtime.get()
+                {
+                    c.runtime.set(EnterRuntime::Entered {
+                        allow_block_in_place: true,
+                    });
+                }
+            })
+        }
+    }
+}

tokio/src/runtime/context/current.rs

@@ -0,0 +1,99 @@
+use super::{Context, CONTEXT};
+
+use crate::runtime::{scheduler, TryCurrentError};
+use crate::util::markers::SyncNotSend;
+
+use std::cell::{Cell, RefCell};
+use std::marker::PhantomData;
+
+#[derive(Debug)]
+#[must_use]
+pub(crate) struct SetCurrentGuard {
+    // The previous handle
+    prev: Option<scheduler::Handle>,
+
+    // The depth for this guard
+    depth: usize,
+
+    // Don't let the type move across threads.
+    _p: PhantomData<SyncNotSend>,
+}
+
+pub(super) struct HandleCell {
+    /// Current handle
+    handle: RefCell<Option<scheduler::Handle>>,
+
+    /// Tracks the number of nested calls to `try_set_current`.
+    depth: Cell<usize>,
+}
+
+/// Sets this [`Handle`] as the current active [`Handle`].
+///
+/// [`Handle`]: crate::runtime::scheduler::Handle
+pub(crate) fn try_set_current(handle: &scheduler::Handle) -> Option<SetCurrentGuard> {
+    CONTEXT.try_with(|ctx| ctx.set_current(handle)).ok()
+}
+
+pub(crate) fn with_current<F, R>(f: F) -> Result<R, TryCurrentError>
+where
+    F: FnOnce(&scheduler::Handle) -> R,
+{
+    match CONTEXT.try_with(|ctx| ctx.current.handle.borrow().as_ref().map(f)) {
+        Ok(Some(ret)) => Ok(ret),
+        Ok(None) => Err(TryCurrentError::new_no_context()),
+        Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()),
+    }
+}
+
+impl Context {
+    pub(super) fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard {
+        let old_handle = self.current.handle.borrow_mut().replace(handle.clone());
+        let depth = self.current.depth.get();
+
+        if depth == usize::MAX {
+            panic!("reached max `enter` depth");
+        }
+
+        let depth = depth + 1;
+        self.current.depth.set(depth);
+
+        SetCurrentGuard {
+            prev: old_handle,
+            depth,
+            _p: PhantomData,
+        }
+    }
+}
+
+impl HandleCell {
+    pub(super) const fn new() -> HandleCell {
+        HandleCell {
+            handle: RefCell::new(None),
+            depth: Cell::new(0),
+        }
+    }
+}
+
+impl Drop for SetCurrentGuard {
+    fn drop(&mut self) {
+        CONTEXT.with(|ctx| {
+            let depth = ctx.current.depth.get();
+
+            if depth != self.depth {
+                if !std::thread::panicking() {
+                    panic!(
+                        "`EnterGuard` values dropped out of order. Guards returned by \
+                         `tokio::runtime::Handle::enter()` must be dropped in the reverse \
+                         order as they were acquired."
+                    );
+                } else {
+                    // Just return... this will leave handles in a wonky state though...
+                    return;
+                }
+            }
+
+            *ctx.current.handle.borrow_mut() = self.prev.take();
+            ctx.current.depth.set(depth - 1);
+        });
+    }
+}

tokio/src/runtime/context/runtime.rs

@@ -0,0 +1,99 @@
+use super::{BlockingRegionGuard, SetCurrentGuard, CONTEXT};
+
+use crate::runtime::scheduler;
+use crate::util::rand::{FastRand, RngSeed};
+
+use std::fmt;
+
+#[derive(Debug, Clone, Copy)]
+#[must_use]
+pub(crate) enum EnterRuntime {
+    /// Currently in a runtime context.
+    #[cfg_attr(not(feature = "rt"), allow(dead_code))]
+    Entered { allow_block_in_place: bool },
+
+    /// Not in a runtime context **or** a blocking region.
+    NotEntered,
+}
+
+/// Guard tracking that a caller has entered a runtime context.
+#[must_use]
+pub(crate) struct EnterRuntimeGuard {
+    /// Tracks that the current thread has entered a blocking function call.
+    pub(crate) blocking: BlockingRegionGuard,
+
+    #[allow(dead_code)] // Only tracking the guard.
+    pub(crate) handle: SetCurrentGuard,
+
+    // Tracks the previous random number generator seed
+    old_seed: RngSeed,
+}
+
+/// Marks the current thread as being within the dynamic extent of an
+/// executor.
+#[track_caller]
+pub(crate) fn enter_runtime<F, R>(handle: &scheduler::Handle, allow_block_in_place: bool, f: F) -> R
+where
+    F: FnOnce(&mut BlockingRegionGuard) -> R,
+{
+    let maybe_guard = CONTEXT.with(|c| {
+        if c.runtime.get().is_entered() {
+            None
+        } else {
+            // Set the entered flag
+            c.runtime.set(EnterRuntime::Entered {
+                allow_block_in_place,
+            });
+
+            // Generate a new seed
+            let rng_seed = handle.seed_generator().next_seed();
+
+            // Swap the RNG seed
+            let mut rng = c.rng.get().unwrap_or_else(FastRand::new);
+            let old_seed = rng.replace_seed(rng_seed);
+            c.rng.set(Some(rng));
+
+            Some(EnterRuntimeGuard {
+                blocking: BlockingRegionGuard::new(),
+                handle: c.set_current(handle),
+                old_seed,
+            })
+        }
+    });
+
+    if let Some(mut guard) = maybe_guard {
+        return f(&mut guard.blocking);
+    }
+
+    panic!(
+        "Cannot start a runtime from within a runtime. This happens \
+            because a function (like `block_on`) attempted to block the \
+            current thread while the thread is being used to drive \
+            asynchronous tasks."
+    );
+}
+
+impl fmt::Debug for EnterRuntimeGuard {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Enter").finish()
+    }
+}
+
+impl Drop for EnterRuntimeGuard {
+    fn drop(&mut self) {
+        CONTEXT.with(|c| {
+            assert!(c.runtime.get().is_entered());
+            c.runtime.set(EnterRuntime::NotEntered);
+            // Replace the previous RNG seed
+            let mut rng = c.rng.get().unwrap_or_else(FastRand::new);
+            rng.replace_seed(self.old_seed.clone());
+            c.rng.set(Some(rng));
+        });
+    }
+}
+
+impl EnterRuntime {
+    pub(crate) fn is_entered(self) -> bool {
+        matches!(self, EnterRuntime::Entered { .. })
+    }
+}

tokio/src/runtime/context/runtime_mt.rs

@@ -0,0 +1,36 @@
+use super::{EnterRuntime, CONTEXT};
+
+/// Returns true if in a runtime context.
+pub(crate) fn current_enter_context() -> EnterRuntime {
+    CONTEXT.with(|c| c.runtime.get())
+}
+
+/// Forces the current "entered" state to be cleared while the closure
+/// is executed.
+pub(crate) fn exit_runtime<F: FnOnce() -> R, R>(f: F) -> R {
+    // Reset in case the closure panics
+    struct Reset(EnterRuntime);
+
+    impl Drop for Reset {
+        fn drop(&mut self) {
+            CONTEXT.with(|c| {
+                assert!(
+                    !c.runtime.get().is_entered(),
+                    "closure claimed permanent executor"
+                );
+                c.runtime.set(self.0);
+            });
+        }
+    }
+
+    let was = CONTEXT.with(|c| {
+        let e = c.runtime.get();
+        assert!(e.is_entered(), "asked to exit when not entered");
+        c.runtime.set(EnterRuntime::NotEntered);
+        e
+    });
+
+    let _reset = Reset(was);
+    // dropping _reset after f() will reset ENTERED
+    f()
+}

tokio/src/runtime/context/scoped.rs

@@ -0,0 +1,56 @@
+use std::cell::Cell;
+use std::ptr;
+
+/// Scoped thread-local storage
+pub(super) struct Scoped<T> {
+    pub(super) inner: Cell<*const T>,
+}
+
+impl<T> Scoped<T> {
+    pub(super) const fn new() -> Scoped<T> {
+        Scoped {
+            inner: Cell::new(ptr::null()),
+        }
+    }
+
+    /// Inserts a value into the scoped cell for the duration of the closure
+    pub(super) fn set<F, R>(&self, t: &T, f: F) -> R
+    where
+        F: FnOnce() -> R,
+    {
+        struct Reset<'a, T> {
+            cell: &'a Cell<*const T>,
+            prev: *const T,
+        }
+
+        impl<T> Drop for Reset<'_, T> {
+            fn drop(&mut self) {
+                self.cell.set(self.prev);
+            }
+        }
+
+        let prev = self.inner.get();
+        self.inner.set(t as *const _);
+
+        let _reset = Reset {
+            cell: &self.inner,
+            prev,
+        };
+
+        f()
+    }
+
+    /// Gets the value out of the scoped cell;
+    pub(super) fn with<F, R>(&self, f: F) -> R
+    where
+        F: FnOnce(Option<&T>) -> R,
+    {
+        let val = self.inner.get();
+
+        if val.is_null() {
+            f(None)
+        } else {
+            unsafe { f(Some(&*val)) }
+        }
+    }
+}

tokio/src/runtime/coop.rs

@@ -200,9 +200,9 @@ cfg_coop! {
         cfg_metrics! {
             #[inline(always)]
             fn inc_budget_forced_yield_count() {
-                if let Ok(handle) = context::try_current() {
+                let _ = context::with_current(|handle| {
                     handle.scheduler_metrics().inc_budget_forced_yield_count();
-                }
+                });
             }
         }
 

tokio/src/runtime/defer.rs

@@ -1,27 +0,0 @@
-use std::task::Waker;
-
-pub(crate) struct Defer {
-    deferred: Vec<Waker>,
-}
-
-impl Defer {
-    pub(crate) fn new() -> Defer {
-        Defer {
-            deferred: Default::default(),
-        }
-    }
-
-    pub(crate) fn defer(&mut self, waker: Waker) {
-        self.deferred.push(waker);
-    }
-
-    pub(crate) fn is_empty(&self) -> bool {
-        self.deferred.is_empty()
-    }
-
-    pub(crate) fn wake(&mut self) {
-        for waker in self.deferred.drain(..) {
-            waker.wake();
-        }
-    }
-}

tokio/src/runtime/dump.rs

@@ -0,0 +1,76 @@
+//! Snapshots of runtime state.
+//!
+//! See [Handle::dump][crate::runtime::Handle::dump].
+
+use std::fmt;
+
+/// A snapshot of a runtime's state.
+///
+/// See [Handle::dump][crate::runtime::Handle::dump].
+#[derive(Debug)]
+pub struct Dump {
+    tasks: Tasks,
+}
+
+/// Snapshots of tasks.
+///
+/// See [Handle::dump][crate::runtime::Handle::dump].
+#[derive(Debug)]
+pub struct Tasks {
+    tasks: Vec<Task>,
+}
+
+/// A snapshot of a task.
+///
+/// See [Handle::dump][crate::runtime::Handle::dump].
+#[derive(Debug)]
+pub struct Task {
+    trace: Trace,
+}
+
+/// An execution trace of a task's last poll.
+///
+/// See [Handle::dump][crate::runtime::Handle::dump].
+#[derive(Debug)]
+pub struct Trace {
+    inner: super::task::trace::Trace,
+}
+
+impl Dump {
+    pub(crate) fn new(tasks: Vec<Task>) -> Self {
+        Self {
+            tasks: Tasks { tasks },
+        }
+    }
+
+    /// Tasks in this snapshot.
+    pub fn tasks(&self) -> &Tasks {
+        &self.tasks
+    }
+}
+
+impl Tasks {
+    /// Iterate over tasks.
+    pub fn iter(&self) -> impl Iterator<Item = &Task> {
+        self.tasks.iter()
+    }
+}
+
+impl Task {
+    pub(crate) fn new(trace: super::task::trace::Trace) -> Self {
+        Self {
+            trace: Trace { inner: trace },
+        }
+    }
+
+    /// A trace of this task's state.
+    pub fn trace(&self) -> &Trace {
+        &self.trace
+    }
+}
+
+impl fmt::Display for Trace {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.inner.fmt(f)
+    }
+}

tokio/src/runtime/handle.rs

@@ -35,9 +35,44 @@ pub struct EnterGuard<'a> {
 
 impl Handle {
     /// Enters the runtime context. This allows you to construct types that must
-    /// have an executor available on creation such as [`Sleep`] or [`TcpStream`].
-    /// It will also allow you to call methods such as [`tokio::spawn`] and [`Handle::current`]
-    /// without panicking.
+    /// have an executor available on creation such as [`Sleep`] or
+    /// [`TcpStream`]. It will also allow you to call methods such as
+    /// [`tokio::spawn`] and [`Handle::current`] without panicking.
+    ///
+    /// # Panics
+    ///
+    /// When calling `Handle::enter` multiple times, the returned guards
+    /// **must** be dropped in the reverse order that they were acquired.
+    /// Failure to do so will result in a panic and possible memory leaks.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Runtime;
+    ///
+    /// let rt = Runtime::new().unwrap();
+    ///
+    /// let _guard = rt.enter();
+    /// tokio::spawn(async {
+    ///     println!("Hello world!");
+    /// });
+    /// ```
+    ///
+    /// Do **not** do the following, this shows a scenario that will result in a
+    /// panic and possible memory leak.
+    ///
+    /// ```should_panic
+    /// use tokio::runtime::Runtime;
+    ///
+    /// let rt1 = Runtime::new().unwrap();
+    /// let rt2 = Runtime::new().unwrap();
+    ///
+    /// let enter1 = rt1.enter();
+    /// let enter2 = rt2.enter();
+    ///
+    /// drop(enter1);
+    /// drop(enter2);
+    /// ```
     ///
     /// [`Sleep`]: struct@crate::time::Sleep
     /// [`TcpStream`]: struct@crate::net::TcpStream
@@ -109,7 +144,9 @@ impl Handle {
     ///
     /// Contrary to `current`, this never panics
     pub fn try_current() -> Result<Self, TryCurrentError> {
-        context::try_current().map(|inner| Handle { inner })
+        context::with_current(|inner| Handle {
+            inner: inner.clone(),
+        })
     }
 
     /// Spawns a future onto the Tokio runtime.
@@ -252,19 +289,24 @@ impl Handle {
     /// [`tokio::time`]: crate::time
     #[track_caller]
     pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        ))]
+        let future = super::task::trace::Trace::root(future);
+
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let future =
             crate::util::trace::task(future, "block_on", None, super::task::Id::next().as_u64());
 
         // Enter the runtime context. This sets the current driver handles and
         // prevents blocking an existing runtime.
-        let mut enter = context::enter_runtime(&self.inner, true);
-
-        // Block on the future
-        enter
-            .blocking
-            .block_on(future)
-            .expect("failed to park thread")
+        context::enter_runtime(&self.inner, true, |blocking| {
+            blocking.block_on(future).expect("failed to park thread")
+        })
     }
 
     #[track_caller]
@@ -274,6 +316,14 @@ impl Handle {
         F::Output: Send + 'static,
     {
         let id = crate::runtime::task::Id::next();
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        ))]
+        let future = super::task::trace::Trace::root(future);
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let future = crate::util::trace::task(future, "task", _name, id.as_u64());
         self.inner.spawn(future, id)
@@ -305,6 +355,8 @@ impl Handle {
             scheduler::Handle::CurrentThread(_) => RuntimeFlavor::CurrentThread,
             #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
             scheduler::Handle::MultiThread(_) => RuntimeFlavor::MultiThread,
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+            scheduler::Handle::MultiThreadAlt(_) => RuntimeFlavor::MultiThreadAlt,
         }
     }
 }
@@ -321,6 +373,161 @@ cfg_metrics! {
     }
 }
 
+cfg_taskdump! {
+    impl Handle {
+        /// Captures a snapshot of the runtime's state.
+        ///
+        /// This functionality is experimental, and comes with a number of
+        /// requirements and limitations.
+        ///
+        /// # Examples
+        ///
+        /// This can be used to get call traces of each task in the runtime.
+        /// Calls to `Handle::dump` should usually be enclosed in a
+        /// [timeout][crate::time::timeout], so that dumping does not escalate a
+        /// single blocked runtime thread into an entirely blocked runtime.
+        ///
+        /// ```
+        /// # use tokio::runtime::Runtime;
+        /// # fn dox() {
+        /// # let rt = Runtime::new().unwrap();
+        /// # rt.spawn(async {
+        /// use tokio::runtime::Handle;
+        /// use tokio::time::{timeout, Duration};
+        ///
+        /// // Inside an async block or function.
+        /// let handle = Handle::current();
+        /// if let Ok(dump) = timeout(Duration::from_secs(2), handle.dump()).await {
+        ///     for (i, task) in dump.tasks().iter().enumerate() {
+        ///         let trace = task.trace();
+        ///         println!("TASK {i}:");
+        ///         println!("{trace}\n");
+        ///     }
+        /// }
+        /// # });
+        /// # }
+        /// ```
+        ///
+        /// This produces highly detailed traces of tasks; e.g.:
+        ///
+        /// ```plain
+        /// TASK 0:
+        /// ╼ dump::main::{{closure}}::a::{{closure}} at /tokio/examples/dump.rs:18:20
+        /// └╼ dump::main::{{closure}}::b::{{closure}} at /tokio/examples/dump.rs:23:20
+        ///    └╼ dump::main::{{closure}}::c::{{closure}} at /tokio/examples/dump.rs:28:24
+        ///       └╼ tokio::sync::barrier::Barrier::wait::{{closure}} at /tokio/tokio/src/sync/barrier.rs:129:10
+        ///          └╼ <tokio::util::trace::InstrumentedAsyncOp<F> as core::future::future::Future>::poll at /tokio/tokio/src/util/trace.rs:77:46
+        ///             └╼ tokio::sync::barrier::Barrier::wait_internal::{{closure}} at /tokio/tokio/src/sync/barrier.rs:183:36
+        ///                └╼ tokio::sync::watch::Receiver<T>::changed::{{closure}} at /tokio/tokio/src/sync/watch.rs:604:55
+        ///                   └╼ tokio::sync::watch::changed_impl::{{closure}} at /tokio/tokio/src/sync/watch.rs:755:18
+        ///                      └╼ <tokio::sync::notify::Notified as core::future::future::Future>::poll at /tokio/tokio/src/sync/notify.rs:1103:9
+        ///                         └╼ tokio::sync::notify::Notified::poll_notified at /tokio/tokio/src/sync/notify.rs:996:32
+        /// ```
+        ///
+        /// # Requirements
+        ///
+        /// ## Debug Info Must Be Available
+        ///
+        /// To produce task traces, the application must **not** be compiled
+        /// with split debuginfo. On Linux, including debuginfo within the
+        /// application binary is the (correct) default. You can further ensure
+        /// this behavior with the following directive in your `Cargo.toml`:
+        ///
+        /// ```toml
+        /// [profile.*]
+        /// split-debuginfo = "off"
+        /// ```
+        ///
+        /// ## Unstable Features
+        ///
+        /// This functionality is **unstable**, and requires both the
+        /// `tokio_unstable` and `tokio_taskdump` cfg flags to be set.
+        ///
+        /// You can do this by setting the `RUSTFLAGS` environment variable
+        /// before invoking `cargo`; e.g.:
+        /// ```bash
+        /// RUSTFLAGS="--cfg tokio_unstable --cfg tokio_taskdump" cargo run --example dump
+        /// ```
+        ///
+        /// Or by [configuring][cargo-config] `rustflags` in
+        /// `.cargo/config.toml`:
+        /// ```text
+        /// [build]
+        /// rustflags = ["--cfg tokio_unstable", "--cfg tokio_taskdump"]
+        /// ```
+        ///
+        /// [cargo-config]:
+        ///     https://doc.rust-lang.org/cargo/reference/config.html
+        ///
+        /// ## Platform Requirements
+        ///
+        /// Task dumps are supported on Linux atop aarch64, x86 and x86_64.
+        ///
+        /// ## Current Thread Runtime Requirements
+        ///
+        /// On the `current_thread` runtime, task dumps may only be requested
+        /// from *within* the context of the runtime being dumped. Do not, for
+        /// example, await `Handle::dump()` on a different runtime.
+        ///
+        /// # Limitations
+        ///
+        /// ## Performance
+        ///
+        /// Although enabling the `tokio_taskdump` feature imposes virtually no
+        /// additional runtime overhead, actually calling `Handle::dump` is
+        /// expensive. The runtime must synchronize and pause its workers, then
+        /// re-poll every task in a special tracing mode. Avoid requesting dumps
+        /// often.
+        ///
+        /// ## Local Executors
+        ///
+        /// Tasks managed by local executors (e.g., `FuturesUnordered` and
+        /// [`LocalSet`][crate::task::LocalSet]) may not appear in task dumps.
+        ///
+        /// ## Non-Termination When Workers Are Blocked
+        ///
+        /// The future produced by `Handle::dump` may never produce `Ready` if
+        /// another runtime worker is blocked for more than 250ms. This may
+        /// occur if a dump is requested during shutdown, or if another runtime
+        /// worker is infinite looping or synchronously deadlocked. For these
+        /// reasons, task dumping should usually be paired with an explicit
+        /// [timeout][crate::time::timeout].
+        pub async fn dump(&self) -> crate::runtime::Dump {
+            match &self.inner {
+                scheduler::Handle::CurrentThread(handle) => handle.dump(),
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThread(handle) => {
+                    // perform the trace in a separate thread so that the
+                    // trace itself does not appear in the taskdump.
+                    let handle = handle.clone();
+                    spawn_thread(async {
+                        let handle = handle;
+                        handle.dump().await
+                    }).await
+                },
+            }
+        }
+    }
+
+    cfg_rt_multi_thread! {
+        /// Spawn a new thread and asynchronously await on its result.
+        async fn spawn_thread<F>(f: F) -> <F as Future>::Output
+        where
+            F: Future + Send + 'static,
+            <F as Future>::Output: Send + 'static
+        {
+            let (tx, rx) = crate::sync::oneshot::channel();
+            crate::loom::thread::spawn(|| {
+                let rt = crate::runtime::Builder::new_current_thread().build().unwrap();
+                rt.block_on(async {
+                    let _ = tx.send(f.await);
+                });
+            });
+            rx.await.unwrap()
+        }
+    }
+}
+
 /// Error returned by `try_current` when no Runtime has been started
 #[derive(Debug)]
 pub struct TryCurrentError {

tokio/src/runtime/io/mod.rs

@@ -63,6 +63,18 @@ pub(crate) struct ReadyEvent {
     is_shutdown: bool,
 }
 
+cfg_net_unix!(
+    impl ReadyEvent {
+        pub(crate) fn with_ready(&self, ready: Ready) -> Self {
+            Self {
+                ready,
+                tick: self.tick,
+                is_shutdown: self.is_shutdown,
+            }
+        }
+    }
+);
+
 struct IoDispatcher {
     allocator: slab::Allocator<ScheduledIo>,
     is_shutdown: bool,

tokio/src/runtime/io/platform.rs

@@ -1,44 +0,0 @@
-pub(crate) use self::sys::*;
-
-#[cfg(unix)]
-mod sys {
-    use mio::unix::UnixReady;
-    use mio::Ready;
-
-    pub(crate) fn hup() -> Ready {
-        UnixReady::hup().into()
-    }
-
-    pub(crate) fn is_hup(ready: Ready) -> bool {
-        UnixReady::from(ready).is_hup()
-    }
-
-    pub(crate) fn error() -> Ready {
-        UnixReady::error().into()
-    }
-
-    pub(crate) fn is_error(ready: Ready) -> bool {
-        UnixReady::from(ready).is_error()
-    }
-}
-
-#[cfg(windows)]
-mod sys {
-    use mio::Ready;
-
-    pub(crate) fn hup() -> Ready {
-        Ready::empty()
-    }
-
-    pub(crate) fn is_hup(_: Ready) -> bool {
-        false
-    }
-
-    pub(crate) fn error() -> Ready {
-        Ready::empty()
-    }
-
-    pub(crate) fn is_error(_: Ready) -> bool {
-        false
-    }
-}

tokio/src/runtime/io/registration.rs

@@ -144,6 +144,7 @@ impl Registration {
         cx: &mut Context<'_>,
         direction: Direction,
     ) -> Poll<io::Result<ReadyEvent>> {
+        ready!(crate::trace::trace_leaf(cx));
         // Keep track of task budget
         let coop = ready!(crate::runtime::coop::poll_proceed(cx));
         let ev = ready!(self.shared.poll_readiness(cx, direction));

tokio/src/runtime/metrics/batch.rs

@@ -1,7 +1,7 @@
-use crate::runtime::WorkerMetrics;
+use crate::runtime::metrics::{HistogramBatch, WorkerMetrics};
 
 use std::sync::atomic::Ordering::Relaxed;
-use std::time::Instant;
+use std::time::{Duration, Instant};
 
 pub(crate) struct MetricsBatch {
     /// Number of times the worker parked.
@@ -32,11 +32,26 @@ pub(crate) struct MetricsBatch {
 
     /// The total busy duration in nanoseconds.
     busy_duration_total: u64,
-    last_resume_time: Instant,
+
+    /// Instant at which work last resumed (continued after park).
+    processing_scheduled_tasks_started_at: Instant,
+
+    /// If `Some`, tracks poll times in nanoseconds
+    poll_timer: Option<PollTimer>,
+}
+
+struct PollTimer {
+    /// Histogram of poll counts within each band.
+    poll_counts: HistogramBatch,
+
+    /// Instant when the most recent task started polling.
+    poll_started_at: Instant,
 }
 
 impl MetricsBatch {
-    pub(crate) fn new() -> MetricsBatch {
+    pub(crate) fn new(worker_metrics: &WorkerMetrics) -> MetricsBatch {
+        let now = Instant::now();
+
         MetricsBatch {
             park_count: 0,
             noop_count: 0,
@@ -47,7 +62,14 @@ impl MetricsBatch {
             local_schedule_count: 0,
             overflow_count: 0,
             busy_duration_total: 0,
-            last_resume_time: Instant::now(),
+            processing_scheduled_tasks_started_at: now,
+            poll_timer: worker_metrics
+                .poll_count_histogram
+                .as_ref()
+                .map(|worker_poll_counts| PollTimer {
+                    poll_counts: HistogramBatch::from_histogram(worker_poll_counts),
+                    poll_started_at: now,
+                }),
         }
     }
 
@@ -68,6 +90,11 @@ impl MetricsBatch {
             .local_schedule_count
             .store(self.local_schedule_count, Relaxed);
         worker.overflow_count.store(self.overflow_count, Relaxed);
+
+        if let Some(poll_timer) = &self.poll_timer {
+            let dst = worker.poll_count_histogram.as_ref().unwrap();
+            poll_timer.poll_counts.submit(dst);
+        }
     }
 
     /// The worker is about to park.
@@ -79,23 +106,39 @@ impl MetricsBatch {
         } else {
             self.poll_count_on_last_park = self.poll_count;
         }
-
-        let busy_duration = self.last_resume_time.elapsed();
-        let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX);
-        self.busy_duration_total += busy_duration;
     }
 
-    pub(crate) fn returned_from_park(&mut self) {
-        self.last_resume_time = Instant::now();
+    /// Start processing a batch of tasks
+    pub(crate) fn start_processing_scheduled_tasks(&mut self) {
+        self.processing_scheduled_tasks_started_at = Instant::now();
+    }
+
+    /// Stop processing a batch of tasks
+    pub(crate) fn end_processing_scheduled_tasks(&mut self) {
+        let busy_duration = self.processing_scheduled_tasks_started_at.elapsed();
+        self.busy_duration_total += duration_as_u64(busy_duration);
+    }
+
+    /// Start polling an individual task
+    pub(crate) fn start_poll(&mut self) {
+        self.poll_count += 1;
+
+        if let Some(poll_timer) = &mut self.poll_timer {
+            poll_timer.poll_started_at = Instant::now();
+        }
+    }
+
+    /// Stop polling an individual task
+    pub(crate) fn end_poll(&mut self) {
+        if let Some(poll_timer) = &mut self.poll_timer {
+            let elapsed = duration_as_u64(poll_timer.poll_started_at.elapsed());
+            poll_timer.poll_counts.measure(elapsed, 1);
+        }
     }
 
     pub(crate) fn inc_local_schedule_count(&mut self) {
         self.local_schedule_count += 1;
     }
-
-    pub(crate) fn incr_poll_count(&mut self) {
-        self.poll_count += 1;
-    }
 }
 
 cfg_rt_multi_thread! {
@@ -113,3 +156,7 @@ cfg_rt_multi_thread! {
         }
     }
 }
+
+fn duration_as_u64(dur: Duration) -> u64 {
+    u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX)
+}

tokio/src/runtime/metrics/histogram.rs

@@ -0,0 +1,502 @@
+use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed};
+
+use std::cmp;
+use std::ops::Range;
+
+#[derive(Debug)]
+pub(crate) struct Histogram {
+    /// The histogram buckets
+    buckets: Box<[AtomicU64]>,
+
+    /// Bucket scale, linear or log
+    scale: HistogramScale,
+
+    /// Minimum resolution
+    resolution: u64,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct HistogramBuilder {
+    /// Histogram scale
+    pub(crate) scale: HistogramScale,
+
+    /// Must be a power of 2
+    pub(crate) resolution: u64,
+
+    /// Number of buckets
+    pub(crate) num_buckets: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct HistogramBatch {
+    buckets: Box<[u64]>,
+    scale: HistogramScale,
+    resolution: u64,
+}
+
+cfg_unstable! {
+    /// Whether the histogram used to aggregate a metric uses a linear or
+    /// logarithmic scale.
+    #[derive(Debug, Copy, Clone, Eq, PartialEq)]
+    #[non_exhaustive]
+    pub enum HistogramScale {
+        /// Linear bucket scale
+        Linear,
+
+        /// Logarithmic bucket scale
+        Log,
+    }
+}
+
+impl Histogram {
+    pub(crate) fn num_buckets(&self) -> usize {
+        self.buckets.len()
+    }
+
+    pub(crate) fn get(&self, bucket: usize) -> u64 {
+        self.buckets[bucket].load(Relaxed)
+    }
+
+    pub(crate) fn bucket_range(&self, bucket: usize) -> Range<u64> {
+        match self.scale {
+            HistogramScale::Log => Range {
+                start: if bucket == 0 {
+                    0
+                } else {
+                    self.resolution << (bucket - 1)
+                },
+                end: if bucket == self.buckets.len() - 1 {
+                    u64::MAX
+                } else {
+                    self.resolution << bucket
+                },
+            },
+            HistogramScale::Linear => Range {
+                start: self.resolution * bucket as u64,
+                end: if bucket == self.buckets.len() - 1 {
+                    u64::MAX
+                } else {
+                    self.resolution * (bucket as u64 + 1)
+                },
+            },
+        }
+    }
+}
+
+impl HistogramBatch {
+    pub(crate) fn from_histogram(histogram: &Histogram) -> HistogramBatch {
+        let buckets = vec![0; histogram.buckets.len()].into_boxed_slice();
+
+        HistogramBatch {
+            buckets,
+            scale: histogram.scale,
+            resolution: histogram.resolution,
+        }
+    }
+
+    pub(crate) fn measure(&mut self, value: u64, count: u64) {
+        self.buckets[self.value_to_bucket(value)] += count;
+    }
+
+    pub(crate) fn submit(&self, histogram: &Histogram) {
+        debug_assert_eq!(self.scale, histogram.scale);
+        debug_assert_eq!(self.resolution, histogram.resolution);
+        debug_assert_eq!(self.buckets.len(), histogram.buckets.len());
+
+        for i in 0..self.buckets.len() {
+            histogram.buckets[i].store(self.buckets[i], Relaxed);
+        }
+    }
+
+    fn value_to_bucket(&self, value: u64) -> usize {
+        match self.scale {
+            HistogramScale::Linear => {
+                let max = self.buckets.len() - 1;
+                cmp::min(value / self.resolution, max as u64) as usize
+            }
+            HistogramScale::Log => {
+                let max = self.buckets.len() - 1;
+
+                if value < self.resolution {
+                    0
+                } else {
+                    let significant_digits = 64 - value.leading_zeros();
+                    let bucket_digits = 64 - (self.resolution - 1).leading_zeros();
+                    cmp::min(significant_digits as usize - bucket_digits as usize, max)
+                }
+            }
+        }
+    }
+}
+
+impl HistogramBuilder {
+    pub(crate) fn new() -> HistogramBuilder {
+        HistogramBuilder {
+            scale: HistogramScale::Linear,
+            // Resolution is in nanoseconds.
+            resolution: 100_000,
+            num_buckets: 10,
+        }
+    }
+
+    pub(crate) fn build(&self) -> Histogram {
+        let mut resolution = self.resolution;
+
+        assert!(resolution > 0);
+
+        if matches!(self.scale, HistogramScale::Log) {
+            resolution = resolution.next_power_of_two();
+        }
+
+        Histogram {
+            buckets: (0..self.num_buckets)
+                .map(|_| AtomicU64::new(0))
+                .collect::<Vec<_>>()
+                .into_boxed_slice(),
+            resolution,
+            scale: self.scale,
+        }
+    }
+}
+
+impl Default for HistogramBuilder {
+    fn default() -> HistogramBuilder {
+        HistogramBuilder::new()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    macro_rules! assert_bucket_eq {
+        ($h:expr, $bucket:expr, $val:expr) => {{
+            assert_eq!($h.buckets[$bucket], $val);
+        }};
+    }
+
+    #[test]
+    fn log_scale_resolution_1() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Log,
+            resolution: 1,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..1);
+        assert_eq!(h.bucket_range(1), 1..2);
+        assert_eq!(h.bucket_range(2), 2..4);
+        assert_eq!(h.bucket_range(3), 4..8);
+        assert_eq!(h.bucket_range(9), 256..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(1, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(2, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 1);
+
+        b.measure(3, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 2);
+
+        b.measure(4, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 2);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(100, 1);
+        assert_bucket_eq!(b, 7, 1);
+
+        b.measure(128, 1);
+        assert_bucket_eq!(b, 8, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+    }
+
+    #[test]
+    fn log_scale_resolution_2() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Log,
+            resolution: 2,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..2);
+        assert_eq!(h.bucket_range(1), 2..4);
+        assert_eq!(h.bucket_range(2), 4..8);
+        assert_eq!(h.bucket_range(3), 8..16);
+        assert_eq!(h.bucket_range(9), 512..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(1, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(2, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(3, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(4, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 1);
+
+        b.measure(5, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 2);
+
+        b.measure(6, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 3);
+
+        b.measure(7, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 4);
+
+        b.measure(8, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 4);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(100, 1);
+        assert_bucket_eq!(b, 6, 1);
+
+        b.measure(128, 1);
+        assert_bucket_eq!(b, 7, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+
+    #[test]
+    fn linear_scale_resolution_1() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Linear,
+            resolution: 1,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..1);
+        assert_eq!(h.bucket_range(1), 1..2);
+        assert_eq!(h.bucket_range(2), 2..3);
+        assert_eq!(h.bucket_range(3), 3..4);
+        assert_eq!(h.bucket_range(9), 9..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(1, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(2, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 1);
+        assert_bucket_eq!(b, 3, 0);
+
+        b.measure(3, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 1);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(5, 1);
+        assert_bucket_eq!(b, 5, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+
+    #[test]
+    fn linear_scale_resolution_100() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Linear,
+            resolution: 100,
+            num_buckets: 10,
+        }
+        .build();
+
+        assert_eq!(h.bucket_range(0), 0..100);
+        assert_eq!(h.bucket_range(1), 100..200);
+        assert_eq!(h.bucket_range(2), 200..300);
+        assert_eq!(h.bucket_range(3), 300..400);
+        assert_eq!(h.bucket_range(9), 900..u64::MAX);
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 1);
+        assert_bucket_eq!(b, 0, 1);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(50, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(100, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 1);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(101, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(200, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 1);
+
+        b.measure(299, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 2);
+
+        b.measure(222, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 3);
+
+        b.measure(300, 1);
+        assert_bucket_eq!(b, 0, 2);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 3);
+        assert_bucket_eq!(b, 3, 1);
+
+        b.measure(888, 1);
+        assert_bucket_eq!(b, 8, 1);
+
+        b.measure(4096, 1);
+        assert_bucket_eq!(b, 9, 1);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+
+    #[test]
+    fn inc_by_more_than_one() {
+        let h = HistogramBuilder {
+            scale: HistogramScale::Linear,
+            resolution: 100,
+            num_buckets: 10,
+        }
+        .build();
+
+        let mut b = HistogramBatch::from_histogram(&h);
+
+        b.measure(0, 3);
+        assert_bucket_eq!(b, 0, 3);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(50, 5);
+        assert_bucket_eq!(b, 0, 8);
+        assert_bucket_eq!(b, 1, 0);
+
+        b.measure(100, 2);
+        assert_bucket_eq!(b, 0, 8);
+        assert_bucket_eq!(b, 1, 2);
+        assert_bucket_eq!(b, 2, 0);
+
+        b.measure(101, 19);
+        assert_bucket_eq!(b, 0, 8);
+        assert_bucket_eq!(b, 1, 21);
+        assert_bucket_eq!(b, 2, 0);
+
+        for bucket in h.buckets.iter() {
+            assert_eq!(bucket.load(Relaxed), 0);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+
+        b.submit(&h);
+
+        for i in 0..h.buckets.len() {
+            assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]);
+        }
+    }
+}

tokio/src/runtime/metrics/mock.rs

@@ -6,6 +6,9 @@ pub(crate) struct WorkerMetrics {}
 
 pub(crate) struct MetricsBatch {}
 
+#[derive(Clone, Default)]
+pub(crate) struct HistogramBuilder {}
+
 impl SchedulerMetrics {
     pub(crate) fn new() -> Self {
         Self {}
@@ -20,19 +23,27 @@ impl WorkerMetrics {
         Self {}
     }
 
+    pub(crate) fn from_config(config: &crate::runtime::Config) -> Self {
+        // Prevent the dead-code warning from being triggered
+        let _ = &config.metrics_poll_count_histogram;
+        Self::new()
+    }
+
     pub(crate) fn set_queue_depth(&self, _len: usize) {}
 }
 
 impl MetricsBatch {
-    pub(crate) fn new() -> Self {
+    pub(crate) fn new(_: &WorkerMetrics) -> Self {
         Self {}
     }
 
     pub(crate) fn submit(&mut self, _to: &WorkerMetrics) {}
     pub(crate) fn about_to_park(&mut self) {}
-    pub(crate) fn returned_from_park(&mut self) {}
-    pub(crate) fn incr_poll_count(&mut self) {}
     pub(crate) fn inc_local_schedule_count(&mut self) {}
+    pub(crate) fn start_processing_scheduled_tasks(&mut self) {}
+    pub(crate) fn end_processing_scheduled_tasks(&mut self) {}
+    pub(crate) fn start_poll(&mut self) {}
+    pub(crate) fn end_poll(&mut self) {}
 }
 
 cfg_rt_multi_thread! {

tokio/src/runtime/metrics/mod.rs

@@ -12,6 +12,11 @@ cfg_metrics! {
     mod batch;
     pub(crate) use batch::MetricsBatch;
 
+    mod histogram;
+    pub(crate) use histogram::{Histogram, HistogramBatch, HistogramBuilder};
+    #[allow(unreachable_pub)] // rust-lang/rust#57411
+    pub use histogram::HistogramScale;
+
     mod runtime;
     #[allow(unreachable_pub)] // rust-lang/rust#57411
     pub use runtime::RuntimeMetrics;
@@ -31,5 +36,5 @@ cfg_metrics! {
 cfg_not_metrics! {
     mod mock;
 
-    pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch};
+    pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch, HistogramBuilder};
 }

tokio/src/runtime/metrics/runtime.rs

@@ -1,5 +1,6 @@
 use crate::runtime::Handle;
 
+use std::ops::Range;
 use std::sync::atomic::Ordering::Relaxed;
 use std::time::Duration;
 
@@ -68,6 +69,25 @@ impl RuntimeMetrics {
         self.handle.inner.num_blocking_threads()
     }
 
+    /// Returns the number of active tasks in the runtime.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Handle;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///    let metrics = Handle::current().metrics();
+    ///
+    ///     let n = metrics.active_tasks_count();
+    ///     println!("Runtime has {} active tasks", n);
+    /// }
+    /// ```
+    pub fn active_tasks_count(&self) -> usize {
+        self.handle.inner.active_tasks_count()
+    }
+
     /// Returns the number of idle threads, which have spawned by the runtime
     /// for `spawn_blocking` calls.
     ///
@@ -559,6 +579,196 @@ impl RuntimeMetrics {
         self.handle.inner.worker_local_queue_depth(worker)
     }
 
+    /// Returns `true` if the runtime is tracking the distribution of task poll
+    /// times.
+    ///
+    /// Task poll times are not instrumented by default as doing so requires
+    /// calling [`Instant::now()`] twice per task poll. The feature is enabled
+    /// by calling [`enable_metrics_poll_count_histogram()`] when building the
+    /// runtime.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let enabled = metrics.poll_count_histogram_enabled();
+    ///
+    ///             println!("Tracking task poll time distribution: {:?}", enabled);
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`enable_metrics_poll_count_histogram()`]: crate::runtime::Builder::enable_metrics_poll_count_histogram
+    /// [`Instant::now()`]: std::time::Instant::now
+    pub fn poll_count_histogram_enabled(&self) -> bool {
+        self.handle
+            .inner
+            .worker_metrics(0)
+            .poll_count_histogram
+            .is_some()
+    }
+
+    /// Returns the number of histogram buckets tracking the distribution of
+    /// task poll times.
+    ///
+    /// This value is configured by calling
+    /// [`metrics_poll_count_histogram_buckets()`] when building the runtime.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let buckets = metrics.poll_count_histogram_num_buckets();
+    ///
+    ///             println!("Histogram buckets: {:?}", buckets);
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`metrics_poll_count_histogram_buckets()`]:
+    ///     crate::runtime::Builder::metrics_poll_count_histogram_buckets
+    pub fn poll_count_histogram_num_buckets(&self) -> usize {
+        self.handle
+            .inner
+            .worker_metrics(0)
+            .poll_count_histogram
+            .as_ref()
+            .map(|histogram| histogram.num_buckets())
+            .unwrap_or_default()
+    }
+
+    /// Returns the range of task poll times tracked by the given bucket.
+    ///
+    /// This value is configured by calling
+    /// [`metrics_poll_count_histogram_resolution()`] when building the runtime.
+    ///
+    /// # Panics
+    ///
+    /// The method panics if `bucket` represents an invalid bucket index, i.e.
+    /// is greater than or equal to `poll_count_histogram_num_buckets()`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let buckets = metrics.poll_count_histogram_num_buckets();
+    ///
+    ///             for i in 0..buckets {
+    ///                 let range = metrics.poll_count_histogram_bucket_range(i);
+    ///                 println!("Histogram bucket {} range: {:?}", i, range);
+    ///             }
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`metrics_poll_count_histogram_resolution()`]:
+    ///     crate::runtime::Builder::metrics_poll_count_histogram_resolution
+    #[track_caller]
+    pub fn poll_count_histogram_bucket_range(&self, bucket: usize) -> Range<Duration> {
+        self.handle
+            .inner
+            .worker_metrics(0)
+            .poll_count_histogram
+            .as_ref()
+            .map(|histogram| {
+                let range = histogram.bucket_range(bucket);
+                std::ops::Range {
+                    start: Duration::from_nanos(range.start),
+                    end: Duration::from_nanos(range.end),
+                }
+            })
+            .unwrap_or_default()
+    }
+
+    /// Returns the number of times the given worker polled tasks with a poll
+    /// duration within the given bucket's range.
+    ///
+    /// Each worker maintains its own histogram and the counts for each bucket
+    /// starts at zero when the runtime is created. Each time the worker polls a
+    /// task, it tracks the duration the task poll time took and increments the
+    /// associated bucket by 1.
+    ///
+    /// Each bucket is a monotonically increasing counter. It is never
+    /// decremented or reset to zero.
+    ///
+    /// # Arguments
+    ///
+    /// `worker` is the index of the worker being queried. The given value must
+    /// be between 0 and `num_workers()`. The index uniquely identifies a single
+    /// worker and will continue to identify the worker throughout the lifetime
+    /// of the runtime instance.
+    ///
+    /// `bucket` is the index of the bucket being queried. The bucket is scoped
+    /// to the worker. The range represented by the bucket can be queried by
+    /// calling [`poll_count_histogram_bucket_range()`]. Each worker maintains
+    /// identical bucket ranges.
+    ///
+    /// # Panics
+    ///
+    /// The method panics when `worker` represents an invalid worker, i.e. is
+    /// greater than or equal to `num_workers()` or if `bucket` represents an
+    /// invalid bucket.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::{self, Handle};
+    ///
+    /// fn main() {
+    ///     runtime::Builder::new_current_thread()
+    ///         .enable_metrics_poll_count_histogram()
+    ///         .build()
+    ///         .unwrap()
+    ///         .block_on(async {
+    ///             let metrics = Handle::current().metrics();
+    ///             let buckets = metrics.poll_count_histogram_num_buckets();
+    ///
+    ///             for worker in 0..metrics.num_workers() {
+    ///                 for i in 0..buckets {
+    ///                     let count = metrics.poll_count_histogram_bucket_count(worker, i);
+    ///                     println!("Poll count {}", count);
+    ///                 }
+    ///             }
+    ///         });
+    /// }
+    /// ```
+    ///
+    /// [`poll_count_histogram_bucket_range()`]: crate::runtime::RuntimeMetrics::poll_count_histogram_bucket_range
+    #[track_caller]
+    pub fn poll_count_histogram_bucket_count(&self, worker: usize, bucket: usize) -> u64 {
+        self.handle
+            .inner
+            .worker_metrics(worker)
+            .poll_count_histogram
+            .as_ref()
+            .map(|histogram| histogram.get(bucket))
+            .unwrap_or_default()
+    }
+
     /// Returns the number of tasks currently scheduled in the blocking
     /// thread pool, spawned using `spawn_blocking`.
     ///

tokio/src/runtime/metrics/worker.rs

@@ -1,5 +1,7 @@
 use crate::loom::sync::atomic::Ordering::Relaxed;
 use crate::loom::sync::atomic::{AtomicU64, AtomicUsize};
+use crate::runtime::metrics::Histogram;
+use crate::runtime::Config;
 
 /// Retrieve runtime worker metrics.
 ///
@@ -38,9 +40,21 @@ pub(crate) struct WorkerMetrics {
     /// Number of tasks currently in the local queue. Used only by the
     /// current-thread scheduler.
     pub(crate) queue_depth: AtomicUsize,
+
+    /// If `Some`, tracks the the number of polls by duration range.
+    pub(super) poll_count_histogram: Option<Histogram>,
 }
 
 impl WorkerMetrics {
+    pub(crate) fn from_config(config: &Config) -> WorkerMetrics {
+        let mut worker_metrics = WorkerMetrics::new();
+        worker_metrics.poll_count_histogram = config
+            .metrics_poll_count_histogram
+            .as_ref()
+            .map(|histogram_builder| histogram_builder.build());
+        worker_metrics
+    }
+
     pub(crate) fn new() -> WorkerMetrics {
         WorkerMetrics {
             park_count: AtomicU64::new(0),
@@ -52,6 +66,7 @@ impl WorkerMetrics {
             busy_duration_total: AtomicU64::new(0),
             local_schedule_count: AtomicU64::new(0),
             queue_depth: AtomicUsize::new(0),
+            poll_count_histogram: None,
         }
     }
 

tokio/src/runtime/mod.rs

@@ -186,6 +186,7 @@ pub(crate) mod coop;
 pub(crate) mod park;
 
 mod driver;
+use driver::Driver;
 
 pub(crate) mod scheduler;
 
@@ -230,8 +231,10 @@ cfg_rt! {
         pub use crate::util::rand::RngSeed;
     }
 
-    mod defer;
-    pub(crate) use defer::Defer;
+    cfg_taskdump! {
+        pub mod dump;
+        pub use dump::Dump;
+    }
 
     mod handle;
     pub use handle::{EnterGuard, Handle, TryCurrentError};
@@ -244,9 +247,9 @@ cfg_rt! {
 
     cfg_metrics! {
         mod metrics;
-        pub use metrics::RuntimeMetrics;
+        pub use metrics::{RuntimeMetrics, HistogramScale};
 
-        pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
+        pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics, HistogramBuilder};
 
         cfg_net! {
         pub(crate) use metrics::IoDriverMetrics;
@@ -255,7 +258,7 @@ cfg_rt! {
 
     cfg_not_metrics! {
         pub(crate) mod metrics;
-        pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch};
+        pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch, HistogramBuilder};
     }
 
     /// After thread starts / before thread stops

tokio/src/runtime/park.rs

@@ -284,11 +284,6 @@ impl CachedParkThread {
                 return Ok(v);
             }
 
-            // Wake any yielded tasks before parking in order to avoid
-            // blocking.
-            #[cfg(feature = "rt")]
-            crate::runtime::context::with_defer(|defer| defer.wake());
-
             self.park();
         }
     }

tokio/src/runtime/runtime.rs

@@ -9,6 +9,10 @@ use std::time::Duration;
 cfg_rt_multi_thread! {
     use crate::runtime::Builder;
     use crate::runtime::scheduler::MultiThread;
+
+    cfg_unstable! {
+        use crate::runtime::scheduler::MultiThreadAlt;
+    }
 }
 
 /// The Tokio runtime.
@@ -84,6 +88,9 @@ pub enum RuntimeFlavor {
     CurrentThread,
     /// The flavor that executes tasks across multiple threads.
     MultiThread,
+    /// The flavor that executes tasks across multiple threads.
+    #[cfg(tokio_unstable)]
+    MultiThreadAlt,
 }
 
 /// The runtime scheduler is either a multi-thread or a current-thread executor.
@@ -95,6 +102,10 @@ pub(super) enum Scheduler {
     /// Execute tasks across multiple threads.
     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
     MultiThread(MultiThread),
+
+    /// Execute tasks across multiple threads.
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+    MultiThreadAlt(MultiThreadAlt),
 }
 
 impl Runtime {
@@ -288,6 +299,15 @@ impl Runtime {
     /// [handle]: fn@Handle::block_on
     #[track_caller]
     pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+        #[cfg(all(
+            tokio_unstable,
+            tokio_taskdump,
+            feature = "rt",
+            target_os = "linux",
+            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        ))]
+        let future = super::task::trace::Trace::root(future);
+
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let future = crate::util::trace::task(
             future,
@@ -302,6 +322,8 @@ impl Runtime {
             Scheduler::CurrentThread(exec) => exec.block_on(&self.handle.inner, future),
             #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
             Scheduler::MultiThread(exec) => exec.block_on(&self.handle.inner, future),
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+            Scheduler::MultiThreadAlt(exec) => exec.block_on(&self.handle.inner, future),
         }
     }
 
@@ -422,6 +444,12 @@ impl Drop for Runtime {
                 // already in the runtime's context.
                 multi_thread.shutdown(&self.handle.inner);
             }
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+            Scheduler::MultiThreadAlt(multi_thread) => {
+                // The threaded scheduler drops its tasks on its worker threads, which is
+                // already in the runtime's context.
+                multi_thread.shutdown(&self.handle.inner);
+            }
         }
     }
 }

tokio/src/runtime/scheduler/block_in_place.rs

@@ -0,0 +1,21 @@
+use crate::runtime::scheduler;
+
+#[track_caller]
+pub(crate) fn block_in_place<F, R>(f: F) -> R
+where
+    F: FnOnce() -> R,
+{
+    #[cfg(tokio_unstable)]
+    {
+        use crate::runtime::{Handle, RuntimeFlavor::MultiThreadAlt};
+
+        match Handle::try_current().map(|h| h.runtime_flavor()) {
+            Ok(MultiThreadAlt) => {
+                return scheduler::multi_thread_alt::block_in_place(f);
+            }
+            _ => {}
+        }
+    }
+
+    scheduler::multi_thread::block_in_place(f)
+}

tokio/src/runtime/scheduler/current_thread.rs

@@ -1,10 +1,10 @@
 use crate::future::poll_fn;
 use crate::loom::sync::atomic::AtomicBool;
-use crate::loom::sync::{Arc, Mutex};
+use crate::loom::sync::Arc;
 use crate::runtime::driver::{self, Driver};
+use crate::runtime::scheduler::{self, Defer, Inject};
 use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task};
-use crate::runtime::{blocking, context, scheduler, Config};
-use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
+use crate::runtime::{blocking, context, Config, MetricsBatch, SchedulerMetrics, WorkerMetrics};
 use crate::sync::notify::Notify;
 use crate::util::atomic_cell::AtomicCell;
 use crate::util::{waker_ref, RngSeedGenerator, Wake, WakerRef};
@@ -15,6 +15,7 @@ use std::fmt;
 use std::future::Future;
 use std::sync::atomic::Ordering::{AcqRel, Release};
 use std::task::Poll::{Pending, Ready};
+use std::task::Waker;
 use std::time::Duration;
 
 /// Executes tasks on the current thread
@@ -46,7 +47,7 @@ pub(crate) struct Handle {
 /// a function that will perform the scheduling work and acts as a capability token.
 struct Core {
     /// Scheduler run queue
-    tasks: VecDeque<task::Notified<Arc<Handle>>>,
+    tasks: VecDeque<Notified>,
 
     /// Current tick
     tick: u32,
@@ -59,6 +60,9 @@ struct Core {
     /// Metrics batch
     metrics: MetricsBatch,
 
+    /// How often to check the global queue
+    global_queue_interval: u32,
+
     /// True if a task panicked without being handled and the runtime is
     /// configured to shutdown on unhandled panic.
     unhandled_panic: bool,
@@ -66,8 +70,8 @@ struct Core {
 
 /// Scheduler state shared between threads.
 struct Shared {
-    /// Remote run queue. None if the `Runtime` has been dropped.
-    queue: Mutex<Option<VecDeque<task::Notified<Arc<Handle>>>>>,
+    /// Remote run queue
+    inject: Inject<Arc<Handle>>,
 
     /// Collection of all active tasks spawned onto this executor.
     owned: OwnedTasks<Arc<Handle>>,
@@ -86,20 +90,29 @@ struct Shared {
 }
 
 /// Thread-local context.
-struct Context {
+///
+/// pub(crate) to store in `runtime::context`.
+pub(crate) struct Context {
     /// Scheduler handle
     handle: Arc<Handle>,
 
     /// Scheduler core, enabling the holder of `Context` to execute the
     /// scheduler.
     core: RefCell<Option<Box<Core>>>,
+
+    /// Deferred tasks, usually ones that called `task::yield_now()`.
+    pub(crate) defer: Defer,
 }
 
+type Notified = task::Notified<Arc<Handle>>;
+
 /// Initial queue capacity.
 const INITIAL_CAPACITY: usize = 64;
 
-// Tracks the current CurrentThread.
-scoped_thread_local!(static CURRENT: Context);
+/// Used if none is specified. This is a temporary constant and will be removed
+/// as we unify tuning logic between the multi-thread and current-thread
+/// schedulers.
+const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31;
 
 impl CurrentThread {
     pub(crate) fn new(
@@ -109,14 +122,21 @@ impl CurrentThread {
         seed_generator: RngSeedGenerator,
         config: Config,
     ) -> (CurrentThread, Arc<Handle>) {
+        let worker_metrics = WorkerMetrics::from_config(&config);
+
+        // Get the configured global queue interval, or use the default.
+        let global_queue_interval = config
+            .global_queue_interval
+            .unwrap_or(DEFAULT_GLOBAL_QUEUE_INTERVAL);
+
         let handle = Arc::new(Handle {
             shared: Shared {
-                queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))),
+                inject: Inject::new(),
                 owned: OwnedTasks::new(),
                 woken: AtomicBool::new(false),
                 config,
                 scheduler_metrics: SchedulerMetrics::new(),
-                worker_metrics: WorkerMetrics::new(),
+                worker_metrics,
             },
             driver: driver_handle,
             blocking_spawner,
@@ -127,7 +147,8 @@ impl CurrentThread {
             tasks: VecDeque::with_capacity(INITIAL_CAPACITY),
             tick: 0,
             driver: Some(driver),
-            metrics: MetricsBatch::new(),
+            metrics: MetricsBatch::new(&handle.shared.worker_metrics),
+            global_queue_interval,
             unhandled_panic: false,
         })));
 
@@ -143,48 +164,49 @@ impl CurrentThread {
     pub(crate) fn block_on<F: Future>(&self, handle: &scheduler::Handle, future: F) -> F::Output {
         pin!(future);
 
-        let mut enter = crate::runtime::context::enter_runtime(handle, false);
-        let handle = handle.as_current_thread();
+        crate::runtime::context::enter_runtime(handle, false, |blocking| {
+            let handle = handle.as_current_thread();
 
-        // Attempt to steal the scheduler core and block_on the future if we can
-        // there, otherwise, lets select on a notification that the core is
-        // available or the future is complete.
-        loop {
-            if let Some(core) = self.take_core(handle) {
-                return core.block_on(future);
-            } else {
-                let notified = self.notify.notified();
-                pin!(notified);
+            // Attempt to steal the scheduler core and block_on the future if we can
+            // there, otherwise, lets select on a notification that the core is
+            // available or the future is complete.
+            loop {
+                if let Some(core) = self.take_core(handle) {
+                    return core.block_on(future);
+                } else {
+                    let notified = self.notify.notified();
+                    pin!(notified);
 
-                if let Some(out) = enter
-                    .blocking
-                    .block_on(poll_fn(|cx| {
-                        if notified.as_mut().poll(cx).is_ready() {
-                            return Ready(None);
-                        }
+                    if let Some(out) = blocking
+                        .block_on(poll_fn(|cx| {
+                            if notified.as_mut().poll(cx).is_ready() {
+                                return Ready(None);
+                            }
 
-                        if let Ready(out) = future.as_mut().poll(cx) {
-                            return Ready(Some(out));
-                        }
+                            if let Ready(out) = future.as_mut().poll(cx) {
+                                return Ready(Some(out));
+                            }
 
-                        Pending
-                    }))
-                    .expect("Failed to `Enter::block_on`")
-                {
-                    return out;
+                            Pending
+                        }))
+                        .expect("Failed to `Enter::block_on`")
+                    {
+                        return out;
+                    }
                 }
             }
-        }
+        })
     }
 
     fn take_core(&self, handle: &Arc<Handle>) -> Option<CoreGuard<'_>> {
         let core = self.core.take()?;
 
         Some(CoreGuard {
-            context: Context {
+            context: scheduler::Context::CurrentThread(Context {
                 handle: handle.clone(),
                 core: RefCell::new(Some(core)),
-            },
+                defer: Defer::new(),
+            }),
             scheduler: self,
         })
     }
@@ -201,44 +223,60 @@ impl CurrentThread {
             None => panic!("Oh no! We never placed the Core back, this is a bug!"),
         };
 
-        core.enter(|mut core, _context| {
-            // Drain the OwnedTasks collection. This call also closes the
-            // collection, ensuring that no tasks are ever pushed after this
-            // call returns.
-            handle.shared.owned.close_and_shutdown_all();
+        // Check that the thread-local is not being destroyed
+        let tls_available = context::with_current(|_| ()).is_ok();
 
-            // Drain local queue
-            // We already shut down every task, so we just need to drop the task.
-            while let Some(task) = core.pop_task(handle) {
-                drop(task);
-            }
+        if tls_available {
+            core.enter(|core, _context| {
+                let core = shutdown2(core, handle);
+                (core, ())
+            });
+        } else {
+            // Shutdown without setting the context. `tokio::spawn` calls will
+            // fail, but those will fail either way because the thread-local is
+            // not available anymore.
+            let context = core.context.expect_current_thread();
+            let core = context.core.borrow_mut().take().unwrap();
 
-            // Drain remote queue and set it to None
-            let remote_queue = handle.shared.queue.lock().take();
-
-            // Using `Option::take` to replace the shared queue with `None`.
-            // We already shut down every task, so we just need to drop the task.
-            if let Some(remote_queue) = remote_queue {
-                for task in remote_queue {
-                    drop(task);
-                }
-            }
-
-            assert!(handle.shared.owned.is_empty());
-
-            // Submit metrics
-            core.metrics.submit(&handle.shared.worker_metrics);
-
-            // Shutdown the resource drivers
-            if let Some(driver) = core.driver.as_mut() {
-                driver.shutdown(&handle.driver);
-            }
-
-            (core, ())
-        });
+            let core = shutdown2(core, handle);
+            *context.core.borrow_mut() = Some(core);
+        }
     }
 }
 
+fn shutdown2(mut core: Box<Core>, handle: &Handle) -> Box<Core> {
+    // Drain the OwnedTasks collection. This call also closes the
+    // collection, ensuring that no tasks are ever pushed after this
+    // call returns.
+    handle.shared.owned.close_and_shutdown_all();
+
+    // Drain local queue
+    // We already shut down every task, so we just need to drop the task.
+    while let Some(task) = core.next_local_task(handle) {
+        drop(task);
+    }
+
+    // Close the injection queue
+    handle.shared.inject.close();
+
+    // Drain remote queue
+    while let Some(task) = handle.shared.inject.pop() {
+        drop(task);
+    }
+
+    assert!(handle.shared.owned.is_empty());
+
+    // Submit metrics
+    core.submit_metrics(handle);
+
+    // Shutdown the resource drivers
+    if let Some(driver) = core.driver.as_mut() {
+        driver.shutdown(&handle.driver);
+    }
+
+    core
+}
+
 impl fmt::Debug for CurrentThread {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt.debug_struct("CurrentThread").finish()
@@ -248,7 +286,23 @@ impl fmt::Debug for CurrentThread {
 // ===== impl Core =====
 
 impl Core {
-    fn pop_task(&mut self, handle: &Handle) -> Option<task::Notified<Arc<Handle>>> {
+    /// Get and increment the current tick
+    fn tick(&mut self) {
+        self.tick = self.tick.wrapping_add(1);
+    }
+
+    fn next_task(&mut self, handle: &Handle) -> Option<Notified> {
+        if self.tick % self.global_queue_interval == 0 {
+            handle
+                .next_remote_task()
+                .or_else(|| self.next_local_task(handle))
+        } else {
+            self.next_local_task(handle)
+                .or_else(|| handle.next_remote_task())
+        }
+    }
+
+    fn next_local_task(&mut self, handle: &Handle) -> Option<Notified> {
         let ret = self.tasks.pop_front();
         handle
             .shared
@@ -257,7 +311,7 @@ impl Core {
         ret
     }
 
-    fn push_task(&mut self, handle: &Handle, task: task::Notified<Arc<Handle>>) {
+    fn push_task(&mut self, handle: &Handle, task: Notified) {
         self.tasks.push_back(task);
         self.metrics.inc_local_schedule_count();
         handle
@@ -265,14 +319,18 @@ impl Core {
             .worker_metrics
             .set_queue_depth(self.tasks.len());
     }
+
+    fn submit_metrics(&mut self, handle: &Handle) {
+        self.metrics.submit(&handle.shared.worker_metrics);
+    }
 }
 
-fn did_defer_tasks() -> bool {
-    context::with_defer(|deferred| !deferred.is_empty()).unwrap()
-}
-
-fn wake_deferred_tasks() {
-    context::with_defer(|deferred| deferred.wake());
+#[cfg(tokio_taskdump)]
+fn wake_deferred_tasks_and_free(context: &Context) {
+    let wakers = context.defer.take_deferred();
+    for waker in wakers {
+        waker.wake();
+    }
 }
 
 // ===== impl Context =====
@@ -281,8 +339,10 @@ impl Context {
     /// Execute the closure with the given scheduler core stored in the
     /// thread-local context.
     fn run_task<R>(&self, mut core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) {
-        core.metrics.incr_poll_count();
-        self.enter(core, || crate::runtime::coop::budget(f))
+        core.metrics.start_poll();
+        let mut ret = self.enter(core, || crate::runtime::coop::budget(f));
+        ret.0.metrics.end_poll();
+        ret
     }
 
     /// Blocks the current thread until an event is received by the driver,
@@ -303,15 +363,14 @@ impl Context {
         if core.tasks.is_empty() {
             // Park until the thread is signaled
             core.metrics.about_to_park();
-            core.metrics.submit(&handle.shared.worker_metrics);
+            core.submit_metrics(handle);
 
             let (c, _) = self.enter(core, || {
                 driver.park(&handle.driver);
-                wake_deferred_tasks();
+                self.defer.wake();
             });
 
             core = c;
-            core.metrics.returned_from_park();
         }
 
         if let Some(f) = &handle.shared.config.after_unpark {
@@ -330,10 +389,11 @@ impl Context {
     fn park_yield(&self, mut core: Box<Core>, handle: &Handle) -> Box<Core> {
         let mut driver = core.driver.take().expect("driver missing");
 
-        core.metrics.submit(&handle.shared.worker_metrics);
+        core.submit_metrics(handle);
+
         let (mut core, _) = self.enter(core, || {
             driver.park_timeout(&handle.driver, Duration::from_millis(0));
-            wake_deferred_tasks();
+            self.defer.wake();
         });
 
         core.driver = Some(driver);
@@ -353,6 +413,10 @@ impl Context {
         let core = self.core.borrow_mut().take().expect("core missing");
         (core, ret)
     }
+
+    pub(crate) fn defer(&self, waker: &Waker) {
+        self.defer.defer(waker);
+    }
 }
 
 // ===== impl Handle =====
@@ -377,11 +441,58 @@ impl Handle {
         handle
     }
 
-    fn pop(&self) -> Option<task::Notified<Arc<Handle>>> {
-        match self.shared.queue.lock().as_mut() {
-            Some(queue) => queue.pop_front(),
-            None => None,
-        }
+    /// Capture a snapshot of this runtime's state.
+    #[cfg(all(
+        tokio_unstable,
+        tokio_taskdump,
+        target_os = "linux",
+        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    ))]
+    pub(crate) fn dump(&self) -> crate::runtime::Dump {
+        use crate::runtime::dump;
+        use task::trace::trace_current_thread;
+
+        let mut traces = vec![];
+
+        // todo: how to make this work outside of a runtime context?
+        context::with_scheduler(|maybe_context| {
+            // drain the local queue
+            let context = if let Some(context) = maybe_context {
+                context.expect_current_thread()
+            } else {
+                return;
+            };
+            let mut maybe_core = context.core.borrow_mut();
+            let core = if let Some(core) = maybe_core.as_mut() {
+                core
+            } else {
+                return;
+            };
+            let local = &mut core.tasks;
+
+            if self.shared.inject.is_closed() {
+                return;
+            }
+
+            traces = trace_current_thread(&self.shared.owned, local, &self.shared.inject)
+                .into_iter()
+                .map(dump::Task::new)
+                .collect();
+
+            // Avoid double borrow panic
+            drop(maybe_core);
+
+            // Taking a taskdump could wakes every task, but we probably don't want
+            // the `yield_now` vector to be that large under normal circumstances.
+            // Therefore, we free its allocation.
+            wake_deferred_tasks_and_free(context);
+        });
+
+        dump::Dump::new(traces)
+    }
+
+    fn next_remote_task(&self) -> Option<Notified> {
+        self.shared.inject.pop()
     }
 
     fn waker_ref(me: &Arc<Self>) -> WakerRef<'_> {
@@ -404,14 +515,7 @@ cfg_metrics! {
         }
 
         pub(crate) fn injection_queue_depth(&self) -> usize {
-            // TODO: avoid having to lock. The multi-threaded injection queue
-            // could probably be used here.
-            self.shared
-                .queue
-                .lock()
-                .as_ref()
-                .map(|queue| queue.len())
-                .unwrap_or(0)
+            self.shared.inject.len()
         }
 
         pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
@@ -419,6 +523,10 @@ cfg_metrics! {
             &self.shared.worker_metrics
         }
 
+        pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+            self.worker_metrics(worker).queue_depth()
+        }
+
         pub(crate) fn num_blocking_threads(&self) -> usize {
             self.blocking_spawner.num_threads()
         }
@@ -430,6 +538,10 @@ cfg_metrics! {
         pub(crate) fn blocking_queue_depth(&self) -> usize {
             self.blocking_spawner.queue_depth()
         }
+
+        pub(crate) fn active_tasks_count(&self) -> usize {
+            self.shared.owned.active_tasks_count()
+        }
     }
 }
 
@@ -447,8 +559,10 @@ impl Schedule for Arc<Handle> {
     }
 
     fn schedule(&self, task: task::Notified<Self>) {
-        CURRENT.with(|maybe_cx| match maybe_cx {
-            Some(cx) if Arc::ptr_eq(self, &cx.handle) => {
+        use scheduler::Context::CurrentThread;
+
+        context::with_scheduler(|maybe_cx| match maybe_cx {
+            Some(CurrentThread(cx)) if Arc::ptr_eq(self, &cx.handle) => {
                 let mut core = cx.core.borrow_mut();
 
                 // If `None`, the runtime is shutting down, so there is no need
@@ -461,14 +575,9 @@ impl Schedule for Arc<Handle> {
                 // Track that a task was scheduled from **outside** of the runtime.
                 self.shared.scheduler_metrics.inc_remote_schedule_count();
 
-                // If the queue is None, then the runtime has shut down. We
-                // don't need to do anything with the notification in that case.
-                let mut guard = self.shared.queue.lock();
-                if let Some(queue) = guard.as_mut() {
-                    queue.push_back(task);
-                    drop(guard);
-                    self.driver.unpark();
-                }
+                // Schedule the task
+                self.shared.inject.push(task);
+                self.driver.unpark();
             }
         });
     }
@@ -482,11 +591,14 @@ impl Schedule for Arc<Handle> {
                     // Do nothing
                 }
                 UnhandledPanic::ShutdownRuntime => {
+                    use scheduler::Context::CurrentThread;
+
                     // This hook is only called from within the runtime, so
-                    // `CURRENT` should match with `&self`, i.e. there is no
-                    // opportunity for a nested scheduler to be called.
-                    CURRENT.with(|maybe_cx| match maybe_cx {
-                        Some(cx) if Arc::ptr_eq(self, &cx.handle) => {
+                    // `context::with_scheduler` should match with `&self`, i.e.
+                    // there is no opportunity for a nested scheduler to be
+                    // called.
+                    context::with_scheduler(|maybe_cx| match maybe_cx {
+                        Some(CurrentThread(cx)) if Arc::ptr_eq(self, &cx.handle) => {
                             let mut core = cx.core.borrow_mut();
 
                             // If `None`, the runtime is shutting down, so there is no need to signal shutdown
@@ -520,7 +632,7 @@ impl Wake for Handle {
 /// Used to ensure we always place the `Core` value back into its slot in
 /// `CurrentThread`, even if the future panics.
 struct CoreGuard<'a> {
-    context: Context,
+    context: scheduler::Context,
     scheduler: &'a CurrentThread,
 }
 
@@ -533,6 +645,8 @@ impl CoreGuard<'_> {
 
             pin!(future);
 
+            core.metrics.start_processing_scheduled_tasks();
+
             'outer: loop {
                 let handle = &context.handle;
 
@@ -554,25 +668,23 @@ impl CoreGuard<'_> {
                         return (core, None);
                     }
 
-                    // Get and increment the current tick
-                    let tick = core.tick;
-                    core.tick = core.tick.wrapping_add(1);
+                    core.tick();
 
-                    let entry = if tick % handle.shared.config.global_queue_interval == 0 {
-                        handle.pop().or_else(|| core.tasks.pop_front())
-                    } else {
-                        core.tasks.pop_front().or_else(|| handle.pop())
-                    };
+                    let entry = core.next_task(handle);
 
                     let task = match entry {
                         Some(entry) => entry,
                         None => {
-                            core = if did_defer_tasks() {
+                            core.metrics.end_processing_scheduled_tasks();
+
+                            core = if !context.defer.is_empty() {
                                 context.park_yield(core, handle)
                             } else {
                                 context.park(core, handle)
                             };
 
+                            core.metrics.start_processing_scheduled_tasks();
+
                             // Try polling the `block_on` future next
                             continue 'outer;
                         }
@@ -587,9 +699,13 @@ impl CoreGuard<'_> {
                     core = c;
                 }
 
+                core.metrics.end_processing_scheduled_tasks();
+
                 // Yield to the driver, this drives the timer and pulls any
                 // pending I/O events.
                 core = context.park_yield(core, handle);
+
+                core.metrics.start_processing_scheduled_tasks();
             }
         });
 
@@ -608,13 +724,15 @@ impl CoreGuard<'_> {
     where
         F: FnOnce(Box<Core>, &Context) -> (Box<Core>, R),
     {
+        let context = self.context.expect_current_thread();
+
         // Remove `core` from `context` to pass into the closure.
-        let core = self.context.core.borrow_mut().take().expect("core missing");
+        let core = context.core.borrow_mut().take().expect("core missing");
 
         // Call the closure and place `core` back
-        let (core, ret) = CURRENT.set(&self.context, || f(core, &self.context));
+        let (core, ret) = context::set_scheduler(&self.context, || f(core, context));
 
-        *self.context.core.borrow_mut() = Some(core);
+        *context.core.borrow_mut() = Some(core);
 
         ret
     }
@@ -622,7 +740,9 @@ impl CoreGuard<'_> {
 
 impl Drop for CoreGuard<'_> {
     fn drop(&mut self) {
-        if let Some(core) = self.context.core.borrow_mut().take() {
+        let context = self.context.expect_current_thread();
+
+        if let Some(core) = context.core.borrow_mut().take() {
             // Replace old scheduler back into the state to allow
             // other threads to pick it up and drive it.
             self.scheduler.core.set(core);

tokio/src/runtime/scheduler/defer.rs

@@ -0,0 +1,43 @@
+use std::cell::RefCell;
+use std::task::Waker;
+
+pub(crate) struct Defer {
+    deferred: RefCell<Vec<Waker>>,
+}
+
+impl Defer {
+    pub(crate) fn new() -> Defer {
+        Defer {
+            deferred: Default::default(),
+        }
+    }
+
+    pub(crate) fn defer(&self, waker: &Waker) {
+        let mut deferred = self.deferred.borrow_mut();
+
+        // If the same task adds itself a bunch of times, then only add it once.
+        if let Some(last) = deferred.last() {
+            if last.will_wake(waker) {
+                return;
+            }
+        }
+
+        deferred.push(waker.clone());
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.deferred.borrow().is_empty()
+    }
+
+    pub(crate) fn wake(&self) {
+        while let Some(waker) = self.deferred.borrow_mut().pop() {
+            waker.wake();
+        }
+    }
+
+    #[cfg(tokio_taskdump)]
+    pub(crate) fn take_deferred(&self) -> Vec<Waker> {
+        let mut deferred = self.deferred.borrow_mut();
+        std::mem::take(&mut *deferred)
+    }
+}

tokio/src/runtime/scheduler/inject.rs

@@ -0,0 +1,72 @@
+//! Inject queue used to send wakeups to a work-stealing scheduler
+
+use crate::loom::sync::Mutex;
+use crate::runtime::task;
+
+mod pop;
+pub(crate) use pop::Pop;
+
+mod shared;
+pub(crate) use shared::Shared;
+
+mod synced;
+pub(crate) use synced::Synced;
+
+cfg_rt_multi_thread! {
+    mod rt_multi_thread;
+}
+
+cfg_metrics! {
+    mod metrics;
+}
+
+/// Growable, MPMC queue used to inject new tasks into the scheduler and as an
+/// overflow queue when the local, fixed-size, array queue overflows.
+pub(crate) struct Inject<T: 'static> {
+    shared: Shared<T>,
+    synced: Mutex<Synced>,
+}
+
+impl<T: 'static> Inject<T> {
+    pub(crate) fn new() -> Inject<T> {
+        let (shared, synced) = Shared::new();
+
+        Inject {
+            shared,
+            synced: Mutex::new(synced),
+        }
+    }
+
+    // Kind of annoying to have to include the cfg here
+    #[cfg(tokio_taskdump)]
+    pub(crate) fn is_closed(&self) -> bool {
+        let synced = self.synced.lock();
+        self.shared.is_closed(&synced)
+    }
+
+    /// Closes the injection queue, returns `true` if the queue is open when the
+    /// transition is made.
+    pub(crate) fn close(&self) -> bool {
+        let mut synced = self.synced.lock();
+        self.shared.close(&mut synced)
+    }
+
+    /// Pushes a value into the queue.
+    ///
+    /// This does nothing if the queue is closed.
+    pub(crate) fn push(&self, task: task::Notified<T>) {
+        let mut synced = self.synced.lock();
+        // safety: passing correct `Synced`
+        unsafe { self.shared.push(&mut synced, task) }
+    }
+
+    pub(crate) fn pop(&self) -> Option<task::Notified<T>> {
+        if self.shared.is_empty() {
+            return None;
+        }
+
+        let mut synced = self.synced.lock();
+        // safety: passing correct `Synced`
+        unsafe { self.shared.pop(&mut synced) }
+    }
+}

tokio/src/runtime/scheduler/inject/metrics.rs

@@ -0,0 +1,7 @@
+use super::Inject;
+
+impl<T: 'static> Inject<T> {
+    pub(crate) fn len(&self) -> usize {
+        self.shared.len()
+    }
+}

tokio/src/runtime/scheduler/inject/pop.rs

@@ -0,0 +1,55 @@
+use super::Synced;
+
+use crate::runtime::task;
+
+use std::marker::PhantomData;
+
+pub(crate) struct Pop<'a, T: 'static> {
+    len: usize,
+    synced: &'a mut Synced,
+    _p: PhantomData<T>,
+}
+
+impl<'a, T: 'static> Pop<'a, T> {
+    pub(super) fn new(len: usize, synced: &'a mut Synced) -> Pop<'a, T> {
+        Pop {
+            len,
+            synced,
+            _p: PhantomData,
+        }
+    }
+}
+
+impl<'a, T: 'static> Iterator for Pop<'a, T> {
+    type Item = task::Notified<T>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.len == 0 {
+            return None;
+        }
+
+        let ret = self.synced.pop();
+
+        // Should be `Some` when `len > 0`
+        debug_assert!(ret.is_some());
+
+        self.len -= 1;
+        ret
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.len, Some(self.len))
+    }
+}
+
+impl<'a, T: 'static> ExactSizeIterator for Pop<'a, T> {
+    fn len(&self) -> usize {
+        self.len
+    }
+}
+
+impl<'a, T: 'static> Drop for Pop<'a, T> {
+    fn drop(&mut self) {
+        for _ in self.by_ref() {}
+    }
+}

tokio/src/runtime/scheduler/inject/rt_multi_thread.rs

@@ -0,0 +1,98 @@
+use super::{Shared, Synced};
+
+use crate::runtime::scheduler::Lock;
+use crate::runtime::task;
+
+use std::sync::atomic::Ordering::Release;
+
+impl<'a> Lock<Synced> for &'a mut Synced {
+    type Handle = &'a mut Synced;
+
+    fn lock(self) -> Self::Handle {
+        self
+    }
+}
+
+impl AsMut<Synced> for Synced {
+    fn as_mut(&mut self) -> &mut Synced {
+        self
+    }
+}
+
+impl<T: 'static> Shared<T> {
+    /// Pushes several values into the queue.
+    ///
+    /// # Safety
+    ///
+    /// Must be called with the same `Synced` instance returned by `Inject::new`
+    #[inline]
+    pub(crate) unsafe fn push_batch<L, I>(&self, shared: L, mut iter: I)
+    where
+        L: Lock<Synced>,
+        I: Iterator<Item = task::Notified<T>>,
+    {
+        let first = match iter.next() {
+            Some(first) => first.into_raw(),
+            None => return,
+        };
+
+        // Link up all the tasks.
+        let mut prev = first;
+        let mut counter = 1;
+
+        // We are going to be called with an `std::iter::Chain`, and that
+        // iterator overrides `for_each` to something that is easier for the
+        // compiler to optimize than a loop.
+        iter.for_each(|next| {
+            let next = next.into_raw();
+
+            // safety: Holding the Notified for a task guarantees exclusive
+            // access to the `queue_next` field.
+            unsafe { prev.set_queue_next(Some(next)) };
+            prev = next;
+            counter += 1;
+        });
+
+        // Now that the tasks are linked together, insert them into the
+        // linked list.
+        self.push_batch_inner(shared, first, prev, counter);
+    }
+
+    /// Inserts several tasks that have been linked together into the queue.
+    ///
+    /// The provided head and tail may be be the same task. In this case, a
+    /// single task is inserted.
+    #[inline]
+    unsafe fn push_batch_inner<L>(
+        &self,
+        shared: L,
+        batch_head: task::RawTask,
+        batch_tail: task::RawTask,
+        num: usize,
+    ) where
+        L: Lock<Synced>,
+    {
+        debug_assert!(unsafe { batch_tail.get_queue_next().is_none() });
+
+        let mut synced = shared.lock();
+        let synced = synced.as_mut();
+
+        if let Some(tail) = synced.tail {
+            unsafe {
+                tail.set_queue_next(Some(batch_head));
+            }
+        } else {
+            synced.head = Some(batch_head);
+        }
+
+        synced.tail = Some(batch_tail);
+
+        // Increment the count.
+        //
+        // safety: All updates to the len atomic are guarded by the mutex. As
+        // such, a non-atomic load followed by a store is safe.
+        let len = self.len.unsync_load();
+
+        self.len.store(len + num, Release);
+    }
+}

tokio/src/runtime/scheduler/inject/shared.rs

@@ -0,0 +1,119 @@
+use super::{Pop, Synced};
+
+use crate::loom::sync::atomic::AtomicUsize;
+use crate::runtime::task;
+
+use std::marker::PhantomData;
+use std::sync::atomic::Ordering::{Acquire, Release};
+
+pub(crate) struct Shared<T: 'static> {
+    /// Number of pending tasks in the queue. This helps prevent unnecessary
+    /// locking in the hot path.
+    pub(super) len: AtomicUsize,
+
+    _p: PhantomData<T>,
+}
+
+unsafe impl<T> Send for Shared<T> {}
+unsafe impl<T> Sync for Shared<T> {}
+
+impl<T: 'static> Shared<T> {
+    pub(crate) fn new() -> (Shared<T>, Synced) {
+        let inject = Shared {
+            len: AtomicUsize::new(0),
+            _p: PhantomData,
+        };
+
+        let synced = Synced {
+            is_closed: false,
+            head: None,
+            tail: None,
+        };
+
+        (inject, synced)
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    // Kind of annoying to have to include the cfg here
+    #[cfg(any(tokio_taskdump, all(feature = "rt-multi-thread", not(tokio_wasi))))]
+    pub(crate) fn is_closed(&self, synced: &Synced) -> bool {
+        synced.is_closed
+    }
+
+    /// Closes the injection queue, returns `true` if the queue is open when the
+    /// transition is made.
+    pub(crate) fn close(&self, synced: &mut Synced) -> bool {
+        if synced.is_closed {
+            return false;
+        }
+
+        synced.is_closed = true;
+        true
+    }
+
+    pub(crate) fn len(&self) -> usize {
+        self.len.load(Acquire)
+    }
+
+    /// Pushes a value into the queue.
+    ///
+    /// This does nothing if the queue is closed.
+    ///
+    /// # Safety
+    ///
+    /// Must be called with the same `Synced` instance returned by `Inject::new`
+    pub(crate) unsafe fn push(&self, synced: &mut Synced, task: task::Notified<T>) {
+        if synced.is_closed {
+            return;
+        }
+
+        // safety: only mutated with the lock held
+        let len = self.len.unsync_load();
+        let task = task.into_raw();
+
+        // The next pointer should already be null
+        debug_assert!(unsafe { task.get_queue_next().is_none() });
+
+        if let Some(tail) = synced.tail {
+            // safety: Holding the Notified for a task guarantees exclusive
+            // access to the `queue_next` field.
+            unsafe { tail.set_queue_next(Some(task)) };
+        } else {
+            synced.head = Some(task);
+        }
+
+        synced.tail = Some(task);
+        self.len.store(len + 1, Release);
+    }
+
+    /// Pop a value from the queue.
+    ///
+    /// # Safety
+    ///
+    /// Must be called with the same `Synced` instance returned by `Inject::new`
+    pub(crate) unsafe fn pop(&self, synced: &mut Synced) -> Option<task::Notified<T>> {
+        self.pop_n(synced, 1).next()
+    }
+
+    /// Pop `n` values from the queue
+    ///
+    /// # Safety
+    ///
+    /// Must be called with the same `Synced` instance returned by `Inject::new`
+    pub(crate) unsafe fn pop_n<'a>(&'a self, synced: &'a mut Synced, n: usize) -> Pop<'a, T> {
+        use std::cmp;
+
+        // safety: All updates to the len atomic are guarded by the mutex. As
+        // such, a non-atomic load followed by a store is safe.
+        let len = self.len.unsync_load();
+        let n = cmp::min(n, len);
+
+        // Decrement the count.
+        self.len.store(len - n, Release);
+
+        Pop::new(n, synced)
+    }
+}

tokio/src/runtime/scheduler/inject/synced.rs

@@ -0,0 +1,32 @@
+use crate::runtime::task;
+
+pub(crate) struct Synced {
+    /// True if the queue is closed.
+    pub(super) is_closed: bool,
+
+    /// Linked-list head.
+    pub(super) head: Option<task::RawTask>,
+
+    /// Linked-list tail.
+    pub(super) tail: Option<task::RawTask>,
+}
+
+unsafe impl Send for Synced {}
+unsafe impl Sync for Synced {}
+
+impl Synced {
+    pub(super) fn pop<T: 'static>(&mut self) -> Option<task::Notified<T>> {
+        let task = self.head?;
+
+        self.head = unsafe { task.get_queue_next() };
+
+        if self.head.is_none() {
+            self.tail = None;
+        }
+
+        unsafe { task.set_queue_next(None) };
+
+        // safety: a `Notified` is pushed into the queue and now it is popped!
+        Some(unsafe { task::Notified::from_raw(task) })
+    }
+}

tokio/src/runtime/scheduler/lock.rs

@@ -0,0 +1,6 @@
+/// A lock (mutex) yielding generic data.
+pub(crate) trait Lock<T> {
+    type Handle: AsMut<T>;
+
+    fn lock(self) -> Self::Handle;
+}

tokio/src/runtime/scheduler/mod.rs

@@ -1,11 +1,28 @@
 cfg_rt! {
     pub(crate) mod current_thread;
     pub(crate) use current_thread::CurrentThread;
+
+    mod defer;
+    use defer::Defer;
+
+    pub(crate) mod inject;
+    pub(crate) use inject::Inject;
 }
 
 cfg_rt_multi_thread! {
+    mod block_in_place;
+    pub(crate) use block_in_place::block_in_place;
+
+    mod lock;
+    use lock::Lock;
+
     pub(crate) mod multi_thread;
     pub(crate) use multi_thread::MultiThread;
+
+    cfg_unstable! {
+        pub(crate) mod multi_thread_alt;
+        pub(crate) use multi_thread_alt::MultiThread as MultiThreadAlt;
+    }
 }
 
 use crate::runtime::driver;
@@ -18,6 +35,9 @@ pub(crate) enum Handle {
     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
     MultiThread(Arc<multi_thread::Handle>),
 
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+    MultiThreadAlt(Arc<multi_thread_alt::Handle>),
+
     // TODO: This is to avoid triggering "dead code" warnings many other places
     // in the codebase. Remove this during a later cleanup
     #[cfg(not(feature = "rt"))]
@@ -25,6 +45,17 @@ pub(crate) enum Handle {
     Disabled,
 }
 
+#[cfg(feature = "rt")]
+pub(super) enum Context {
+    CurrentThread(current_thread::Context),
+
+    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+    MultiThread(multi_thread::Context),
+
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+    MultiThreadAlt(multi_thread_alt::Context),
+}
+
 impl Handle {
     #[cfg_attr(not(feature = "full"), allow(dead_code))]
     pub(crate) fn driver(&self) -> &driver::Handle {
@@ -35,6 +66,9 @@ impl Handle {
             #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
             Handle::MultiThread(ref h) => &h.driver,
 
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+            Handle::MultiThreadAlt(ref h) => &h.driver,
+
             #[cfg(not(feature = "rt"))]
             Handle::Disabled => unreachable!(),
         }
@@ -48,23 +82,33 @@ cfg_rt! {
     use crate::runtime::context;
     use crate::task::JoinHandle;
     use crate::util::RngSeedGenerator;
+    use std::task::Waker;
+
+    macro_rules! match_flavor {
+        ($self:expr, $ty:ident($h:ident) => $e:expr) => {
+            match $self {
+                $ty::CurrentThread($h) => $e,
+
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                $ty::MultiThread($h) => $e,
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                $ty::MultiThreadAlt($h) => $e,
+            }
+        }
+    }
 
     impl Handle {
         #[track_caller]
         pub(crate) fn current() -> Handle {
-            match context::try_current() {
+            match context::with_current(Clone::clone) {
                 Ok(handle) => handle,
                 Err(e) => panic!("{}", e),
             }
         }
 
         pub(crate) fn blocking_spawner(&self) -> &blocking::Spawner {
-            match self {
-                Handle::CurrentThread(h) => &h.blocking_spawner,
-
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                Handle::MultiThread(h) => &h.blocking_spawner,
-            }
+            match_flavor!(self, Handle(h) => &h.blocking_spawner)
         }
 
         pub(crate) fn spawn<F>(&self, future: F, id: Id) -> JoinHandle<F::Output>
@@ -77,6 +121,9 @@ cfg_rt! {
 
                 #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                 Handle::MultiThread(h) => multi_thread::Handle::spawn(h, future, id),
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                Handle::MultiThreadAlt(h) => multi_thread_alt::Handle::spawn(h, future, id),
             }
         }
 
@@ -86,6 +133,9 @@ cfg_rt! {
 
                 #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                 Handle::MultiThread(ref h) => h.shutdown(),
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                Handle::MultiThreadAlt(ref h) => h.shutdown(),
             }
         }
 
@@ -95,6 +145,9 @@ cfg_rt! {
 
                 #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                 Handle::MultiThread(h) => &h.seed_generator,
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                Handle::MultiThreadAlt(h) => &h.seed_generator,
             }
         }
 
@@ -105,6 +158,17 @@ cfg_rt! {
                 _ => panic!("not a CurrentThread handle"),
             }
         }
+
+        cfg_rt_multi_thread! {
+            cfg_unstable! {
+                pub(crate) fn expect_multi_thread_alt(&self) -> &Arc<multi_thread_alt::Handle> {
+                    match self {
+                        Handle::MultiThreadAlt(handle) => handle,
+                        _ => panic!("not a `MultiThreadAlt` handle"),
+                    }
+                }
+            }
+        }
     }
 
     cfg_metrics! {
@@ -116,6 +180,8 @@ cfg_rt! {
                     Handle::CurrentThread(_) => 1,
                     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                     Handle::MultiThread(handle) => handle.num_workers(),
+                    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThreadAlt(handle) => handle.num_workers(),
                 }
             }
 
@@ -124,6 +190,8 @@ cfg_rt! {
                     Handle::CurrentThread(handle) => handle.num_blocking_threads(),
                     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                     Handle::MultiThread(handle) => handle.num_blocking_threads(),
+                    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThreadAlt(handle) => handle.num_blocking_threads(),
                 }
             }
 
@@ -132,6 +200,18 @@ cfg_rt! {
                     Handle::CurrentThread(handle) => handle.num_idle_blocking_threads(),
                     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                     Handle::MultiThread(handle) => handle.num_idle_blocking_threads(),
+                    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThreadAlt(handle) => handle.num_idle_blocking_threads(),
+                }
+            }
+
+            pub(crate) fn active_tasks_count(&self) -> usize {
+                match self {
+                    Handle::CurrentThread(handle) => handle.active_tasks_count(),
+                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThread(handle) => handle.active_tasks_count(),
+                    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThreadAlt(handle) => handle.active_tasks_count(),
                 }
             }
 
@@ -140,6 +220,8 @@ cfg_rt! {
                     Handle::CurrentThread(handle) => handle.scheduler_metrics(),
                     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                     Handle::MultiThread(handle) => handle.scheduler_metrics(),
+                    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThreadAlt(handle) => handle.scheduler_metrics(),
                 }
             }
 
@@ -148,30 +230,55 @@ cfg_rt! {
                     Handle::CurrentThread(handle) => handle.worker_metrics(worker),
                     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
                     Handle::MultiThread(handle) => handle.worker_metrics(worker),
+                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                    Handle::MultiThreadAlt(handle) => handle.worker_metrics(worker),
                 }
             }
 
             pub(crate) fn injection_queue_depth(&self) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.injection_queue_depth(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.injection_queue_depth(),
-                }
+                match_flavor!(self, Handle(handle) => handle.injection_queue_depth())
             }
 
             pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.worker_metrics(worker).queue_depth(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.worker_local_queue_depth(worker),
-                }
+                match_flavor!(self, Handle(handle) => handle.worker_local_queue_depth(worker))
             }
 
             pub(crate) fn blocking_queue_depth(&self) -> usize {
+                match_flavor!(self, Handle(handle) => handle.blocking_queue_depth())
+            }
+        }
+    }
+
+    impl Context {
+        #[track_caller]
+        pub(crate) fn expect_current_thread(&self) -> &current_thread::Context {
+            match self {
+                Context::CurrentThread(context) => context,
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                _ => panic!("expected `CurrentThread::Context`")
+            }
+        }
+
+        pub(crate) fn defer(&self, waker: &Waker) {
+            match_flavor!(self, Context(context) => context.defer(waker))
+        }
+
+        cfg_rt_multi_thread! {
+            #[track_caller]
+            pub(crate) fn expect_multi_thread(&self) -> &multi_thread::Context {
                 match self {
-                    Handle::CurrentThread(handle) => handle.blocking_queue_depth(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.blocking_queue_depth(),
+                    Context::MultiThread(context) => context,
+                    _ => panic!("expected `MultiThread::Context`")
+                }
+            }
+
+            cfg_unstable! {
+                #[track_caller]
+                pub(crate) fn expect_multi_thread_alt(&self) -> &multi_thread_alt::Context {
+                    match self {
+                        Context::MultiThreadAlt(context) => context,
+                        _ => panic!("expected `MultiThreadAlt::Context`")
+                    }
                 }
             }
         }

tokio/src/runtime/scheduler/multi_thread/counters.rs

@@ -0,0 +1,62 @@
+#[cfg(tokio_internal_mt_counters)]
+mod imp {
+    use std::sync::atomic::AtomicUsize;
+    use std::sync::atomic::Ordering::Relaxed;
+
+    static NUM_MAINTENANCE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_NOTIFY_LOCAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_UNPARKS_LOCAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_SCHEDULES: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_CAPPED: AtomicUsize = AtomicUsize::new(0);
+
+    impl Drop for super::Counters {
+        fn drop(&mut self) {
+            let notifies_local = NUM_NOTIFY_LOCAL.load(Relaxed);
+            let unparks_local = NUM_UNPARKS_LOCAL.load(Relaxed);
+            let maintenance = NUM_MAINTENANCE.load(Relaxed);
+            let lifo_scheds = NUM_LIFO_SCHEDULES.load(Relaxed);
+            let lifo_capped = NUM_LIFO_CAPPED.load(Relaxed);
+
+            println!("---");
+            println!("notifies (local): {}", notifies_local);
+            println!(" unparks (local): {}", unparks_local);
+            println!("     maintenance: {}", maintenance);
+            println!("  LIFO schedules: {}", lifo_scheds);
+            println!("     LIFO capped: {}", lifo_capped);
+        }
+    }
+
+    pub(crate) fn inc_num_inc_notify_local() {
+        NUM_NOTIFY_LOCAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_unparks_local() {
+        NUM_UNPARKS_LOCAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_maintenance() {
+        NUM_MAINTENANCE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_lifo_schedules() {
+        NUM_LIFO_SCHEDULES.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_lifo_capped() {
+        NUM_LIFO_CAPPED.fetch_add(1, Relaxed);
+    }
+}
+
+#[cfg(not(tokio_internal_mt_counters))]
+mod imp {
+    pub(crate) fn inc_num_inc_notify_local() {}
+    pub(crate) fn inc_num_unparks_local() {}
+    pub(crate) fn inc_num_maintenance() {}
+    pub(crate) fn inc_lifo_schedules() {}
+    pub(crate) fn inc_lifo_capped() {}
+}
+
+#[derive(Debug)]
+pub(crate) struct Counters;
+
+pub(super) use imp::*;

tokio/src/runtime/scheduler/multi_thread/handle.rs

@@ -9,6 +9,14 @@ use crate::util::RngSeedGenerator;
 
 use std::fmt;
 
+cfg_metrics! {
+    mod metrics;
+}
+
+cfg_taskdump! {
+    mod taskdump;
+}
+
 /// Handle to the multi thread scheduler
 pub(crate) struct Handle {
     /// Task spawner
@@ -53,44 +61,6 @@ impl Handle {
     }
 }
 
-cfg_metrics! {
-    use crate::runtime::{SchedulerMetrics, WorkerMetrics};
-
-    impl Handle {
-        pub(crate) fn num_workers(&self) -> usize {
-            self.shared.worker_metrics.len()
-        }
-
-        pub(crate) fn num_blocking_threads(&self) -> usize {
-            self.blocking_spawner.num_threads()
-        }
-
-        pub(crate) fn num_idle_blocking_threads(&self) -> usize {
-            self.blocking_spawner.num_idle_threads()
-        }
-
-        pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
-            &self.shared.scheduler_metrics
-        }
-
-        pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
-            &self.shared.worker_metrics[worker]
-        }
-
-        pub(crate) fn injection_queue_depth(&self) -> usize {
-            self.shared.injection_queue_depth()
-        }
-
-        pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
-            self.shared.worker_local_queue_depth(worker)
-        }
-
-        pub(crate) fn blocking_queue_depth(&self) -> usize {
-            self.blocking_spawner.queue_depth()
-        }
-    }
-}
-
 impl fmt::Debug for Handle {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt.debug_struct("multi_thread::Handle { ... }").finish()

tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs

@@ -0,0 +1,41 @@
+use super::Handle;
+
+use crate::runtime::{SchedulerMetrics, WorkerMetrics};
+
+impl Handle {
+    pub(crate) fn num_workers(&self) -> usize {
+        self.shared.worker_metrics.len()
+    }
+
+    pub(crate) fn num_blocking_threads(&self) -> usize {
+        self.blocking_spawner.num_threads()
+    }
+
+    pub(crate) fn num_idle_blocking_threads(&self) -> usize {
+        self.blocking_spawner.num_idle_threads()
+    }
+
+    pub(crate) fn active_tasks_count(&self) -> usize {
+        self.shared.owned.active_tasks_count()
+    }
+
+    pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
+        &self.shared.scheduler_metrics
+    }
+
+    pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
+        &self.shared.worker_metrics[worker]
+    }
+
+    pub(crate) fn injection_queue_depth(&self) -> usize {
+        self.shared.injection_queue_depth()
+    }
+
+    pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+        self.shared.worker_local_queue_depth(worker)
+    }
+
+    pub(crate) fn blocking_queue_depth(&self) -> usize {
+        self.blocking_spawner.queue_depth()
+    }
+}

tokio/src/runtime/scheduler/multi_thread/handle/taskdump.rs

@@ -0,0 +1,26 @@
+use super::Handle;
+
+use crate::runtime::Dump;
+
+impl Handle {
+    pub(crate) async fn dump(&self) -> Dump {
+        let trace_status = &self.shared.trace_status;
+
+        // If a dump is in progress, block.
+        trace_status.start_trace_request(&self).await;
+
+        let result = loop {
+            if let Some(result) = trace_status.take_result() {
+                break result;
+            } else {
+                self.notify_all();
+                trace_status.result_ready.notified().await;
+            }
+        };
+
+        // Allow other queued dumps to proceed.
+        trace_status.end_trace_request(&self).await;
+
+        result
+    }
+}

tokio/src/runtime/scheduler/multi_thread/idle.rs

@@ -1,7 +1,7 @@
 //! Coordinates idling workers
 
 use crate::loom::sync::atomic::AtomicUsize;
-use crate::loom::sync::Mutex;
+use crate::runtime::scheduler::multi_thread::Shared;
 
 use std::fmt;
 use std::sync::atomic::Ordering::{self, SeqCst};
@@ -13,13 +13,16 @@ pub(super) struct Idle {
     /// Used as a fast-path to avoid acquiring the lock when needed.
     state: AtomicUsize,
 
-    /// Sleeping workers
-    sleepers: Mutex<Vec<usize>>,
-
     /// Total number of workers.
     num_workers: usize,
 }
 
+/// Data synchronized by the scheduler mutex
+pub(super) struct Synced {
+    /// Sleeping workers
+    sleepers: Vec<usize>,
+}
+
 const UNPARK_SHIFT: usize = 16;
 const UNPARK_MASK: usize = !SEARCH_MASK;
 const SEARCH_MASK: usize = (1 << UNPARK_SHIFT) - 1;
@@ -28,19 +31,24 @@ const SEARCH_MASK: usize = (1 << UNPARK_SHIFT) - 1;
 struct State(usize);
 
 impl Idle {
-    pub(super) fn new(num_workers: usize) -> Idle {
+    pub(super) fn new(num_workers: usize) -> (Idle, Synced) {
         let init = State::new(num_workers);
 
-        Idle {
+        let idle = Idle {
             state: AtomicUsize::new(init.into()),
-            sleepers: Mutex::new(Vec::with_capacity(num_workers)),
             num_workers,
-        }
+        };
+
+        let synced = Synced {
+            sleepers: Vec::with_capacity(num_workers),
+        };
+
+        (idle, synced)
     }
 
     /// If there are no workers actively searching, returns the index of a
     /// worker currently sleeping.
-    pub(super) fn worker_to_notify(&self) -> Option<usize> {
+    pub(super) fn worker_to_notify(&self, shared: &Shared) -> Option<usize> {
         // If at least one worker is spinning, work being notified will
         // eventually be found. A searching thread will find **some** work and
         // notify another worker, eventually leading to our work being found.
@@ -55,7 +63,7 @@ impl Idle {
         }
 
         // Acquire the lock
-        let mut sleepers = self.sleepers.lock();
+        let mut lock = shared.synced.lock();
 
         // Check again, now that the lock is acquired
         if !self.notify_should_wakeup() {
@@ -67,7 +75,7 @@ impl Idle {
         State::unpark_one(&self.state, 1);
 
         // Get the worker to unpark
-        let ret = sleepers.pop();
+        let ret = lock.idle.sleepers.pop();
         debug_assert!(ret.is_some());
 
         ret
@@ -75,15 +83,20 @@ impl Idle {
 
     /// Returns `true` if the worker needs to do a final check for submitted
     /// work.
-    pub(super) fn transition_worker_to_parked(&self, worker: usize, is_searching: bool) -> bool {
+    pub(super) fn transition_worker_to_parked(
+        &self,
+        shared: &Shared,
+        worker: usize,
+        is_searching: bool,
+    ) -> bool {
         // Acquire the lock
-        let mut sleepers = self.sleepers.lock();
+        let mut lock = shared.synced.lock();
 
         // Decrement the number of unparked threads
         let ret = State::dec_num_unparked(&self.state, is_searching);
 
         // Track the sleeping worker
-        sleepers.push(worker);
+        lock.idle.sleepers.push(worker);
 
         ret
     }
@@ -113,8 +126,9 @@ impl Idle {
     /// within the worker's park routine.
     ///
     /// Returns `true` if the worker was parked before calling the method.
-    pub(super) fn unpark_worker_by_id(&self, worker_id: usize) -> bool {
-        let mut sleepers = self.sleepers.lock();
+    pub(super) fn unpark_worker_by_id(&self, shared: &Shared, worker_id: usize) -> bool {
+        let mut lock = shared.synced.lock();
+        let sleepers = &mut lock.idle.sleepers;
 
         for index in 0..sleepers.len() {
             if sleepers[index] == worker_id {
@@ -131,9 +145,9 @@ impl Idle {
     }
 
     /// Returns `true` if `worker_id` is contained in the sleep set.
-    pub(super) fn is_parked(&self, worker_id: usize) -> bool {
-        let sleepers = self.sleepers.lock();
-        sleepers.contains(&worker_id)
+    pub(super) fn is_parked(&self, shared: &Shared, worker_id: usize) -> bool {
+        let lock = shared.synced.lock();
+        lock.idle.sleepers.contains(&worker_id)
     }
 
     fn notify_should_wakeup(&self) -> bool {

tokio/src/runtime/scheduler/multi_thread/mod.rs

@@ -1,18 +1,39 @@
 //! Multi-threaded runtime
 
+mod counters;
+use counters::Counters;
+
 mod handle;
 pub(crate) use handle::Handle;
 
+mod overflow;
+pub(crate) use overflow::Overflow;
+
 mod idle;
 use self::idle::Idle;
 
+mod stats;
+pub(crate) use stats::Stats;
+
 mod park;
 pub(crate) use park::{Parker, Unparker};
 
 pub(crate) mod queue;
 
 mod worker;
-pub(crate) use worker::Launch;
+pub(crate) use worker::{Context, Launch, Shared};
+
+cfg_taskdump! {
+    mod trace;
+    use trace::TraceStatus;
+
+    pub(crate) use worker::Synced;
+}
+
+cfg_not_taskdump! {
+    mod trace_mock;
+    use trace_mock::TraceStatus;
+}
 
 pub(crate) use worker::block_in_place;
 
@@ -62,11 +83,9 @@ impl MultiThread {
     where
         F: Future,
     {
-        let mut enter = crate::runtime::context::enter_runtime(handle, true);
-        enter
-            .blocking
-            .block_on(future)
-            .expect("failed to park thread")
+        crate::runtime::context::enter_runtime(handle, true, |blocking| {
+            blocking.block_on(future).expect("failed to park thread")
+        })
     }
 
     pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) {