num spawns counter

Closes #5777

.github/labeler.yml

@@ -1,8 +1,28 @@
 
-R-loom:
+R-loom-sync:
 - tokio/src/sync/*
 - tokio/src/sync/**/*
-- tokio-util/src/sync/*
-- tokio-util/src/sync/**/*
-- tokio/src/runtime/*
-- tokio/src/runtime/**/*
+
+R-loom-time-driver:
+- tokio/src/runtime/time/*
+- tokio/src/runtime/time/**/*
+
+R-loom-current-thread:
+- tokio/src/runtime/scheduler/*
+- tokio/src/runtime/scheduler/current_thread/*
+- tokio/src/runtime/task/*
+- tokio/src/runtime/task/**
+
+R-loom-multi-thread:
+- tokio/src/runtime/scheduler/*
+- tokio/src/runtime/scheduler/multi_thread/*
+- tokio/src/runtime/scheduler/multi_thread/**
+- tokio/src/runtime/task/*
+- tokio/src/runtime/task/**
+
+R-loom-multi-thread-alt:
+- tokio/src/runtime/scheduler/*
+- tokio/src/runtime/scheduler/multi_thread_alt/*
+- tokio/src/runtime/scheduler/multi_thread_alt/**
+- tokio/src/runtime/task/*
+- tokio/src/runtime/task/**

.github/workflows/ci.yml

@@ -6,6 +6,10 @@ on:
 
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 env:
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: 1
@@ -21,7 +25,7 @@ env:
   # - tokio-util/Cargo.toml
   # - tokio-test/Cargo.toml
   # - tokio-stream/Cargo.toml
-  rust_min: 1.56.0
+  rust_min: 1.63.0
 
 defaults:
   run:
@@ -36,15 +40,19 @@ jobs:
     name: all systems go
     runs-on: ubuntu-latest
     needs:
-      - test
+      - test-tokio-full
+      - test-workspace-all-features
+      - test-integration-tests-per-feature
       - test-parking_lot
       - valgrind
       - test-unstable
       - miri
       - asan
       - cross-check
-      - cross-test
-      - no-atomic-u64
+      - cross-test-with-parking_lot
+      - cross-test-without-parking_lot
+      - no-atomic-u64-test
+      - no-atomic-u64-check
       - features
       - minrust
       - minimal-versions
@@ -76,7 +84,7 @@ jobs:
     steps:
       - run: exit 0
 
-  test:
+  test-tokio-full:
     needs: basics
     name: test tokio full
     runs-on: ${{ matrix.os }}
@@ -89,24 +97,77 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
             toolchain: ${{ env.rust_stable }}
-      - name: Install Rust
-        run: rustup update stable
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
       - uses: Swatinem/rust-cache@v2
-      - name: Install cargo-hack
-        uses: taiki-e/install-action@cargo-hack
 
       # Run `tokio` with `full` features. This excludes testing utilities which
       # can alter the runtime behavior of Tokio.
       - name: test tokio full
-        run: cargo test --features full
+        run: |
+          set -euxo pipefail
+          cargo nextest run --features full
+          cargo test --doc --features full
         working-directory: tokio
 
+  test-workspace-all-features:
+    needs: basics
+    name: test all crates in the workspace with all features
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+          - ubuntu-latest
+          - macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
+      - uses: Swatinem/rust-cache@v2
+
       # Test **all** crates in the workspace with all features.
       - name: test all --all-features
-        run: cargo test --workspace --all-features
+        run: |
+          set -euxo pipefail
+          cargo nextest run --workspace --all-features
+          cargo test --doc --workspace --all-features
+
+  test-integration-tests-per-feature:
+    needs: basics
+    name: Run integration tests for each feature
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+          - ubuntu-latest
+          - macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - name: Install cargo-hack
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-hack
+
+      - uses: Swatinem/rust-cache@v2
 
       # Run integration tests for each feature
       - name: test tests-integration --each-feature
@@ -118,9 +179,9 @@ jobs:
         run: cargo hack test --each-feature
         working-directory: tests-build
 
-      # Build benchmarks. Run of benchmarks is done by bench.yml workflow.
-      - name: build benches
-        run: cargo build --benches
+      # Check benchmarks. Run of benchmarks is done by bench.yml workflow.
+      - name: Check benches
+        run: cargo check --benches
         working-directory: benches
         # bench.yml workflow runs benchmarks only on linux.
         if: startsWith(matrix.os, 'ubuntu')
@@ -139,15 +200,17 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
             toolchain: ${{ env.rust_stable }}
-      - uses: Swatinem/rust-cache@v2
+
       - name: Enable parking_lot send_guard feature
         # Inserts the line "plsend = ["parking_lot/send_guard"]" right after [features]
         run: sed -i '/\[features\]/a plsend = ["parking_lot/send_guard"]' tokio/Cargo.toml
-      - name: Compile tests with all features enabled
-        run: cargo build --workspace --all-features --tests
+
+      - uses: Swatinem/rust-cache@v2
+      - name: Check tests with all features enabled
+        run: cargo check --workspace --all-features --tests
 
   valgrind:
     name: valgrind
@@ -156,14 +219,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
             toolchain: ${{ env.rust_stable }}
-      - uses: Swatinem/rust-cache@v2
 
       - name: Install Valgrind
         uses: taiki-e/install-action@valgrind
 
+      - uses: Swatinem/rust-cache@v2
       # Compile tests
       - name: cargo build test-mem
         run: cargo build --features rt-net --bin test-mem
@@ -171,7 +234,7 @@ jobs:
 
       # Run with valgrind
       - name: Run valgrind test-mem
-        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/debug/test-mem
+        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all --fair-sched=yes ./target/debug/test-mem
 
       # Compile tests
       - name: cargo build test-process-signal
@@ -180,7 +243,7 @@ jobs:
 
       # Run with valgrind
       - name: Run valgrind test-process-signal
-        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/debug/test-process-signal
+        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all --fair-sched=yes ./target/debug/test-process-signal
 
   test-unstable:
     name: test tokio full --unstable
@@ -195,13 +258,22 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
             toolchain: ${{ env.rust_stable }}
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
       - uses: Swatinem/rust-cache@v2
       # Run `tokio` with "unstable" cfg flag.
       - name: test tokio full --cfg unstable
-        run: cargo test --all-features
+        run: |
+          set -euxo pipefail
+          cargo nextest run --all-features
+          cargo test --doc --all-features
         working-directory: tokio
         env:
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings
@@ -220,13 +292,22 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
             toolchain: ${{ env.rust_stable }}
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
       - uses: Swatinem/rust-cache@v2
       # Run `tokio` with "unstable" and "taskdump" cfg flags.
       - name: test tokio full --cfg unstable --cfg taskdump
-        run: cargo test --all-features
+        run: |
+          set -euxo pipefail
+          cargo nextest run --all-features
+          cargo test --doc --all-features
         working-directory: tokio
         env:
           RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
@@ -245,13 +326,20 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
             toolchain: ${{ env.rust_stable }}
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
       - uses: Swatinem/rust-cache@v2
       # Run `tokio` with "unstable" and "taskdump" cfg flags.
       - name: check tokio full --cfg unstable --cfg internal-mt-counters
-        run: cargo test --all-features
+        run: |
+          set -euxo pipefail
+          cargo nextest run --all-features
+          cargo test --doc --all-features
         working-directory: tokio
         env:
           RUSTFLAGS: --cfg tokio_unstable --cfg tokio_internal_mt_counters -Dwarnings
@@ -266,7 +354,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
           components: miri
@@ -274,7 +362,8 @@ jobs:
       - name: miri
         # Many of tests in tokio/tests and doctests use #[tokio::test] or
         # #[tokio::main] that calls epoll_create1 that Miri does not support.
-        run: cargo miri test --features full --lib --no-fail-fast
+        run: |
+          cargo miri test --features full --lib --no-fail-fast
         working-directory: tokio
         env:
           MIRIFLAGS: -Zmiri-disable-isolation -Zmiri-strict-provenance -Zmiri-retag-fields
@@ -289,9 +378,10 @@ jobs:
         # Required to resolve symbols in sanitizer output
         run: sudo apt-get install -y llvm
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
+
       - uses: Swatinem/rust-cache@v2
       - name: asan
         run: cargo test --workspace --all-features --target x86_64-unknown-linux-gnu --tests -- --test-threads 1 --nocapture
@@ -300,19 +390,17 @@ jobs:
           # Ignore `trybuild` errors as they are irrelevant and flaky on nightly
           TRYBUILD: overwrite
 
-  # Re-enable this after the next release.
-  #
-  #semver:
-  #  name: semver
-  #  needs: basics
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #    - uses: actions/checkout@v3
-  #    - name: Check semver
-  #      uses: obi1kenobi/cargo-semver-checks-action@v2
-  #      with:
-  #        rust-toolchain: ${{ env.rust_stable }}
-  #        release-type: minor
+  semver:
+    name: semver
+    needs: basics
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check semver
+        uses: obi1kenobi/cargo-semver-checks-action@v2
+        with:
+          rust-toolchain: ${{ env.rust_stable }}
+          release-type: minor
 
   cross-check:
     name: cross-check
@@ -329,18 +417,17 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
           target: ${{ matrix.target }}
-      - name: Install cross
-        uses: taiki-e/install-action@cross
-      - run: cross check --workspace --all-features --target ${{ matrix.target }}
+
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo check --workspace --all-features --target ${{ matrix.target }}
         env:
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings
 
-  cross-test:
-    name: cross-test
+  cross-test-with-parking_lot:
     needs: basics
     runs-on: ubuntu-latest
     strategy:
@@ -348,91 +435,163 @@ jobs:
         include:
           - target: i686-unknown-linux-gnu
             rustflags: --cfg tokio_taskdump
-          - target: arm-unknown-linux-gnueabihf
+          - target: armv5te-unknown-linux-gnueabi
           - target: armv7-unknown-linux-gnueabihf
           - target: aarch64-unknown-linux-gnu
             rustflags: --cfg tokio_taskdump
-
-          # Run a platform without AtomicU64 and no const Mutex::new
-          - target: arm-unknown-linux-gnueabihf
-            rustflags: --cfg tokio_no_const_mutex_new
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust stable
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
           target: ${{ matrix.target }}
-      - name: Install cross
-        uses: taiki-e/install-action@cross
-      # First run with all features (including parking_lot)
-      - run: cross test -p tokio --all-features --target ${{ matrix.target }} --tests
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
+      - uses: taiki-e/setup-cross-toolchain-action@v1
+        with:
+          target: ${{ matrix.target }}
+
+      - uses: Swatinem/rust-cache@v2
+      - name: Tests run with all features (including parking_lot)
+        run: |
+          set -euxo pipefail
+          cargo nextest run -p tokio --all-features --target ${{ matrix.target }}
+          cargo test --doc -p tokio --all-features --target ${{ matrix.target }}
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_tuning_tests ${{ matrix.rustflags }}
-      # Now run without parking_lot
+          RUST_TEST_THREADS: 1
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_tuning_tests ${{ matrix.rustflags }}
+
+  cross-test-without-parking_lot:
+    needs: basics
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: i686-unknown-linux-gnu
+            rustflags: --cfg tokio_taskdump
+          - target: armv5te-unknown-linux-gnueabi
+          - target: armv7-unknown-linux-gnueabihf
+          - target: aarch64-unknown-linux-gnu
+            rustflags: --cfg tokio_taskdump
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust stable
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ env.rust_stable }}
+          target: ${{ matrix.target }}
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
+      - uses: taiki-e/setup-cross-toolchain-action@v1
+        with:
+          target: ${{ matrix.target }}
+
       - name: Remove `parking_lot` from `full` feature
         run: sed -i '0,/parking_lot/{/parking_lot/d;}' tokio/Cargo.toml
+
+      - uses: Swatinem/rust-cache@v2
       # The `tokio_no_parking_lot` cfg is here to ensure the `sed` above does not silently break.
-      - run: cross test -p tokio --features full,test-util --target ${{ matrix.target }} --tests
+      - name: Tests run with all features (without parking_lot)
+        run: |
+          set -euxo pipefail
+          cargo nextest run -p tokio --features full,test-util --target ${{ matrix.target }}
+          cargo test --doc -p tokio --features full,test-util --target ${{ matrix.target }}
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_parking_lot --cfg tokio_no_tuning_tests ${{ matrix.rustflags }}
+          RUST_TEST_THREADS: 1
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_parking_lot --cfg tokio_no_tuning_tests ${{ matrix.rustflags }}
 
   # See https://github.com/tokio-rs/tokio/issues/5187
-  no-atomic-u64:
-    name: Test i686-unknown-linux-gnu without AtomicU64
+  no-atomic-u64-test:
+    name: Test tokio --all-features on i686-unknown-linux-gnu without AtomicU64
     needs: basics
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ env.rust_nightly }}
+          components: rust-src
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
+      - uses: taiki-e/setup-cross-toolchain-action@v1
+        with:
+          target: i686-unknown-linux-gnu
+
+      - uses: Swatinem/rust-cache@v2
+      - name: test tokio --all-features
+        run: |
+          cargo nextest run -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features
+          cargo test --doc -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features
+        env:
+          RUST_TEST_THREADS: 1
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_tuning_tests
+
+  no-atomic-u64-check:
+    name: Check tokio --feature-powerset --depth 2 on i686-unknown-linux-gnu without AtomicU64
+    needs: basics
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
           components: rust-src
       - name: Install cargo-hack
-        uses: taiki-e/install-action@cargo-hack
-      # Install linker and libraries for i686-unknown-linux-gnu
-      - uses: taiki-e/setup-cross-toolchain-action@v1
+        uses: taiki-e/install-action@v2
         with:
-          target: i686-unknown-linux-gnu
-      - run: cargo test -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features -- --test-threads 1 --nocapture
-        env:
-          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64
+          tool: cargo-hack
+
+      - uses: Swatinem/rust-cache@v2
+
       # https://github.com/tokio-rs/tokio/pull/5356
       # https://github.com/tokio-rs/tokio/issues/5373
-      - run: cargo hack build -p tokio --feature-powerset --depth 2 --keep-going
+      - name: Check
+        run: cargo hack check -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --feature-powerset --depth 2 --keep-going
         env:
-          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64 --cfg tokio_no_const_mutex_new
-      - run: cargo hack build -p tokio --feature-powerset --depth 2 --keep-going
-        env:
-          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64
+          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
 
   features:
-    name: features
+    name: features ${{ matrix.name }}
     needs: basics
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - { name: "", rustflags: "" }
+          # Try with unstable feature flags
+          - { name: "--unstable", rustflags: "--cfg tokio_unstable -Dwarnings" }
+          # Try with unstable and taskdump feature flags
+          - { name: "--unstable --taskdump", rustflags: "--cfg tokio_unstable -Dwarnings --cfg tokio_taskdump" }
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
           target: ${{ matrix.target }}
-      - uses: Swatinem/rust-cache@v2
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
-      - name: check --feature-powerset
-        run: cargo hack check --all --feature-powerset --depth 2 --keep-going
-      # Try with unstable feature flags
-      - name: check --feature-powerset --unstable
+
+      - uses: Swatinem/rust-cache@v2
+      - name: check --feature-powerset ${{ matrix.name }}
         run: cargo hack check --all --feature-powerset --depth 2 --keep-going
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
-      # Try with unstable and taskdump feature flags
-      - name: check --feature-powerset --unstable --taskdump
-        run: cargo hack check --all --feature-powerset --depth 2 --keep-going
-        env:
-          RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
+          RUSTFLAGS: ${{ matrix.rustflags }}
 
   minrust:
     name: minrust
@@ -440,7 +599,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_min }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_min }}
       - uses: Swatinem/rust-cache@v2
@@ -456,12 +615,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
-      - uses: Swatinem/rust-cache@v2
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
+
+      - uses: Swatinem/rust-cache@v2
       - name: "check --all-features -Z minimal-versions"
         run: |
           # Remove dev-dependencies from Cargo.toml to prevent the next `cargo update`
@@ -487,7 +647,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
           components: rustfmt
@@ -507,14 +667,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_clippy }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_clippy }}
           components: clippy
       - uses: Swatinem/rust-cache@v2
       # Run clippy
       - name: "clippy --all"
-        run: cargo clippy --all --tests --all-features
+        run: cargo clippy --all --tests --all-features --no-deps
 
   docs:
     name: docs
@@ -522,12 +682,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
       - uses: Swatinem/rust-cache@v2
       - name: "doc --lib --all-features"
-        run: cargo doc --lib --no-deps --all-features --document-private-items
+        run: |
+          cargo doc --lib --no-deps --all-features --document-private-items
         env:
           RUSTFLAGS: --cfg docsrs --cfg tokio_unstable --cfg tokio_taskdump
           RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings
@@ -539,7 +700,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
       - uses: Swatinem/rust-cache@v2
@@ -574,18 +735,23 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
-      - uses: Swatinem/rust-cache@v2
-      - name: Test hyper
+
+      - name: Clone hyper
+        run: git clone https://github.com/hyperium/hyper.git
+
+      - name: Checkout the latest release because HEAD maybe contains breakage.
         run: |
           set -x
-          git clone https://github.com/hyperium/hyper.git
-          cd hyper
-          # checkout the latest release because HEAD maybe contains breakage.
           tag=$(git describe --abbrev=0 --tags)
           git checkout "${tag}"
+        working-directory: hyper
+
+      - name: Patch hyper to use tokio from this repository
+        run: |
+          set -x
           echo '[workspace]' >>Cargo.toml
           echo '[patch.crates-io]' >>Cargo.toml
           echo 'tokio = { path = "../tokio" }' >>Cargo.toml
@@ -593,7 +759,20 @@ jobs:
           echo 'tokio-stream = { path = "../tokio-stream" }' >>Cargo.toml
           echo 'tokio-test = { path = "../tokio-test" }' >>Cargo.toml
           git diff
-          cargo test --features full
+        working-directory: hyper
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # The cargo workspaces and target directory configuration.
+          # These entries are separated by newlines and have the form
+          # `$workspace -> $target`. The `$target` part is treated as a directory
+          # relative to the `$workspace` and defaults to "target" if not explicitly given.
+          # default: ". -> target"
+          workspaces: "./hyper"
+
+      - name: Test hyper
+        run: cargo test --features full
+        working-directory: hyper
 
   x86_64-fortanix-unknown-sgx:
     name: build tokio for x86_64-fortanix-unknown-sgx
@@ -602,7 +781,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
           target: x86_64-fortanix-unknown-sgx
@@ -634,12 +813,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
-      - uses: Swatinem/rust-cache@v2
       - name: Install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+        uses: taiki-e/install-action@wasm-pack
+
+      - uses: Swatinem/rust-cache@v2
       - name: test tokio
         run: wasm-pack test --node -- --features "macros sync"
         working-directory: tokio
@@ -651,24 +831,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_stable }}
-      - uses: Swatinem/rust-cache@v2
+          targets: wasm32-wasi
 
       # Install dependencies
-      - name: Install cargo-hack
-        uses: taiki-e/install-action@cargo-hack
-
-      - name: Install wasm32-wasi target
-        run: rustup target add wasm32-wasi
-
-      - name: Install wasmtime
-        uses: taiki-e/install-action@wasmtime
-
-      - name: Install cargo-wasi
-        run: cargo install cargo-wasi
+      - name: Install cargo-hack, wasmtime, and cargo-wasi
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-hack,wasmtime,cargo-wasi
 
+      - uses: Swatinem/rust-cache@v2
       - name: WASI test tokio full
         run: cargo test -p tokio --target wasm32-wasi --features full
         env:
@@ -708,7 +882,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ matrix.rust }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ matrix.rust }}
       - uses: Swatinem/rust-cache@v2
@@ -727,7 +901,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ env.rust_nightly }}
       - uses: Swatinem/rust-cache@v2

.github/workflows/labeler.yml

@@ -4,6 +4,10 @@ on:
 
 # See .github/labeler.yml file
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 

.github/workflows/loom.yml

@@ -7,8 +7,14 @@ on:
 
 name: Loom
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 env:
-  RUSTFLAGS: -Dwarnings
+  RUSTFLAGS: -Dwarnings --cfg loom --cfg tokio_unstable -C debug_assertions
+  LOOM_MAX_PREEMPTIONS: 2
+  LOOM_MAX_BRANCHES: 10000
   RUST_BACKTRACE: 1
   # Change to specific Rust release to pin
   rust_stable: stable
@@ -17,26 +23,66 @@ permissions:
   contents: read
 
 jobs:
-  loom:
-    name: loom
+  loom-sync:
+    name: loom tokio::sync
     # base_ref is null when it's not a pull request
-    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom') || (github.base_ref == null))
+    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom-sync') || (github.base_ref == null))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      - name: run tests
+        run: cargo test --lib --release --features full -- --nocapture sync::tests
+        working-directory: tokio
+
+  loom-time-driver:
+    name: loom time driver
+    # base_ref is null when it's not a pull request
+    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom-time-driver') || (github.base_ref == null))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      - name: run tests
+        run: cargo test --lib --release --features full -- --nocapture runtime::time::tests
+        working-directory: tokio
+
+  loom-current-thread:
+    name: loom current-thread scheduler
+    # base_ref is null when it's not a pull request
+    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom-current-thread') || (github.base_ref == null))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      - name: run tests
+        run: cargo test --lib --release --features full -- --nocapture loom_current_thread
+        working-directory: tokio
+
+  loom-multi-thread:
+    name: loom multi-thread scheduler
+    # base_ref is null when it's not a pull request
+    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom-multi-thread') || (github.base_ref == null))
     runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
-          - scope: --skip loom_pool
-            max_preemptions: 2
-          - scope: loom_pool::group_a
-            max_preemptions: 2
-          - scope: loom_pool::group_b
-            max_preemptions: 2
-          - scope: loom_pool::group_c
-            max_preemptions: 2
-          - scope: loom_pool::group_d
-            max_preemptions: 2
-          - scope: time::driver
-            max_preemptions: 2
+          - scope: loom_multi_thread::group_a
+          - scope: loom_multi_thread::group_b
+          - scope: loom_multi_thread::group_c
+          - scope: loom_multi_thread::group_d
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
@@ -45,10 +91,32 @@ jobs:
             toolchain: ${{ env.rust_stable }}
       - uses: Swatinem/rust-cache@v2
       - name: loom ${{ matrix.scope }}
-        run: cargo test --lib --release --features full -- --nocapture $SCOPE
+        run: cargo test --lib --release --features full -- $SCOPE
+        working-directory: tokio
+        env:
+          SCOPE: ${{ matrix.scope }}
+
+  loom-multi-thread-alt:
+    name: loom ALT multi-thread scheduler
+    # base_ref is null when it's not a pull request
+    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom-multi-thread-alt') || (github.base_ref == null))
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - scope: loom_multi_thread_alt::group_a
+          - scope: loom_multi_thread_alt::group_b
+          - scope: loom_multi_thread_alt::group_c
+          - scope: loom_multi_thread_alt::group_d
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+            toolchain: ${{ env.rust_stable }}
+      - uses: Swatinem/rust-cache@v2
+      - name: loom ${{ matrix.scope }}
+        run: cargo test --lib --release --features full -- $SCOPE
         working-directory: tokio
         env:
-          RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings -C debug-assertions
-          LOOM_MAX_PREEMPTIONS: ${{ matrix.max_preemptions }}
-          LOOM_MAX_BRANCHES: 10000
           SCOPE: ${{ matrix.scope }}

.github/workflows/pr-audit.yml

@@ -8,6 +8,10 @@ on:
     paths:
       - '**/Cargo.toml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 

.github/workflows/stress-test.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - master
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 env:
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: 1
@@ -28,14 +32,14 @@ jobs:
         uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-      - uses: Swatinem/rust-cache@v2
       - name: Install Valgrind
         uses: taiki-e/install-action@valgrind
 
+      - uses: Swatinem/rust-cache@v2
       # Compiles each of the stress test examples.
       - name: Compile stress test examples
         run: cargo build -p stress-test --release --example ${{ matrix.stress-test }}
 
       # Runs each of the examples using Valgrind. Detects leaks and displays them.
       - name: Run valgrind
-        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/release/examples/${{ matrix.stress-test }}
+        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all --fair-sched=yes ./target/release/examples/${{ matrix.stress-test }}

README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.28.2", features = ["full"] }
+tokio = { version = "1.32.0", features = ["full"] }
 ```
 Then, on your main.rs:
 
@@ -187,12 +187,13 @@ When updating this, also update:
 
 Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
 least** 6 months. When increasing the MSRV, the new Rust version must have been
-released at least six months ago. The current MSRV is 1.56.0.
+released at least six months ago. The current MSRV is 1.63.
 
 Note that the MSRV is not increased automatically, and only as part of a minor
 release. The MSRV history for past minor releases can be found below:
 
- * 1.27 to now - Rust 1.56
+ * 1.30 to now - Rust 1.63
+ * 1.27 to 1.29 - Rust 1.56
  * 1.17 to 1.26 - Rust 1.49
  * 1.15 to 1.16 - Rust 1.46
  * 1.0 to 1.14 - Rust 1.45
@@ -215,7 +216,6 @@ warrants a patch release with a fix for the bug, it will be backported and
 released as a new patch release for each LTS minor version. Our current LTS
 releases are:
 
- * `1.18.x` - LTS release until June 2023. (MSRV 1.49)
  * `1.20.x` - LTS release until September 2023. (MSRV 1.49)
  * `1.25.x` - LTS release until March 2024. (MSRV 1.49)
 
@@ -230,6 +230,12 @@ can use the following dependency specification:
 tokio = { version = "~1.18", features = [...] }
 ```
 
+### Previous LTS releases
+
+ * `1.8.x` - LTS release until February 2022.
+ * `1.14.x` - LTS release until June 2022.
+ * `1.18.x` - LTS release until June 2023.
+
 ## License
 
 This project is licensed under the [MIT license].

examples/proxy.rs

@@ -22,8 +22,7 @@
 
 #![warn(rust_2018_idioms)]
 
-use tokio::io;
-use tokio::io::AsyncWriteExt;
+use tokio::io::copy_bidirectional;
 use tokio::net::{TcpListener, TcpStream};
 
 use futures::FutureExt;
@@ -44,36 +43,19 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
     let listener = TcpListener::bind(listen_addr).await?;
 
-    while let Ok((inbound, _)) = listener.accept().await {
-        let transfer = transfer(inbound, server_addr.clone()).map(|r| {
-            if let Err(e) = r {
-                println!("Failed to transfer; error={}", e);
-            }
-        });
+    while let Ok((mut inbound, _)) = listener.accept().await {
+        let mut outbound = TcpStream::connect(server_addr.clone()).await?;
 
-        tokio::spawn(transfer);
+        tokio::spawn(async move {
+            copy_bidirectional(&mut inbound, &mut outbound)
+                .map(|r| {
+                    if let Err(e) = r {
+                        println!("Failed to transfer; error={}", e);
+                    }
+                })
+                .await
+        });
     }
 
     Ok(())
 }
-
-async fn transfer(mut inbound: TcpStream, proxy_addr: String) -> Result<(), Box<dyn Error>> {
-    let mut outbound = TcpStream::connect(proxy_addr).await?;
-
-    let (mut ri, mut wi) = inbound.split();
-    let (mut ro, mut wo) = outbound.split();
-
-    let client_to_server = async {
-        io::copy(&mut ri, &mut wo).await?;
-        wo.shutdown().await
-    };
-
-    let server_to_client = async {
-        io::copy(&mut ro, &mut wi).await?;
-        wi.shutdown().await
-    };
-
-    tokio::try_join!(client_to_server, server_to_client)?;
-
-    Ok(())
-}

tokio-stream/Cargo.toml

@@ -37,7 +37,7 @@ signal = ["tokio/signal"]
 
 [dependencies]
 futures-core = { version = "0.3.0" }
-pin-project-lite = "0.2.7"
+pin-project-lite = "0.2.11"
 tokio = { version = "1.15.0", path = "../tokio", features = ["sync"] }
 tokio-util = { version = "0.7.0", path = "../tokio-util", optional = true }
 

tokio-stream/src/stream_ext.rs

@@ -994,7 +994,7 @@ pub trait StreamExt: Stream {
     /// assert!(timeout_stream.try_next().await.is_ok(), "expected no more timeouts");
     /// # }
     /// ```
-    #[cfg(all(feature = "time"))]
+    #[cfg(feature = "time")]
     #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
     fn timeout(self, duration: Duration) -> Timeout<Self>
     where
@@ -1083,7 +1083,7 @@ pub trait StreamExt: Stream {
     /// assert!(timeout_stream.try_next().await.is_ok(), "expected non-timeout");
     /// # }
     /// ```
-    #[cfg(all(feature = "time"))]
+    #[cfg(feature = "time")]
     #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
     fn timeout_repeating(self, interval: Interval) -> TimeoutRepeating<Self>
     where
@@ -1113,7 +1113,7 @@ pub trait StreamExt: Stream {
     /// }
     /// # }
     /// ```
-    #[cfg(all(feature = "time"))]
+    #[cfg(feature = "time")]
     #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
     fn throttle(self, duration: Duration) -> Throttle<Self>
     where

tokio-stream/src/stream_ext/take.rs

@@ -64,11 +64,11 @@ where
 
         let (lower, upper) = self.stream.size_hint();
 
-        let lower = cmp::min(lower, self.remaining as usize);
+        let lower = cmp::min(lower, self.remaining);
 
         let upper = match upper {
-            Some(x) if x < self.remaining as usize => Some(x),
-            _ => Some(self.remaining as usize),
+            Some(x) if x < self.remaining => Some(x),
+            _ => Some(self.remaining),
         };
 
         (lower, upper)

tokio-test/src/io.rs

@@ -409,6 +409,20 @@ impl AsyncWrite for Mock {
             // If a sleep is set, it has already fired
             self.inner.sleep = None;
 
+            if self.inner.actions.is_empty() {
+                match self.inner.poll_action(cx) {
+                    Poll::Pending => {
+                        // do not propagate pending
+                    }
+                    Poll::Ready(Some(action)) => {
+                        self.inner.actions.push_back(action);
+                    }
+                    Poll::Ready(None) => {
+                        panic!("unexpected write");
+                    }
+                }
+            }
+
             match self.inner.write(buf) {
                 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
                     if let Some(rem) = self.inner.remaining_wait() {

tokio-test/src/lib.rs

@@ -12,6 +12,7 @@
 //! Tokio and Futures based testing utilities
 
 pub mod io;
+pub mod stream_mock;
 
 mod macros;
 pub mod task;

tokio-test/src/stream_mock.rs

@@ -0,0 +1,168 @@
+#![cfg(not(loom))]
+
+//! A mock stream implementing [`Stream`].
+//!
+//! # Overview
+//! This crate provides a `StreamMock` that can be used to test code that interacts with streams.
+//! It allows you to mock the behavior of a stream and control the items it yields and the waiting
+//! intervals between items.
+//!
+//! # Usage
+//! To use the `StreamMock`, you need to create a builder using[`StreamMockBuilder`]. The builder
+//! allows you to enqueue actions such as returning items or waiting for a certain duration.
+//!
+//! # Example
+//! ```rust
+//!
+//! use futures_util::StreamExt;
+//! use std::time::Duration;
+//! use tokio_test::stream_mock::StreamMockBuilder;
+//!
+//! async fn test_stream_mock_wait() {
+//!     let mut stream_mock = StreamMockBuilder::new()
+//!         .next(1)
+//!         .wait(Duration::from_millis(300))
+//!         .next(2)
+//!         .build();
+//!
+//!     assert_eq!(stream_mock.next().await, Some(1));
+//!     let start = std::time::Instant::now();
+//!     assert_eq!(stream_mock.next().await, Some(2));
+//!     let elapsed = start.elapsed();
+//!     assert!(elapsed >= Duration::from_millis(300));
+//!     assert_eq!(stream_mock.next().await, None);
+//! }
+//! ```
+
+use std::collections::VecDeque;
+use std::pin::Pin;
+use std::task::Poll;
+use std::time::Duration;
+
+use futures_core::{ready, Stream};
+use std::future::Future;
+use tokio::time::{sleep_until, Instant, Sleep};
+
+#[derive(Debug, Clone)]
+enum Action<T: Unpin> {
+    Next(T),
+    Wait(Duration),
+}
+
+/// A builder for [`StreamMock`]
+#[derive(Debug, Clone)]
+pub struct StreamMockBuilder<T: Unpin> {
+    actions: VecDeque<Action<T>>,
+}
+
+impl<T: Unpin> StreamMockBuilder<T> {
+    /// Create a new empty [`StreamMockBuilder`]
+    pub fn new() -> Self {
+        StreamMockBuilder::default()
+    }
+
+    /// Queue an item to be returned by the stream
+    pub fn next(mut self, value: T) -> Self {
+        self.actions.push_back(Action::Next(value));
+        self
+    }
+
+    // Queue an item to be consumed by the sink,
+    // commented out until Sink is implemented.
+    //
+    // pub fn consume(mut self, value: T) -> Self {
+    //    self.actions.push_back(Action::Consume(value));
+    //    self
+    // }
+
+    /// Queue the stream to wait for a duration
+    pub fn wait(mut self, duration: Duration) -> Self {
+        self.actions.push_back(Action::Wait(duration));
+        self
+    }
+
+    /// Build the [`StreamMock`]
+    pub fn build(self) -> StreamMock<T> {
+        StreamMock {
+            actions: self.actions,
+            sleep: None,
+        }
+    }
+}
+
+impl<T: Unpin> Default for StreamMockBuilder<T> {
+    fn default() -> Self {
+        StreamMockBuilder {
+            actions: VecDeque::new(),
+        }
+    }
+}
+
+/// A mock stream implementing [`Stream`]
+///
+/// See [`StreamMockBuilder`] for more information.
+#[derive(Debug)]
+pub struct StreamMock<T: Unpin> {
+    actions: VecDeque<Action<T>>,
+    sleep: Option<Pin<Box<Sleep>>>,
+}
+
+impl<T: Unpin> StreamMock<T> {
+    fn next_action(&mut self) -> Option<Action<T>> {
+        self.actions.pop_front()
+    }
+}
+
+impl<T: Unpin> Stream for StreamMock<T> {
+    type Item = T;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        // Try polling the sleep future first
+        if let Some(ref mut sleep) = self.sleep {
+            ready!(Pin::new(sleep).poll(cx));
+            // Since we're ready, discard the sleep future
+            self.sleep.take();
+        }
+
+        match self.next_action() {
+            Some(action) => match action {
+                Action::Next(item) => Poll::Ready(Some(item)),
+                Action::Wait(duration) => {
+                    // Set up a sleep future and schedule this future to be polled again for it.
+                    self.sleep = Some(Box::pin(sleep_until(Instant::now() + duration)));
+                    cx.waker().wake_by_ref();
+
+                    Poll::Pending
+                }
+            },
+            None => Poll::Ready(None),
+        }
+    }
+}
+
+impl<T: Unpin> Drop for StreamMock<T> {
+    fn drop(&mut self) {
+        // Avoid double panicking to make debugging easier.
+        if std::thread::panicking() {
+            return;
+        }
+
+        let undropped_count = self
+            .actions
+            .iter()
+            .filter(|action| match action {
+                Action::Next(_) => true,
+                Action::Wait(_) => false,
+            })
+            .count();
+
+        assert!(
+            undropped_count == 0,
+            "StreamMock was dropped before all actions were consumed, {} actions were not consumed",
+            undropped_count
+        );
+    }
+}

tokio-test/tests/io.rs

@@ -51,6 +51,29 @@ async fn write() {
     mock.write_all(b"world!").await.expect("write 2");
 }
 
+#[tokio::test]
+async fn write_with_handle() {
+    let (mut mock, mut handle) = Builder::new().build_with_handle();
+    handle.write(b"hello ");
+    handle.write(b"world!");
+
+    mock.write_all(b"hello ").await.expect("write 1");
+    mock.write_all(b"world!").await.expect("write 2");
+}
+
+#[tokio::test]
+async fn read_with_handle() {
+    let (mut mock, mut handle) = Builder::new().build_with_handle();
+    handle.read(b"hello ");
+    handle.read(b"world!");
+
+    let mut buf = vec![0; 6];
+    mock.read_exact(&mut buf).await.expect("read 1");
+    assert_eq!(&buf[..], b"hello ");
+    mock.read_exact(&mut buf).await.expect("read 2");
+    assert_eq!(&buf[..], b"world!");
+}
+
 #[tokio::test]
 async fn write_error() {
     let error = io::Error::new(io::ErrorKind::Other, "cruel");

tokio-test/tests/stream_mock.rs

@@ -0,0 +1,50 @@
+use futures_util::StreamExt;
+use std::time::Duration;
+use tokio_test::stream_mock::StreamMockBuilder;
+
+#[tokio::test]
+async fn test_stream_mock_empty() {
+    let mut stream_mock = StreamMockBuilder::<u32>::new().build();
+
+    assert_eq!(stream_mock.next().await, None);
+    assert_eq!(stream_mock.next().await, None);
+}
+
+#[tokio::test]
+async fn test_stream_mock_items() {
+    let mut stream_mock = StreamMockBuilder::new().next(1).next(2).build();
+
+    assert_eq!(stream_mock.next().await, Some(1));
+    assert_eq!(stream_mock.next().await, Some(2));
+    assert_eq!(stream_mock.next().await, None);
+}
+
+#[tokio::test]
+async fn test_stream_mock_wait() {
+    let mut stream_mock = StreamMockBuilder::new()
+        .next(1)
+        .wait(Duration::from_millis(300))
+        .next(2)
+        .build();
+
+    assert_eq!(stream_mock.next().await, Some(1));
+    let start = std::time::Instant::now();
+    assert_eq!(stream_mock.next().await, Some(2));
+    let elapsed = start.elapsed();
+    assert!(elapsed >= Duration::from_millis(300));
+    assert_eq!(stream_mock.next().await, None);
+}
+
+#[tokio::test]
+#[should_panic(expected = "StreamMock was dropped before all actions were consumed")]
+async fn test_stream_mock_drop_without_consuming_all() {
+    let stream_mock = StreamMockBuilder::new().next(1).next(2).build();
+    drop(stream_mock);
+}
+
+#[tokio::test]
+#[should_panic(expected = "test panic was not masked")]
+async fn test_stream_mock_drop_during_panic_doesnt_mask_panic() {
+    let _stream_mock = StreamMockBuilder::new().next(1).next(2).build();
+    panic!("test panic was not masked");
+}

tokio-util/Cargo.toml

@@ -34,13 +34,13 @@ rt = ["tokio/rt", "tokio/sync", "futures-util", "hashbrown"]
 __docs_rs = ["futures-util"]
 
 [dependencies]
-tokio = { version = "1.22.0", path = "../tokio", features = ["sync"] }
+tokio = { version = "1.28.0", path = "../tokio", features = ["sync"] }
 bytes = "1.0.0"
 futures-core = "0.3.0"
 futures-sink = "0.3.0"
 futures-io = { version = "0.3.0", optional = true }
 futures-util = { version = "0.3.0", optional = true }
-pin-project-lite = "0.2.7"
+pin-project-lite = "0.2.11"
 slab = { version = "0.4.4", optional = true } # Backs `DelayQueue`
 tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true }
 
@@ -56,6 +56,7 @@ async-stream = "0.3.0"
 futures = "0.3.0"
 futures-test = "0.3.5"
 parking_lot = "0.12.0"
+tempfile = "3.1.0"
 
 [package.metadata.docs.rs]
 all-features = true

tokio-util/src/compat.rs

@@ -227,12 +227,14 @@ impl<T: tokio::io::AsyncSeek> futures_io::AsyncSeek for Compat<T> {
         pos: io::SeekFrom,
     ) -> Poll<io::Result<u64>> {
         if self.seek_pos != Some(pos) {
+            // Ensure previous seeks have finished before starting a new one
+            ready!(self.as_mut().project().inner.poll_complete(cx))?;
             self.as_mut().project().inner.start_seek(pos)?;
             *self.as_mut().project().seek_pos = Some(pos);
         }
         let res = ready!(self.as_mut().project().inner.poll_complete(cx));
         *self.as_mut().project().seek_pos = None;
-        Poll::Ready(res.map(|p| p as u64))
+        Poll::Ready(res)
     }
 }
 
@@ -255,7 +257,7 @@ impl<T: futures_io::AsyncSeek> tokio::io::AsyncSeek for Compat<T> {
         };
         let res = ready!(self.as_mut().project().inner.poll_seek(cx, pos));
         *self.as_mut().project().seek_pos = None;
-        Poll::Ready(res.map(|p| p as u64))
+        Poll::Ready(res)
     }
 }
 

tokio-util/src/io/sink_writer.rs

@@ -59,7 +59,7 @@ pin_project! {
     /// [`CopyToBytes`]: crate::io::CopyToBytes
     /// [`Encoder`]: crate::codec::Encoder
     /// [`Sink`]: futures_sink::Sink
-    /// [`codec`]: tokio_util::codec
+    /// [`codec`]: crate::codec
     #[derive(Debug)]
     pub struct SinkWriter<S> {
         #[pin]

tokio-util/src/lib.rs

@@ -57,151 +57,4 @@ pub mod either;
 
 pub use bytes;
 
-#[cfg(any(feature = "io", feature = "codec"))]
-mod util {
-    use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
-
-    use bytes::{Buf, BufMut};
-    use futures_core::ready;
-    use std::io::{self, IoSlice};
-    use std::mem::MaybeUninit;
-    use std::pin::Pin;
-    use std::task::{Context, Poll};
-
-    /// Try to read data from an `AsyncRead` into an implementer of the [`BufMut`] trait.
-    ///
-    /// [`BufMut`]: bytes::Buf
-    ///
-    /// # Example
-    ///
-    /// ```
-    /// use bytes::{Bytes, BytesMut};
-    /// use tokio_stream as stream;
-    /// use tokio::io::Result;
-    /// use tokio_util::io::{StreamReader, poll_read_buf};
-    /// use futures::future::poll_fn;
-    /// use std::pin::Pin;
-    /// # #[tokio::main]
-    /// # async fn main() -> std::io::Result<()> {
-    ///
-    /// // Create a reader from an iterator. This particular reader will always be
-    /// // ready.
-    /// let mut read = StreamReader::new(stream::iter(vec![Result::Ok(Bytes::from_static(&[0, 1, 2, 3]))]));
-    ///
-    /// let mut buf = BytesMut::new();
-    /// let mut reads = 0;
-    ///
-    /// loop {
-    ///     reads += 1;
-    ///     let n = poll_fn(|cx| poll_read_buf(Pin::new(&mut read), cx, &mut buf)).await?;
-    ///
-    ///     if n == 0 {
-    ///         break;
-    ///     }
-    /// }
-    ///
-    /// // one or more reads might be necessary.
-    /// assert!(reads >= 1);
-    /// assert_eq!(&buf[..], &[0, 1, 2, 3]);
-    /// # Ok(())
-    /// # }
-    /// ```
-    #[cfg_attr(not(feature = "io"), allow(unreachable_pub))]
-    pub fn poll_read_buf<T: AsyncRead, B: BufMut>(
-        io: Pin<&mut T>,
-        cx: &mut Context<'_>,
-        buf: &mut B,
-    ) -> Poll<io::Result<usize>> {
-        if !buf.has_remaining_mut() {
-            return Poll::Ready(Ok(0));
-        }
-
-        let n = {
-            let dst = buf.chunk_mut();
-
-            // Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
-            // transparent wrapper around `[MaybeUninit<u8>]`.
-            let dst = unsafe { &mut *(dst as *mut _ as *mut [MaybeUninit<u8>]) };
-            let mut buf = ReadBuf::uninit(dst);
-            let ptr = buf.filled().as_ptr();
-            ready!(io.poll_read(cx, &mut buf)?);
-
-            // Ensure the pointer does not change from under us
-            assert_eq!(ptr, buf.filled().as_ptr());
-            buf.filled().len()
-        };
-
-        // Safety: This is guaranteed to be the number of initialized (and read)
-        // bytes due to the invariants provided by `ReadBuf::filled`.
-        unsafe {
-            buf.advance_mut(n);
-        }
-
-        Poll::Ready(Ok(n))
-    }
-
-    /// Try to write data from an implementer of the [`Buf`] trait to an
-    /// [`AsyncWrite`], advancing the buffer's internal cursor.
-    ///
-    /// This function will use [vectored writes] when the [`AsyncWrite`] supports
-    /// vectored writes.
-    ///
-    /// # Examples
-    ///
-    /// [`File`] implements [`AsyncWrite`] and [`Cursor<&[u8]>`] implements
-    /// [`Buf`]:
-    ///
-    /// ```no_run
-    /// use tokio_util::io::poll_write_buf;
-    /// use tokio::io;
-    /// use tokio::fs::File;
-    ///
-    /// use bytes::Buf;
-    /// use std::io::Cursor;
-    /// use std::pin::Pin;
-    /// use futures::future::poll_fn;
-    ///
-    /// #[tokio::main]
-    /// async fn main() -> io::Result<()> {
-    ///     let mut file = File::create("foo.txt").await?;
-    ///     let mut buf = Cursor::new(b"data to write");
-    ///
-    ///     // Loop until the entire contents of the buffer are written to
-    ///     // the file.
-    ///     while buf.has_remaining() {
-    ///         poll_fn(|cx| poll_write_buf(Pin::new(&mut file), cx, &mut buf)).await?;
-    ///     }
-    ///
-    ///     Ok(())
-    /// }
-    /// ```
-    ///
-    /// [`Buf`]: bytes::Buf
-    /// [`AsyncWrite`]: tokio::io::AsyncWrite
-    /// [`File`]: tokio::fs::File
-    /// [vectored writes]: tokio::io::AsyncWrite::poll_write_vectored
-    #[cfg_attr(not(feature = "io"), allow(unreachable_pub))]
-    pub fn poll_write_buf<T: AsyncWrite, B: Buf>(
-        io: Pin<&mut T>,
-        cx: &mut Context<'_>,
-        buf: &mut B,
-    ) -> Poll<io::Result<usize>> {
-        const MAX_BUFS: usize = 64;
-
-        if !buf.has_remaining() {
-            return Poll::Ready(Ok(0));
-        }
-
-        let n = if io.is_write_vectored() {
-            let mut slices = [IoSlice::new(&[]); MAX_BUFS];
-            let cnt = buf.chunks_vectored(&mut slices);
-            ready!(io.poll_write_vectored(cx, &slices[..cnt]))?
-        } else {
-            ready!(io.poll_write(cx, buf.chunk()))?
-        };
-
-        buf.advance(n);
-
-        Poll::Ready(Ok(n))
-    }
-}
+mod util;

tokio-util/src/sync/cancellation_token.rs

@@ -4,6 +4,7 @@ pub(crate) mod guard;
 mod tree_node;
 
 use crate::loom::sync::Arc;
+use crate::util::MaybeDangling;
 use core::future::Future;
 use core::pin::Pin;
 use core::task::{Context, Poll};
@@ -77,11 +78,23 @@ pin_project! {
     /// [`CancellationToken`] by value instead of using a reference.
     #[must_use = "futures do nothing unless polled"]
     pub struct WaitForCancellationFutureOwned {
-        // Since `future` is the first field, it is dropped before the
-        // cancellation_token field. This ensures that the reference inside the
-        // `Notified` remains valid.
+        // This field internally has a reference to the cancellation token, but camouflages
+        // the relationship with `'static`. To avoid Undefined Behavior, we must ensure
+        // that the reference is only used while the cancellation token is still alive. To
+        // do that, we ensure that the future is the first field, so that it is dropped
+        // before the cancellation token.
+        //
+        // We use `MaybeDanglingFuture` here because without it, the compiler could assert
+        // the reference inside `future` to be valid even after the destructor of that
+        // field runs. (Specifically, when the `WaitForCancellationFutureOwned` is passed
+        // as an argument to a function, the reference can be asserted to be valid for the
+        // rest of that function.) To avoid that, we use `MaybeDangling` which tells the
+        // compiler that the reference stored inside it might not be valid.
+        //
+        // See <https://users.rust-lang.org/t/unsafe-code-review-semi-owning-weak-rwlock-t-guard/95706>
+        // for more info.
         #[pin]
-        future: tokio::sync::futures::Notified<'static>,
+        future: MaybeDangling<tokio::sync::futures::Notified<'static>>,
         cancellation_token: CancellationToken,
     }
 }
@@ -279,7 +292,7 @@ impl WaitForCancellationFutureOwned {
             // # Safety
             //
             // cancellation_token is dropped after future due to the field ordering.
-            future: unsafe { Self::new_future(&cancellation_token) },
+            future: MaybeDangling::new(unsafe { Self::new_future(&cancellation_token) }),
             cancellation_token,
         }
     }
@@ -320,8 +333,9 @@ impl Future for WaitForCancellationFutureOwned {
             // # Safety
             //
             // cancellation_token is dropped after future due to the field ordering.
-            this.future
-                .set(unsafe { Self::new_future(this.cancellation_token) });
+            this.future.set(MaybeDangling::new(unsafe {
+                Self::new_future(this.cancellation_token)
+            }));
         }
     }
 }

tokio-util/src/task/join_map.rs

@@ -316,6 +316,60 @@ where
         self.insert(key, task);
     }
 
+    /// Spawn the blocking code on the blocking threadpool and store it in this `JoinMap` with the provided
+    /// key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// Note that blocking tasks cannot be cancelled after execution starts.
+    /// Replaced blocking tasks will still run to completion if the task has begun
+    /// to execute when it is replaced. A blocking task which is replaced before
+    /// it has been scheduled on a blocking worker thread will be cancelled.
+    ///
+    /// # Panics
+    ///
+    /// This method panics if called outside of a Tokio runtime.
+    ///
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
+    pub fn spawn_blocking<F>(&mut self, key: K, f: F)
+    where
+        F: FnOnce() -> V,
+        F: Send + 'static,
+        V: Send,
+    {
+        let task = self.tasks.spawn_blocking(f);
+        self.insert(key, task)
+    }
+
+    /// Spawn the blocking code on the blocking threadpool of the provided runtime and store it in this
+    /// `JoinMap` with the provided key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// Note that blocking tasks cannot be cancelled after execution starts.
+    /// Replaced blocking tasks will still run to completion if the task has begun
+    /// to execute when it is replaced. A blocking task which is replaced before
+    /// it has been scheduled on a blocking worker thread will be cancelled.
+    ///
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
+    pub fn spawn_blocking_on<F>(&mut self, key: K, f: F, handle: &Handle)
+    where
+        F: FnOnce() -> V,
+        F: Send + 'static,
+        V: Send,
+    {
+        let task = self.tasks.spawn_blocking_on(f, handle);
+        self.insert(key, task);
+    }
+
     /// Spawn the provided task on the current [`LocalSet`] and store it in this
     /// `JoinMap` with the provided key.
     ///

tokio-util/src/util/maybe_dangling.rs

@@ -0,0 +1,67 @@
+use core::future::Future;
+use core::mem::MaybeUninit;
+use core::pin::Pin;
+use core::task::{Context, Poll};
+
+/// A wrapper type that tells the compiler that the contents might not be valid.
+///
+/// This is necessary mainly when `T` contains a reference. In that case, the
+/// compiler will sometimes assume that the reference is always valid; in some
+/// cases it will assume this even after the destructor of `T` runs. For
+/// example, when a reference is used as a function argument, then the compiler
+/// will assume that the reference is valid until the function returns, even if
+/// the reference is destroyed during the function. When the reference is used
+/// as part of a self-referential struct, that assumption can be false. Wrapping
+/// the reference in this type prevents the compiler from making that
+/// assumption.
+///
+/// # Invariants
+///
+/// The `MaybeUninit` will always contain a valid value until the destructor runs.
+//
+// Reference
+// See <https://users.rust-lang.org/t/unsafe-code-review-semi-owning-weak-rwlock-t-guard/95706>
+//
+// TODO: replace this with an official solution once RFC #3336 or similar is available.
+// <https://github.com/rust-lang/rfcs/pull/3336>
+#[repr(transparent)]
+pub(crate) struct MaybeDangling<T>(MaybeUninit<T>);
+
+impl<T> Drop for MaybeDangling<T> {
+    fn drop(&mut self) {
+        // Safety: `0` is always initialized.
+        unsafe { core::ptr::drop_in_place(self.0.as_mut_ptr()) };
+    }
+}
+
+impl<T> MaybeDangling<T> {
+    pub(crate) fn new(inner: T) -> Self {
+        Self(MaybeUninit::new(inner))
+    }
+}
+
+impl<F: Future> Future for MaybeDangling<F> {
+    type Output = F::Output;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        // Safety: `0` is always initialized.
+        let fut = unsafe { self.map_unchecked_mut(|this| this.0.assume_init_mut()) };
+        fut.poll(cx)
+    }
+}
+
+#[test]
+fn maybedangling_runs_drop() {
+    struct SetOnDrop<'a>(&'a mut bool);
+
+    impl Drop for SetOnDrop<'_> {
+        fn drop(&mut self) {
+            *self.0 = true;
+        }
+    }
+
+    let mut success = false;
+
+    drop(MaybeDangling::new(SetOnDrop(&mut success)));
+    assert!(success);
+}

tokio-util/src/util/mod.rs

@@ -0,0 +1,8 @@
+mod maybe_dangling;
+#[cfg(any(feature = "io", feature = "codec"))]
+mod poll_buf;
+
+pub(crate) use maybe_dangling::MaybeDangling;
+#[cfg(any(feature = "io", feature = "codec"))]
+#[cfg_attr(not(feature = "io"), allow(unreachable_pub))]
+pub use poll_buf::{poll_read_buf, poll_write_buf};

tokio-util/src/util/poll_buf.rs

@@ -0,0 +1,145 @@
+use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
+
+use bytes::{Buf, BufMut};
+use futures_core::ready;
+use std::io::{self, IoSlice};
+use std::mem::MaybeUninit;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+/// Try to read data from an `AsyncRead` into an implementer of the [`BufMut`] trait.
+///
+/// [`BufMut`]: bytes::Buf
+///
+/// # Example
+///
+/// ```
+/// use bytes::{Bytes, BytesMut};
+/// use tokio_stream as stream;
+/// use tokio::io::Result;
+/// use tokio_util::io::{StreamReader, poll_read_buf};
+/// use futures::future::poll_fn;
+/// use std::pin::Pin;
+/// # #[tokio::main]
+/// # async fn main() -> std::io::Result<()> {
+///
+/// // Create a reader from an iterator. This particular reader will always be
+/// // ready.
+/// let mut read = StreamReader::new(stream::iter(vec![Result::Ok(Bytes::from_static(&[0, 1, 2, 3]))]));
+///
+/// let mut buf = BytesMut::new();
+/// let mut reads = 0;
+///
+/// loop {
+///     reads += 1;
+///     let n = poll_fn(|cx| poll_read_buf(Pin::new(&mut read), cx, &mut buf)).await?;
+///
+///     if n == 0 {
+///         break;
+///     }
+/// }
+///
+/// // one or more reads might be necessary.
+/// assert!(reads >= 1);
+/// assert_eq!(&buf[..], &[0, 1, 2, 3]);
+/// # Ok(())
+/// # }
+/// ```
+#[cfg_attr(not(feature = "io"), allow(unreachable_pub))]
+pub fn poll_read_buf<T: AsyncRead, B: BufMut>(
+    io: Pin<&mut T>,
+    cx: &mut Context<'_>,
+    buf: &mut B,
+) -> Poll<io::Result<usize>> {
+    if !buf.has_remaining_mut() {
+        return Poll::Ready(Ok(0));
+    }
+
+    let n = {
+        let dst = buf.chunk_mut();
+
+        // Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
+        // transparent wrapper around `[MaybeUninit<u8>]`.
+        let dst = unsafe { &mut *(dst as *mut _ as *mut [MaybeUninit<u8>]) };
+        let mut buf = ReadBuf::uninit(dst);
+        let ptr = buf.filled().as_ptr();
+        ready!(io.poll_read(cx, &mut buf)?);
+
+        // Ensure the pointer does not change from under us
+        assert_eq!(ptr, buf.filled().as_ptr());
+        buf.filled().len()
+    };
+
+    // Safety: This is guaranteed to be the number of initialized (and read)
+    // bytes due to the invariants provided by `ReadBuf::filled`.
+    unsafe {
+        buf.advance_mut(n);
+    }
+
+    Poll::Ready(Ok(n))
+}
+
+/// Try to write data from an implementer of the [`Buf`] trait to an
+/// [`AsyncWrite`], advancing the buffer's internal cursor.
+///
+/// This function will use [vectored writes] when the [`AsyncWrite`] supports
+/// vectored writes.
+///
+/// # Examples
+///
+/// [`File`] implements [`AsyncWrite`] and [`Cursor<&[u8]>`] implements
+/// [`Buf`]:
+///
+/// ```no_run
+/// use tokio_util::io::poll_write_buf;
+/// use tokio::io;
+/// use tokio::fs::File;
+///
+/// use bytes::Buf;
+/// use std::io::Cursor;
+/// use std::pin::Pin;
+/// use futures::future::poll_fn;
+///
+/// #[tokio::main]
+/// async fn main() -> io::Result<()> {
+///     let mut file = File::create("foo.txt").await?;
+///     let mut buf = Cursor::new(b"data to write");
+///
+///     // Loop until the entire contents of the buffer are written to
+///     // the file.
+///     while buf.has_remaining() {
+///         poll_fn(|cx| poll_write_buf(Pin::new(&mut file), cx, &mut buf)).await?;
+///     }
+///
+///     Ok(())
+/// }
+/// ```
+///
+/// [`Buf`]: bytes::Buf
+/// [`AsyncWrite`]: tokio::io::AsyncWrite
+/// [`File`]: tokio::fs::File
+/// [vectored writes]: tokio::io::AsyncWrite::poll_write_vectored
+#[cfg_attr(not(feature = "io"), allow(unreachable_pub))]
+pub fn poll_write_buf<T: AsyncWrite, B: Buf>(
+    io: Pin<&mut T>,
+    cx: &mut Context<'_>,
+    buf: &mut B,
+) -> Poll<io::Result<usize>> {
+    const MAX_BUFS: usize = 64;
+
+    if !buf.has_remaining() {
+        return Poll::Ready(Ok(0));
+    }
+
+    let n = if io.is_write_vectored() {
+        let mut slices = [IoSlice::new(&[]); MAX_BUFS];
+        let cnt = buf.chunks_vectored(&mut slices);
+        ready!(io.poll_write_vectored(cx, &slices[..cnt]))?
+    } else {
+        ready!(io.poll_write(cx, buf.chunk()))?
+    };
+
+    buf.advance(n);
+
+    Poll::Ready(Ok(n))
+}

tokio-util/tests/compat.rs

@@ -0,0 +1,43 @@
+#![cfg(all(feature = "compat"))]
+#![cfg(not(target_os = "wasi"))] // WASI does not support all fs operations
+#![warn(rust_2018_idioms)]
+
+use futures_io::SeekFrom;
+use futures_util::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
+use tempfile::NamedTempFile;
+use tokio::fs::OpenOptions;
+use tokio_util::compat::TokioAsyncWriteCompatExt;
+
+#[tokio::test]
+async fn compat_file_seek() -> futures_util::io::Result<()> {
+    let temp_file = NamedTempFile::new()?;
+    let mut file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(temp_file)
+        .await?
+        .compat_write();
+
+    file.write_all(&[0, 1, 2, 3, 4, 5]).await?;
+    file.write_all(&[6, 7]).await?;
+
+    assert_eq!(file.stream_position().await?, 8);
+
+    // Modify elements at position 2.
+    assert_eq!(file.seek(SeekFrom::Start(2)).await?, 2);
+    file.write_all(&[8, 9]).await?;
+
+    file.flush().await?;
+
+    // Verify we still have 8 elements.
+    assert_eq!(file.seek(SeekFrom::End(0)).await?, 8);
+    // Seek back to the start of the file to read and verify contents.
+    file.seek(SeekFrom::Start(0)).await?;
+
+    let mut buf = Vec::new();
+    let num_bytes = file.read_to_end(&mut buf).await?;
+    assert_eq!(&buf[..num_bytes], &[0, 1, 8, 9, 4, 5, 6, 7]);
+
+    Ok(())
+}

tokio/CHANGELOG.md

@@ -1,3 +1,169 @@
+# 1.32.0 (August 16, 2023)
+
+### Fixed
+
+- sync: fix potential quadradic behavior in `broadcast::Receiver` ([#5925])
+
+### Added
+
+- process: stabilize `Command::raw_arg` ([#5930])
+- io: enable awaiting error readiness ([#5781])
+
+### Unstable
+
+- rt(alt): improve scalability of alt runtime as the number of cores grows ([#5935])
+
+[#5925]: https://github.com/tokio-rs/tokio/pull/5925
+[#5930]: https://github.com/tokio-rs/tokio/pull/5930
+[#5781]: https://github.com/tokio-rs/tokio/pull/5781
+[#5935]: https://github.com/tokio-rs/tokio/pull/5935
+
+# 1.31.0 (August 10, 2023)
+
+### Fixed
+
+* io: delegate `WriteHalf::poll_write_vectored` ([#5914])
+
+### Unstable
+
+* rt(alt): fix memory leak in unstable next-gen scheduler prototype ([#5911])
+* rt: expose mean task poll time metric ([#5927])
+
+[#5914]: https://github.com/tokio-rs/tokio/pull/5914
+[#5911]: https://github.com/tokio-rs/tokio/pull/5911
+[#5927]: https://github.com/tokio-rs/tokio/pull/5927
+
+# 1.30.0 (August 9, 2023)
+
+This release bumps the MSRV of Tokio to 1.63. ([#5887])
+
+### Changed
+
+- tokio: reduce LLVM code generation ([#5859])
+- io: support `--cfg mio_unsupported_force_poll_poll` flag ([#5881])
+- sync: make `const_new` methods always available ([#5885])
+- sync: avoid false sharing in mpsc channel ([#5829])
+- rt: pop at least one task from inject queue ([#5908])
+
+### Added
+
+- sync: add `broadcast::Sender::new` ([#5824])
+- net: implement `UCred` for espidf ([#5868])
+- fs: add `File::options()` ([#5869])
+- time: implement extra reset variants for `Interval` ([#5878])
+- process: add `{ChildStd*}::into_owned_{fd, handle}` ([#5899])
+
+### Removed
+
+- tokio: removed unused `tokio_*` cfgs ([#5890])
+- remove build script to speed up compilation ([#5887])
+
+### Documented
+
+- sync: mention lagging in docs for `broadcast::send` ([#5820])
+- runtime: expand on sharing runtime docs ([#5858])
+- io: use vec in example for `AsyncReadExt::read_exact` ([#5863])
+- time: mark `Sleep` as `!Unpin` in docs ([#5916])
+- process: fix `raw_arg` not showing up in docs ([#5865])
+
+### Unstable
+
+- rt: add runtime ID ([#5864])
+- rt: initial implementation of new threaded runtime ([#5823])
+
+[#5820]: https://github.com/tokio-rs/tokio/pull/5820
+[#5823]: https://github.com/tokio-rs/tokio/pull/5823
+[#5824]: https://github.com/tokio-rs/tokio/pull/5824
+[#5829]: https://github.com/tokio-rs/tokio/pull/5829
+[#5858]: https://github.com/tokio-rs/tokio/pull/5858
+[#5859]: https://github.com/tokio-rs/tokio/pull/5859
+[#5863]: https://github.com/tokio-rs/tokio/pull/5863
+[#5864]: https://github.com/tokio-rs/tokio/pull/5864
+[#5865]: https://github.com/tokio-rs/tokio/pull/5865
+[#5868]: https://github.com/tokio-rs/tokio/pull/5868
+[#5869]: https://github.com/tokio-rs/tokio/pull/5869
+[#5878]: https://github.com/tokio-rs/tokio/pull/5878
+[#5881]: https://github.com/tokio-rs/tokio/pull/5881
+[#5885]: https://github.com/tokio-rs/tokio/pull/5885
+[#5887]: https://github.com/tokio-rs/tokio/pull/5887
+[#5890]: https://github.com/tokio-rs/tokio/pull/5890
+[#5899]: https://github.com/tokio-rs/tokio/pull/5899
+[#5908]: https://github.com/tokio-rs/tokio/pull/5908
+[#5916]: https://github.com/tokio-rs/tokio/pull/5916
+
+# 1.29.1 (June 29, 2023)
+
+### Fixed
+
+- rt: fix nesting two `block_in_place` with a `block_on` between ([#5837])
+
+[#5837]: https://github.com/tokio-rs/tokio/pull/5837
+
+# 1.29.0 (June 27, 2023)
+
+Technically a breaking change, the `Send` implementation is removed from
+`runtime::EnterGuard`. This change fixes a bug and should not impact most users.
+
+### Breaking
+
+- rt: `EnterGuard` should not be `Send` ([#5766])
+
+### Fixed
+
+- fs: reduce blocking ops in `fs::read_dir` ([#5653])
+- rt: fix possible starvation ([#5686], [#5712])
+- rt: fix stacked borrows issue in `JoinSet` ([#5693])
+- rt: panic if `EnterGuard` dropped incorrect order ([#5772])
+- time: do not overflow to signal value ([#5710])
+- fs: wait for in-flight ops before cloning `File` ([#5803])
+
+### Changed
+
+- rt: reduce time to poll tasks scheduled from outside the runtime ([#5705], [#5720])
+
+### Added
+
+- net: add uds doc alias for unix sockets ([#5659])
+- rt: add metric for number of tasks ([#5628])
+- sync: implement more traits for channel errors ([#5666])
+- net: add nodelay methods on TcpSocket ([#5672])
+- sync: add `broadcast::Receiver::blocking_recv` ([#5690])
+- process: add `raw_arg` method to `Command` ([#5704])
+- io: support PRIORITY epoll events ([#5566])
+- task: add `JoinSet::poll_join_next` ([#5721])
+- net: add support for Redox OS ([#5790])
+
+
+### Unstable
+
+- rt: add the ability to dump task backtraces ([#5608], [#5676], [#5708], [#5717])
+- rt: instrument task poll times with a histogram ([#5685])
+
+[#5766]: https://github.com/tokio-rs/tokio/pull/5766
+[#5653]: https://github.com/tokio-rs/tokio/pull/5653
+[#5686]: https://github.com/tokio-rs/tokio/pull/5686
+[#5712]: https://github.com/tokio-rs/tokio/pull/5712
+[#5693]: https://github.com/tokio-rs/tokio/pull/5693
+[#5772]: https://github.com/tokio-rs/tokio/pull/5772
+[#5710]: https://github.com/tokio-rs/tokio/pull/5710
+[#5803]: https://github.com/tokio-rs/tokio/pull/5803
+[#5705]: https://github.com/tokio-rs/tokio/pull/5705
+[#5720]: https://github.com/tokio-rs/tokio/pull/5720
+[#5659]: https://github.com/tokio-rs/tokio/pull/5659
+[#5628]: https://github.com/tokio-rs/tokio/pull/5628
+[#5666]: https://github.com/tokio-rs/tokio/pull/5666
+[#5672]: https://github.com/tokio-rs/tokio/pull/5672
+[#5690]: https://github.com/tokio-rs/tokio/pull/5690
+[#5704]: https://github.com/tokio-rs/tokio/pull/5704
+[#5566]: https://github.com/tokio-rs/tokio/pull/5566
+[#5721]: https://github.com/tokio-rs/tokio/pull/5721
+[#5790]: https://github.com/tokio-rs/tokio/pull/5790
+[#5608]: https://github.com/tokio-rs/tokio/pull/5608
+[#5676]: https://github.com/tokio-rs/tokio/pull/5676
+[#5708]: https://github.com/tokio-rs/tokio/pull/5708
+[#5717]: https://github.com/tokio-rs/tokio/pull/5717
+[#5685]: https://github.com/tokio-rs/tokio/pull/5685
+
 # 1.28.2 (May 28, 2023)
 
 Forward ports 1.18.6 changes.

tokio/Cargo.toml

@@ -6,9 +6,9 @@ name = "tokio"
 #   - README.md
 # - Update CHANGELOG.md.
 # - Create "v1.x.y" git tag.
-version = "1.28.2"
+version = "1.32.0"
 edition = "2021"
-rust-version = "1.56"
+rust-version = "1.63"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 readme = "README.md"
@@ -93,13 +93,10 @@ time = []
 # a few releases.
 stats = []
 
-[build-dependencies]
-autocfg = "1.1"
-
 [dependencies]
 tokio-macros = { version = "~2.1.0", path = "../tokio-macros", optional = true }
 
-pin-project-lite = "0.2.7"
+pin-project-lite = "0.2.11"
 
 # Everything else is optional...
 bytes = { version = "1.0.0", optional = true }
@@ -107,8 +104,8 @@ mio = { version = "0.8.6", optional = true, default-features = false }
 num_cpus = { version = "1.8.0", optional = true }
 parking_lot = { version = "0.12.0", optional = true }
 
-[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dependencies]
-socket2 = { version = "0.4.9", optional = true, features = [ "all" ] }
+[target.'cfg(not(target_family = "wasm"))'.dependencies]
+socket2 = { version = "0.5.3", optional = true, features = [ "all" ] }
 
 # Currently unstable. The API exposed by these features may be broken at any time.
 # Requires `--cfg tokio_unstable` to enable.
@@ -146,21 +143,21 @@ futures = { version = "0.3.0", features = ["async-await"] }
 mockall = "0.11.1"
 async-stream = "0.3"
 
-[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dev-dependencies]
-socket2 = "0.4.9"
+[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
+socket2 = "0.5.3"
 tempfile = "3.1.0"
 
-[target.'cfg(not(all(any(target_arch = "wasm32", target_arch = "wasm64"), target_os = "unknown")))'.dev-dependencies]
+[target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dev-dependencies]
 rand = "0.8.0"
 
-[target.'cfg(all(any(target_arch = "wasm32", target_arch = "wasm64"), not(target_os = "wasi")))'.dev-dependencies]
+[target.'cfg(all(target_family = "wasm", not(target_os = "wasi")))'.dev-dependencies]
 wasm-bindgen-test = "0.3.0"
 
 [target.'cfg(target_os = "freebsd")'.dev-dependencies]
 mio-aio = { version = "0.7.0", features = ["tokio"] }
 
 [target.'cfg(loom)'.dev-dependencies]
-loom = { version = "0.5.2", features = ["futures", "checkpoint"] }
+loom = { version = "0.7", features = ["futures", "checkpoint"] }
 
 [package.metadata.docs.rs]
 all-features = true

tokio/README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.28.2", features = ["full"] }
+tokio = { version = "1.32.0", features = ["full"] }
 ```
 Then, on your main.rs:
 
@@ -187,12 +187,13 @@ When updating this, also update:
 
 Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
 least** 6 months. When increasing the MSRV, the new Rust version must have been
-released at least six months ago. The current MSRV is 1.56.0.
+released at least six months ago. The current MSRV is 1.63.
 
 Note that the MSRV is not increased automatically, and only as part of a minor
 release. The MSRV history for past minor releases can be found below:
 
- * 1.27 to now - Rust 1.56
+ * 1.30 to now - Rust 1.63
+ * 1.27 to 1.29 - Rust 1.56
  * 1.17 to 1.26 - Rust 1.49
  * 1.15 to 1.16 - Rust 1.46
  * 1.0 to 1.14 - Rust 1.45
@@ -215,7 +216,6 @@ warrants a patch release with a fix for the bug, it will be backported and
 released as a new patch release for each LTS minor version. Our current LTS
 releases are:
 
- * `1.18.x` - LTS release until June 2023. (MSRV 1.49)
  * `1.20.x` - LTS release until September 2023. (MSRV 1.49)
  * `1.25.x` - LTS release until March 2024. (MSRV 1.49)
 
@@ -230,6 +230,12 @@ can use the following dependency specification:
 tokio = { version = "~1.18", features = [...] }
 ```
 
+### Previous LTS releases
+
+ * `1.8.x` - LTS release until February 2022.
+ * `1.14.x` - LTS release until June 2022.
+ * `1.18.x` - LTS release until June 2023.
+
 ## License
 
 This project is licensed under the [MIT license].

tokio/build.rs

@@ -1,192 +0,0 @@
-use autocfg::AutoCfg;
-
-const CONST_THREAD_LOCAL_PROBE: &str = r#"
-{
-    thread_local! {
-        static MY_PROBE: usize = const { 10 };
-    }
-
-    MY_PROBE.with(|val| *val)
-}
-"#;
-
-const CONST_MUTEX_NEW_PROBE: &str = r#"
-{
-    static MY_MUTEX: ::std::sync::Mutex<i32> = ::std::sync::Mutex::new(1);
-    *MY_MUTEX.lock().unwrap()
-}
-"#;
-
-const AS_FD_PROBE: &str = r#"
-{
-    #[allow(unused_imports)]
-    #[cfg(unix)]
-    use std::os::unix::prelude::AsFd as _;
-    #[allow(unused_imports)]
-    #[cfg(windows)]
-    use std::os::windows::prelude::AsSocket as _;
-    #[allow(unused_imports)]
-    #[cfg(target_os = "wasi")]
-    use std::os::wasi::prelude::AsFd as _;
-}
-"#;
-
-const TARGET_HAS_ATOMIC_PROBE: &str = r#"
-{
-    #[cfg(target_has_atomic = "ptr")]
-    let _ = ();
-}
-"#;
-
-const TARGET_ATOMIC_U64_PROBE: &str = r#"
-{
-    #[allow(unused_imports)]
-    use std::sync::atomic::AtomicU64 as _;
-}
-"#;
-
-fn main() {
-    let mut enable_const_thread_local = false;
-    let mut enable_target_has_atomic = false;
-    let mut enable_const_mutex_new = false;
-    let mut enable_as_fd = false;
-    let mut target_needs_atomic_u64_fallback = false;
-
-    match AutoCfg::new() {
-        Ok(ac) => {
-            // These checks prefer to call only `probe_rustc_version` if that is
-            // enough to determine whether the feature is supported. This is
-            // because the `probe_expression` call involves a call to rustc,
-            // which the `probe_rustc_version` call avoids.
-
-            // Const-initialized thread locals were stabilized in 1.59.
-            if ac.probe_rustc_version(1, 60) {
-                enable_const_thread_local = true;
-            } else if ac.probe_rustc_version(1, 59) {
-                // This compiler claims to be 1.59, but there are some nightly
-                // compilers that claim to be 1.59 without supporting the
-                // feature. Explicitly probe to check if code using them
-                // compiles.
-                //
-                // The oldest nightly that supports the feature is 2021-12-06.
-                if ac.probe_expression(CONST_THREAD_LOCAL_PROBE) {
-                    enable_const_thread_local = true;
-                }
-            }
-
-            // The `target_has_atomic` cfg was stabilized in 1.60.
-            if ac.probe_rustc_version(1, 61) {
-                enable_target_has_atomic = true;
-            } else if ac.probe_rustc_version(1, 60) {
-                // This compiler claims to be 1.60, but there are some nightly
-                // compilers that claim to be 1.60 without supporting the
-                // feature. Explicitly probe to check if code using them
-                // compiles.
-                //
-                // The oldest nightly that supports the feature is 2022-02-11.
-                if ac.probe_expression(TARGET_HAS_ATOMIC_PROBE) {
-                    enable_target_has_atomic = true;
-                }
-            }
-
-            // If we can't tell using `target_has_atomic`, tell if the target
-            // has `AtomicU64` by trying to use it.
-            if !enable_target_has_atomic && !ac.probe_expression(TARGET_ATOMIC_U64_PROBE) {
-                target_needs_atomic_u64_fallback = true;
-            }
-
-            // The `Mutex::new` method was made const in 1.63.
-            if ac.probe_rustc_version(1, 64) {
-                enable_const_mutex_new = true;
-            } else if ac.probe_rustc_version(1, 63) {
-                // This compiler claims to be 1.63, but there are some nightly
-                // compilers that claim to be 1.63 without supporting the
-                // feature. Explicitly probe to check if code using them
-                // compiles.
-                //
-                // The oldest nightly that supports the feature is 2022-06-20.
-                if ac.probe_expression(CONST_MUTEX_NEW_PROBE) {
-                    enable_const_mutex_new = true;
-                }
-            }
-
-            // The `AsFd` family of traits were made stable in 1.63.
-            if ac.probe_rustc_version(1, 64) {
-                enable_as_fd = true;
-            } else if ac.probe_rustc_version(1, 63) {
-                // This compiler claims to be 1.63, but there are some nightly
-                // compilers that claim to be 1.63 without supporting the
-                // feature. Explicitly probe to check if code using them
-                // compiles.
-                //
-                // The oldest nightly that supports the feature is 2022-06-16.
-                if ac.probe_expression(AS_FD_PROBE) {
-                    enable_as_fd = true;
-                }
-            }
-        }
-
-        Err(e) => {
-            // If we couldn't detect the compiler version and features, just
-            // print a warning. This isn't a fatal error: we can still build
-            // Tokio, we just can't enable cfgs automatically.
-            println!(
-                "cargo:warning=tokio: failed to detect compiler features: {}",
-                e
-            );
-        }
-    }
-
-    if !enable_const_thread_local {
-        // To disable this feature on compilers that support it, you can
-        // explicitly pass this flag with the following environment variable:
-        //
-        // RUSTFLAGS="--cfg tokio_no_const_thread_local"
-        autocfg::emit("tokio_no_const_thread_local")
-    }
-
-    if !enable_target_has_atomic {
-        // To disable this feature on compilers that support it, you can
-        // explicitly pass this flag with the following environment variable:
-        //
-        // RUSTFLAGS="--cfg tokio_no_target_has_atomic"
-        autocfg::emit("tokio_no_target_has_atomic")
-    }
-
-    if !enable_const_mutex_new {
-        // To disable this feature on compilers that support it, you can
-        // explicitly pass this flag with the following environment variable:
-        //
-        // RUSTFLAGS="--cfg tokio_no_const_mutex_new"
-        autocfg::emit("tokio_no_const_mutex_new")
-    }
-
-    if !enable_as_fd {
-        // To disable this feature on compilers that support it, you can
-        // explicitly pass this flag with the following environment variable:
-        //
-        // RUSTFLAGS="--cfg tokio_no_as_fd"
-        autocfg::emit("tokio_no_as_fd");
-    }
-
-    if target_needs_atomic_u64_fallback {
-        // To disable this feature on compilers that support it, you can
-        // explicitly pass this flag with the following environment variable:
-        //
-        // RUSTFLAGS="--cfg tokio_no_atomic_u64"
-        autocfg::emit("tokio_no_atomic_u64")
-    }
-
-    let target = ::std::env::var("TARGET").unwrap_or_default();
-
-    // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54.
-    // Note that these cfgs are unavailable in `Cargo.toml`.
-    if target.starts_with("wasm") {
-        autocfg::emit("tokio_wasm");
-        if target.contains("wasi") {
-            autocfg::emit("tokio_wasi");
-        } else {
-            autocfg::emit("tokio_wasm_not_wasi");
-        }
-    }
-}

tokio/src/doc/os.rs

@@ -11,6 +11,9 @@ pub mod windows {
         /// See [std::os::windows::io::RawHandle](https://doc.rust-lang.org/std/os/windows/io/type.RawHandle.html)
         pub type RawHandle = crate::doc::NotDefinedHere;
 
+        /// See [std::os::windows::io::OwnedHandle](https://doc.rust-lang.org/std/os/windows/io/struct.OwnedHandle.html)
+        pub type OwnedHandle = crate::doc::NotDefinedHere;
+
         /// See [std::os::windows::io::AsRawHandle](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawHandle.html)
         pub trait AsRawHandle {
             /// See [std::os::windows::io::AsRawHandle::as_raw_handle](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawHandle.html#tymethod.as_raw_handle)

tokio/src/fs/file.rs

@@ -3,7 +3,7 @@
 //! [`File`]: File
 
 use self::State::*;
-use crate::fs::asyncify;
+use crate::fs::{asyncify, OpenOptions};
 use crate::io::blocking::Buf;
 use crate::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf};
 use crate::sync::Mutex;
@@ -124,8 +124,6 @@ impl File {
     ///
     /// See [`OpenOptions`] for more details.
     ///
-    /// [`OpenOptions`]: super::OpenOptions
-    ///
     /// # Errors
     ///
     /// This function will return an error if called from outside of the Tokio
@@ -167,8 +165,6 @@ impl File {
     ///
     /// See [`OpenOptions`] for more details.
     ///
-    /// [`OpenOptions`]: super::OpenOptions
-    ///
     /// # Errors
     ///
     /// Results in an error if called from outside of the Tokio runtime or if
@@ -199,6 +195,37 @@ impl File {
         Ok(File::from_std(std_file))
     }
 
+    /// Returns a new [`OpenOptions`] object.
+    ///
+    /// This function returns a new `OpenOptions` object that you can use to
+    /// open or create a file with specific options if `open()` or `create()`
+    /// are not appropriate.
+    ///
+    /// It is equivalent to `OpenOptions::new()`, but allows you to write more
+    /// readable code. Instead of
+    /// `OpenOptions::new().append(true).open("example.log")`,
+    /// you can write `File::options().append(true).open("example.log")`. This
+    /// also avoids the need to import `OpenOptions`.
+    ///
+    /// See the [`OpenOptions::new`] function for more details.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use tokio::fs::File;
+    /// use tokio::io::AsyncWriteExt;
+    ///
+    /// # async fn dox() -> std::io::Result<()> {
+    /// let mut f = File::options().append(true).open("example.log").await?;
+    /// f.write_all(b"new line\n").await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[must_use]
+    pub fn options() -> OpenOptions {
+        OpenOptions::new()
+    }
+
     /// Converts a [`std::fs::File`][std] to a [`tokio::fs::File`][file].
     ///
     /// [std]: std::fs::File
@@ -399,6 +426,7 @@ impl File {
     /// # }
     /// ```
     pub async fn try_clone(&self) -> io::Result<File> {
+        self.inner.lock().await.complete_inflight().await;
         let std = self.std.clone();
         let std_file = asyncify(move || std.try_clone()).await?;
         Ok(File::from_std(std_file))
@@ -730,7 +758,7 @@ impl std::os::unix::io::AsRawFd for File {
     }
 }
 
-#[cfg(all(unix, not(tokio_no_as_fd)))]
+#[cfg(unix)]
 impl std::os::unix::io::AsFd for File {
     fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> {
         unsafe {
@@ -747,9 +775,7 @@ impl std::os::unix::io::FromRawFd for File {
 }
 
 cfg_windows! {
-    use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle};
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsHandle, BorrowedHandle};
+    use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle, AsHandle, BorrowedHandle};
 
     impl AsRawHandle for File {
         fn as_raw_handle(&self) -> RawHandle {
@@ -757,7 +783,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsHandle for File {
         fn as_handle(&self) -> BorrowedHandle<'_> {
             unsafe {

tokio/src/fs/read_dir.rs

@@ -139,7 +139,9 @@ impl ReadDir {
                     target_os = "solaris",
                     target_os = "illumos",
                     target_os = "haiku",
-                    target_os = "vxworks"
+                    target_os = "vxworks",
+                    target_os = "nto",
+                    target_os = "vita",
                 )))]
                 file_type: std.file_type().ok(),
                 std: Arc::new(std),
@@ -200,7 +202,9 @@ pub struct DirEntry {
         target_os = "solaris",
         target_os = "illumos",
         target_os = "haiku",
-        target_os = "vxworks"
+        target_os = "vxworks",
+        target_os = "nto",
+        target_os = "vita",
     )))]
     file_type: Option<FileType>,
     std: Arc<std::fs::DirEntry>,
@@ -331,7 +335,9 @@ impl DirEntry {
             target_os = "solaris",
             target_os = "illumos",
             target_os = "haiku",
-            target_os = "vxworks"
+            target_os = "vxworks",
+            target_os = "nto",
+            target_os = "vita",
         )))]
         if let Some(file_type) = self.file_type {
             return Ok(file_type);

tokio/src/io/async_fd.rs

@@ -176,6 +176,8 @@ use std::{task::Context, task::Poll};
 /// [`AsyncWrite`]: trait@crate::io::AsyncWrite
 pub struct AsyncFd<T: AsRawFd> {
     registration: Registration,
+    // The inner value is always present. the Option is required for `drop` and `into_inner`.
+    // In all other methods `unwrap` is valid, and will never panic.
     inner: Option<T>,
 }
 
@@ -271,13 +273,12 @@ impl<T: AsRawFd> AsyncFd<T> {
     }
 
     fn take_inner(&mut self) -> Option<T> {
-        let fd = self.inner.as_ref().map(AsRawFd::as_raw_fd);
+        let inner = self.inner.take()?;
+        let fd = inner.as_raw_fd();
 
-        if let Some(fd) = fd {
-            let _ = self.registration.deregister(&mut SourceFd(&fd));
-        }
+        let _ = self.registration.deregister(&mut SourceFd(&fd));
 
-        self.inner.take()
+        Some(inner)
     }
 
     /// Deregisters this file descriptor and returns ownership of the backing
@@ -319,11 +320,10 @@ impl<T: AsRawFd> AsyncFd<T> {
     ) -> Poll<io::Result<AsyncFdReadyGuard<'a, T>>> {
         let event = ready!(self.registration.poll_read_ready(cx))?;
 
-        Ok(AsyncFdReadyGuard {
+        Poll::Ready(Ok(AsyncFdReadyGuard {
             async_fd: self,
             event: Some(event),
-        })
-        .into()
+        }))
     }
 
     /// Polls for read readiness.
@@ -357,11 +357,10 @@ impl<T: AsRawFd> AsyncFd<T> {
     ) -> Poll<io::Result<AsyncFdReadyMutGuard<'a, T>>> {
         let event = ready!(self.registration.poll_read_ready(cx))?;
 
-        Ok(AsyncFdReadyMutGuard {
+        Poll::Ready(Ok(AsyncFdReadyMutGuard {
             async_fd: self,
             event: Some(event),
-        })
-        .into()
+        }))
     }
 
     /// Polls for write readiness.
@@ -397,11 +396,10 @@ impl<T: AsRawFd> AsyncFd<T> {
     ) -> Poll<io::Result<AsyncFdReadyGuard<'a, T>>> {
         let event = ready!(self.registration.poll_write_ready(cx))?;
 
-        Ok(AsyncFdReadyGuard {
+        Poll::Ready(Ok(AsyncFdReadyGuard {
             async_fd: self,
             event: Some(event),
-        })
-        .into()
+        }))
     }
 
     /// Polls for write readiness.
@@ -435,11 +433,10 @@ impl<T: AsRawFd> AsyncFd<T> {
     ) -> Poll<io::Result<AsyncFdReadyMutGuard<'a, T>>> {
         let event = ready!(self.registration.poll_write_ready(cx))?;
 
-        Ok(AsyncFdReadyMutGuard {
+        Poll::Ready(Ok(AsyncFdReadyMutGuard {
             async_fd: self,
             event: Some(event),
-        })
-        .into()
+        }))
     }
 
     /// Waits for any of the requested ready states, returning a
@@ -797,7 +794,6 @@ impl<T: AsRawFd> AsRawFd for AsyncFd<T> {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl<T: AsRawFd> std::os::unix::io::AsFd for AsyncFd<T> {
     fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> {
         unsafe { std::os::unix::io::BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -1014,14 +1010,11 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> {
     ) -> Result<io::Result<R>, TryIoError> {
         let result = f(self.async_fd);
 
-        if let Err(e) = result.as_ref() {
-            if e.kind() == io::ErrorKind::WouldBlock {
-                self.clear_ready();
-            }
-        }
-
         match result {
-            Err(err) if err.kind() == io::ErrorKind::WouldBlock => Err(TryIoError(())),
+            Err(err) if err.kind() == io::ErrorKind::WouldBlock => {
+                self.clear_ready();
+                Err(TryIoError(()))
+            }
             result => Ok(result),
         }
     }
@@ -1194,14 +1187,11 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyMutGuard<'a, Inner> {
     ) -> Result<io::Result<R>, TryIoError> {
         let result = f(self.async_fd);
 
-        if let Err(e) = result.as_ref() {
-            if e.kind() == io::ErrorKind::WouldBlock {
-                self.clear_ready();
-            }
-        }
-
         match result {
-            Err(err) if err.kind() == io::ErrorKind::WouldBlock => Err(TryIoError(())),
+            Err(err) if err.kind() == io::ErrorKind::WouldBlock => {
+                self.clear_ready();
+                Err(TryIoError(()))
+            }
             result => Ok(result),
         }
     }

tokio/src/io/interest.rs

@@ -5,13 +5,28 @@ use crate::io::ready::Ready;
 use std::fmt;
 use std::ops;
 
+// These must be unique.
+// same as mio
+const READABLE: usize = 0b0001;
+const WRITABLE: usize = 0b0010;
+// The following are not available on all platforms.
+#[cfg(target_os = "freebsd")]
+const AIO: usize = 0b0100;
+#[cfg(target_os = "freebsd")]
+const LIO: usize = 0b1000;
+#[cfg(any(target_os = "linux", target_os = "android"))]
+const PRIORITY: usize = 0b0001_0000;
+// error is available on all platforms, but behavior is platform-specific
+// mio does not have this interest
+const ERROR: usize = 0b0010_0000;
+
 /// Readiness event interest.
 ///
 /// Specifies the readiness events the caller is interested in when awaiting on
 /// I/O resource readiness states.
 #[cfg_attr(docsrs, doc(cfg(feature = "net")))]
 #[derive(Clone, Copy, Eq, PartialEq)]
-pub struct Interest(mio::Interest);
+pub struct Interest(usize);
 
 impl Interest {
     // The non-FreeBSD definitions in this block are active only when
@@ -19,35 +34,41 @@ impl Interest {
     cfg_aio! {
         /// Interest for POSIX AIO.
         #[cfg(target_os = "freebsd")]
-        pub const AIO: Interest = Interest(mio::Interest::AIO);
+        pub const AIO: Interest = Interest(AIO);
 
         /// Interest for POSIX AIO.
         #[cfg(not(target_os = "freebsd"))]
-        pub const AIO: Interest = Interest(mio::Interest::READABLE);
+        pub const AIO: Interest = Interest(READABLE);
 
         /// Interest for POSIX AIO lio_listio events.
         #[cfg(target_os = "freebsd")]
-        pub const LIO: Interest = Interest(mio::Interest::LIO);
+        pub const LIO: Interest = Interest(LIO);
 
         /// Interest for POSIX AIO lio_listio events.
         #[cfg(not(target_os = "freebsd"))]
-        pub const LIO: Interest = Interest(mio::Interest::READABLE);
+        pub const LIO: Interest = Interest(READABLE);
     }
 
     /// Interest in all readable events.
     ///
     /// Readable interest includes read-closed events.
-    pub const READABLE: Interest = Interest(mio::Interest::READABLE);
+    pub const READABLE: Interest = Interest(READABLE);
 
     /// Interest in all writable events.
     ///
     /// Writable interest includes write-closed events.
-    pub const WRITABLE: Interest = Interest(mio::Interest::WRITABLE);
+    pub const WRITABLE: Interest = Interest(WRITABLE);
+
+    /// Interest in error events.
+    ///
+    /// Passes error interest to the underlying OS selector.
+    /// Behavior is platform-specific, read your platform's documentation.
+    pub const ERROR: Interest = Interest(ERROR);
 
     /// Returns a `Interest` set representing priority completion interests.
     #[cfg(any(target_os = "linux", target_os = "android"))]
     #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
-    pub const PRIORITY: Interest = Interest(mio::Interest::PRIORITY);
+    pub const PRIORITY: Interest = Interest(PRIORITY);
 
     /// Returns true if the value includes readable interest.
     ///
@@ -63,7 +84,7 @@ impl Interest {
     /// assert!(both.is_readable());
     /// ```
     pub const fn is_readable(self) -> bool {
-        self.0.is_readable()
+        self.0 & READABLE != 0
     }
 
     /// Returns true if the value includes writable interest.
@@ -80,7 +101,34 @@ impl Interest {
     /// assert!(both.is_writable());
     /// ```
     pub const fn is_writable(self) -> bool {
-        self.0.is_writable()
+        self.0 & WRITABLE != 0
+    }
+
+    /// Returns true if the value includes error interest.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::io::Interest;
+    ///
+    /// assert!(Interest::ERROR.is_error());
+    /// assert!(!Interest::WRITABLE.is_error());
+    ///
+    /// let combined = Interest::READABLE | Interest::ERROR;
+    /// assert!(combined.is_error());
+    /// ```
+    pub const fn is_error(self) -> bool {
+        self.0 & ERROR != 0
+    }
+
+    #[cfg(target_os = "freebsd")]
+    const fn is_aio(self) -> bool {
+        self.0 & AIO != 0
+    }
+
+    #[cfg(target_os = "freebsd")]
+    const fn is_lio(self) -> bool {
+        self.0 & LIO != 0
     }
 
     /// Returns true if the value includes priority interest.
@@ -99,7 +147,7 @@ impl Interest {
     #[cfg(any(target_os = "linux", target_os = "android"))]
     #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
     pub const fn is_priority(self) -> bool {
-        self.0.is_priority()
+        self.0 & PRIORITY != 0
     }
 
     /// Add together two `Interest` values.
@@ -116,12 +164,60 @@ impl Interest {
     /// assert!(BOTH.is_readable());
     /// assert!(BOTH.is_writable());
     pub const fn add(self, other: Interest) -> Interest {
-        Interest(self.0.add(other.0))
+        Self(self.0 | other.0)
     }
 
     // This function must be crate-private to avoid exposing a `mio` dependency.
-    pub(crate) const fn to_mio(self) -> mio::Interest {
-        self.0
+    pub(crate) fn to_mio(self) -> mio::Interest {
+        fn mio_add(wrapped: &mut Option<mio::Interest>, add: mio::Interest) {
+            match wrapped {
+                Some(inner) => *inner |= add,
+                None => *wrapped = Some(add),
+            }
+        }
+
+        // mio does not allow and empty interest, so use None for empty
+        let mut mio = None;
+
+        if self.is_readable() {
+            mio_add(&mut mio, mio::Interest::READABLE);
+        }
+
+        if self.is_writable() {
+            mio_add(&mut mio, mio::Interest::WRITABLE);
+        }
+
+        #[cfg(any(target_os = "linux", target_os = "android"))]
+        if self.is_priority() {
+            mio_add(&mut mio, mio::Interest::PRIORITY);
+        }
+
+        #[cfg(target_os = "freebsd")]
+        if self.is_aio() {
+            mio_add(&mut mio, mio::Interest::AIO);
+        }
+
+        #[cfg(target_os = "freebsd")]
+        if self.is_lio() {
+            mio_add(&mut mio, mio::Interest::LIO);
+        }
+
+        if self.is_error() {
+            // There is no error interest in mio, because error events are always reported.
+            // But mio interests cannot be empty and an interest is needed just for the registeration.
+            //
+            // read readiness is filtered out in `Interest::mask` or `Ready::from_interest` if
+            // the read interest was not specified by the user.
+            mio_add(&mut mio, mio::Interest::READABLE);
+        }
+
+        // the default `mio::Interest::READABLE` should never be used in practice. Either
+        //
+        // - at least one tokio interest with a mio counterpart was used
+        // - only the error tokio interest was specified
+        //
+        // in both cases, `mio` is Some already
+        mio.unwrap_or(mio::Interest::READABLE)
     }
 
     pub(crate) fn mask(self) -> Ready {
@@ -130,6 +226,7 @@ impl Interest {
             Interest::WRITABLE => Ready::WRITABLE | Ready::WRITE_CLOSED,
             #[cfg(any(target_os = "linux", target_os = "android"))]
             Interest::PRIORITY => Ready::PRIORITY | Ready::READ_CLOSED,
+            Interest::ERROR => Ready::ERROR,
             _ => Ready::EMPTY,
         }
     }
@@ -147,12 +244,67 @@ impl ops::BitOr for Interest {
 impl ops::BitOrAssign for Interest {
     #[inline]
     fn bitor_assign(&mut self, other: Self) {
-        self.0 = (*self | other).0;
+        *self = *self | other
     }
 }
 
 impl fmt::Debug for Interest {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(fmt)
+        let mut separator = false;
+
+        if self.is_readable() {
+            if separator {
+                write!(fmt, " | ")?;
+            }
+            write!(fmt, "READABLE")?;
+            separator = true;
+        }
+
+        if self.is_writable() {
+            if separator {
+                write!(fmt, " | ")?;
+            }
+            write!(fmt, "WRITABLE")?;
+            separator = true;
+        }
+
+        #[cfg(any(target_os = "linux", target_os = "android"))]
+        if self.is_priority() {
+            if separator {
+                write!(fmt, " | ")?;
+            }
+            write!(fmt, "PRIORITY")?;
+            separator = true;
+        }
+
+        #[cfg(target_os = "freebsd")]
+        if self.is_aio() {
+            if separator {
+                write!(fmt, " | ")?;
+            }
+            write!(fmt, "AIO")?;
+            separator = true;
+        }
+
+        #[cfg(target_os = "freebsd")]
+        if self.is_lio() {
+            if separator {
+                write!(fmt, " | ")?;
+            }
+            write!(fmt, "LIO")?;
+            separator = true;
+        }
+
+        if self.is_error() {
+            if separator {
+                write!(fmt, " | ")?;
+            }
+            write!(fmt, "ERROR")?;
+            separator = true;
+        }
+
+        let _ = separator;
+
+        Ok(())
     }
 }

tokio/src/io/mod.rs

@@ -223,11 +223,11 @@ cfg_io_driver_impl! {
         pub use ready::Ready;
     }
 
-    #[cfg_attr(tokio_wasi, allow(unused_imports))]
+    #[cfg_attr(target_os = "wasi", allow(unused_imports))]
     mod poll_evented;
 
     #[cfg(not(loom))]
-    #[cfg_attr(tokio_wasi, allow(unused_imports))]
+    #[cfg_attr(target_os = "wasi", allow(unused_imports))]
     pub(crate) use poll_evented::PollEvented;
 }
 

tokio/src/io/poll_evented.rs

@@ -124,7 +124,7 @@ impl<E: Source> PollEvented<E> {
     }
 
     /// Returns a reference to the registration.
-    #[cfg(any(feature = "net"))]
+    #[cfg(feature = "net")]
     pub(crate) fn registration(&self) -> &Registration {
         &self.registration
     }
@@ -165,8 +165,10 @@ feature! {
                 match self.io.as_ref().unwrap().read(b) {
                     Ok(n) => {
                         // if we read a partially full buffer, this is sufficient on unix to show
-                        // that the socket buffer has been drained
-                        if n > 0 && (!cfg!(windows) && n < len) {
+                        // that the socket buffer has been drained.  Unfortunately this assumption
+                        // fails for level-triggered selectors (like on Windows or poll even for
+                        // UNIX): https://github.com/tokio-rs/tokio/issues/5866
+                        if n > 0 && (!cfg!(windows) && !cfg!(mio_unsupported_force_poll_poll) && n < len) {
                             self.registration.clear_readiness(evt);
                         }
 
@@ -196,8 +198,10 @@ feature! {
                 match self.io.as_ref().unwrap().write(buf) {
                     Ok(n) => {
                         // if we write only part of our buffer, this is sufficient on unix to show
-                        // that the socket buffer is full
-                        if n > 0 && (!cfg!(windows) && n < buf.len()) {
+                        // that the socket buffer is full.  Unfortunately this assumption
+                        // fails for level-triggered selectors (like on Windows or poll even for
+                        // UNIX): https://github.com/tokio-rs/tokio/issues/5866
+                        if n > 0 && (!cfg!(windows) && !cfg!(mio_unsupported_force_poll_poll) && n < buf.len()) {
                             self.registration.clear_readiness(evt);
                         }
 

tokio/src/io/ready.rs

@@ -1,5 +1,7 @@
 #![cfg_attr(not(feature = "net"), allow(unreachable_pub))]
 
+use crate::io::interest::Interest;
+
 use std::fmt;
 use std::ops;
 
@@ -9,6 +11,7 @@ const READ_CLOSED: usize = 0b0_0100;
 const WRITE_CLOSED: usize = 0b0_1000;
 #[cfg(any(target_os = "linux", target_os = "android"))]
 const PRIORITY: usize = 0b1_0000;
+const ERROR: usize = 0b10_0000;
 
 /// Describes the readiness state of an I/O resources.
 ///
@@ -38,13 +41,17 @@ impl Ready {
     #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))]
     pub const PRIORITY: Ready = Ready(PRIORITY);
 
+    /// Returns a `Ready` representing error readiness.
+    pub const ERROR: Ready = Ready(ERROR);
+
     /// Returns a `Ready` representing readiness for all operations.
     #[cfg(any(target_os = "linux", target_os = "android"))]
-    pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED | PRIORITY);
+    pub const ALL: Ready =
+        Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED | ERROR | PRIORITY);
 
     /// Returns a `Ready` representing readiness for all operations.
     #[cfg(not(any(target_os = "linux", target_os = "android")))]
-    pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED);
+    pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED | ERROR);
 
     // Must remain crate-private to avoid adding a public dependency on Mio.
     pub(crate) fn from_mio(event: &mio::event::Event) -> Ready {
@@ -77,6 +84,10 @@ impl Ready {
             ready |= Ready::WRITE_CLOSED;
         }
 
+        if event.is_error() {
+            ready |= Ready::ERROR;
+        }
+
         #[cfg(any(target_os = "linux", target_os = "android"))]
         {
             if event.is_priority() {
@@ -180,6 +191,21 @@ impl Ready {
         self.contains(Ready::PRIORITY)
     }
 
+    /// Returns `true` if the value includes error `readiness`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::io::Ready;
+    ///
+    /// assert!(!Ready::EMPTY.is_error());
+    /// assert!(!Ready::WRITABLE.is_error());
+    /// assert!(Ready::ERROR.is_error());
+    /// ```
+    pub fn is_error(self) -> bool {
+        self.contains(Ready::ERROR)
+    }
+
     /// Returns true if `self` is a superset of `other`.
     ///
     /// `other` may represent more than one readiness operations, in which case
@@ -208,41 +234,39 @@ impl Ready {
     pub(crate) fn as_usize(self) -> usize {
         self.0
     }
-}
 
-cfg_io_readiness! {
-    use crate::io::Interest;
+    pub(crate) fn from_interest(interest: Interest) -> Ready {
+        let mut ready = Ready::EMPTY;
 
-    impl Ready {
-        pub(crate) fn from_interest(interest: Interest) -> Ready {
-            let mut ready = Ready::EMPTY;
-
-            if interest.is_readable() {
-                ready |= Ready::READABLE;
-                ready |= Ready::READ_CLOSED;
-            }
-
-            if interest.is_writable() {
-                ready |= Ready::WRITABLE;
-                ready |= Ready::WRITE_CLOSED;
-            }
-
-            #[cfg(any(target_os = "linux", target_os = "android"))]
-            if interest.is_priority() {
-                ready |= Ready::PRIORITY;
-                ready |= Ready::READ_CLOSED;
-            }
-
-            ready
+        if interest.is_readable() {
+            ready |= Ready::READABLE;
+            ready |= Ready::READ_CLOSED;
         }
 
-        pub(crate) fn intersection(self, interest: Interest) -> Ready {
-            Ready(self.0 & Ready::from_interest(interest).0)
+        if interest.is_writable() {
+            ready |= Ready::WRITABLE;
+            ready |= Ready::WRITE_CLOSED;
         }
 
-        pub(crate) fn satisfies(self, interest: Interest) -> bool {
-            self.0 & Ready::from_interest(interest).0 != 0
+        #[cfg(any(target_os = "linux", target_os = "android"))]
+        if interest.is_priority() {
+            ready |= Ready::PRIORITY;
+            ready |= Ready::READ_CLOSED;
         }
+
+        if interest.is_error() {
+            ready |= Ready::ERROR;
+        }
+
+        ready
+    }
+
+    pub(crate) fn intersection(self, interest: Interest) -> Ready {
+        Ready(self.0 & Ready::from_interest(interest).0)
+    }
+
+    pub(crate) fn satisfies(self, interest: Interest) -> bool {
+        self.0 & Ready::from_interest(interest).0 != 0
     }
 }
 
@@ -287,7 +311,8 @@ impl fmt::Debug for Ready {
         fmt.field("is_readable", &self.is_readable())
             .field("is_writable", &self.is_writable())
             .field("is_read_closed", &self.is_read_closed())
-            .field("is_write_closed", &self.is_write_closed());
+            .field("is_write_closed", &self.is_write_closed())
+            .field("is_error", &self.is_error());
 
         #[cfg(any(target_os = "linux", target_os = "android"))]
         fmt.field("is_priority", &self.is_priority());

tokio/src/io/split.rs

@@ -35,9 +35,12 @@ cfg_io_util! {
     where
         T: AsyncRead + AsyncWrite,
     {
+        let is_write_vectored = stream.is_write_vectored();
+
         let inner = Arc::new(Inner {
             locked: AtomicBool::new(false),
             stream: UnsafeCell::new(stream),
+            is_write_vectored,
         });
 
         let rd = ReadHalf {
@@ -53,6 +56,7 @@ cfg_io_util! {
 struct Inner<T> {
     locked: AtomicBool,
     stream: UnsafeCell<T>,
+    is_write_vectored: bool,
 }
 
 struct Guard<'a, T> {
@@ -131,6 +135,19 @@ impl<T: AsyncWrite> AsyncWrite for WriteHalf<T> {
         let mut inner = ready!(self.inner.poll_lock(cx));
         inner.stream_pin().poll_shutdown(cx)
     }
+
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[io::IoSlice<'_>],
+    ) -> Poll<Result<usize, io::Error>> {
+        let mut inner = ready!(self.inner.poll_lock(cx));
+        inner.stream_pin().poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.is_write_vectored
+    }
 }
 
 impl<T> Inner<T> {

tokio/src/io/stderr.rs

@@ -75,9 +75,7 @@ cfg_io_std! {
 
 #[cfg(unix)]
 mod sys {
-    #[cfg(not(tokio_no_as_fd))]
-    use std::os::unix::io::{AsFd, BorrowedFd};
-    use std::os::unix::io::{AsRawFd, RawFd};
+    use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, RawFd};
 
     use super::Stderr;
 
@@ -87,7 +85,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for Stderr {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -96,9 +93,7 @@ mod sys {
 }
 
 cfg_windows! {
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsHandle, BorrowedHandle};
-    use crate::os::windows::io::{AsRawHandle, RawHandle};
+    use crate::os::windows::io::{AsHandle, BorrowedHandle, AsRawHandle, RawHandle};
 
     impl AsRawHandle for Stderr {
         fn as_raw_handle(&self) -> RawHandle {
@@ -106,7 +101,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsHandle for Stderr {
         fn as_handle(&self) -> BorrowedHandle<'_> {
             unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }

tokio/src/io/stdin.rs

@@ -50,9 +50,7 @@ cfg_io_std! {
 
 #[cfg(unix)]
 mod sys {
-    #[cfg(not(tokio_no_as_fd))]
-    use std::os::unix::io::{AsFd, BorrowedFd};
-    use std::os::unix::io::{AsRawFd, RawFd};
+    use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, RawFd};
 
     use super::Stdin;
 
@@ -62,7 +60,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for Stdin {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -71,9 +68,7 @@ mod sys {
 }
 
 cfg_windows! {
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsHandle, BorrowedHandle};
-    use crate::os::windows::io::{AsRawHandle, RawHandle};
+    use crate::os::windows::io::{AsHandle, BorrowedHandle, AsRawHandle, RawHandle};
 
     impl AsRawHandle for Stdin {
         fn as_raw_handle(&self) -> RawHandle {
@@ -81,7 +76,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsHandle for Stdin {
         fn as_handle(&self) -> BorrowedHandle<'_> {
             unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }

tokio/src/io/stdout.rs

@@ -74,9 +74,7 @@ cfg_io_std! {
 
 #[cfg(unix)]
 mod sys {
-    #[cfg(not(tokio_no_as_fd))]
-    use std::os::unix::io::{AsFd, BorrowedFd};
-    use std::os::unix::io::{AsRawFd, RawFd};
+    use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, RawFd};
 
     use super::Stdout;
 
@@ -86,7 +84,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for Stdout {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -95,9 +92,7 @@ mod sys {
 }
 
 cfg_windows! {
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsHandle, BorrowedHandle};
-    use crate::os::windows::io::{AsRawHandle, RawHandle};
+    use crate::os::windows::io::{AsHandle, BorrowedHandle, AsRawHandle, RawHandle};
 
     impl AsRawHandle for Stdout {
         fn as_raw_handle(&self) -> RawHandle {
@@ -105,7 +100,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsHandle for Stdout {
         fn as_handle(&self) -> BorrowedHandle<'_> {
             unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }

tokio/src/io/util/async_read_ext.rs

@@ -27,7 +27,7 @@ cfg_io_util! {
         ) => {
             $(
                 $(#[$outer])*
-                fn $name<'a>(&'a mut self) -> $($fut)*<&'a mut Self> where Self: Unpin {
+                fn $name(&mut self) -> $($fut)*<&mut Self> where Self: Unpin {
                     $($fut)*::new(self)
                 }
             )*
@@ -230,10 +230,13 @@ cfg_io_util! {
         ///     let mut buffer = BytesMut::with_capacity(10);
         ///
         ///     assert!(buffer.is_empty());
+        ///     assert!(buffer.capacity() >= 10);
         ///
-        ///     // read up to 10 bytes, note that the return value is not needed
-        ///     // to access the data that was read as `buffer`'s internal
-        ///     // cursor is updated.
+        ///     // note that the return value is not needed to access the data
+        ///     // that was read as `buffer`'s internal cursor is updated.
+        ///     //
+        ///     // this might read more than 10 bytes if the capacity of `buffer`
+        ///     // is larger than 10.
         ///     f.read_buf(&mut buffer).await?;
         ///
         ///     println!("The bytes: {:?}", &buffer[..]);
@@ -292,7 +295,8 @@ cfg_io_util! {
         /// #[tokio::main]
         /// async fn main() -> io::Result<()> {
         ///     let mut f = File::open("foo.txt").await?;
-        ///     let mut buffer = [0; 10];
+        ///     let len = 10;
+        ///     let mut buffer = vec![0; len];
         ///
         ///     // read exactly 10 bytes
         ///     f.read_exact(&mut buffer).await?;

tokio/src/io/util/async_write_ext.rs

@@ -30,7 +30,7 @@ cfg_io_util! {
         ) => {
             $(
                 $(#[$outer])*
-                fn $name<'a>(&'a mut self, n: $ty) -> $($fut)*<&'a mut Self> where Self: Unpin {
+                fn $name(&mut self, n: $ty) -> $($fut)*<&mut Self> where Self: Unpin {
                     $($fut)*::new(self, n)
                 }
             )*

tokio/src/lib.rs

@@ -456,26 +456,9 @@ compile_error! {
     "Tokio requires the platform pointer width to be 32, 64, or 128 bits"
 }
 
-// Ensure that our build script has correctly set cfg flags for wasm.
-//
-// Each condition is written all(a, not(b)). This should be read as
-// "if a, then we must also have b".
-#[cfg(any(
-    all(target_arch = "wasm32", not(tokio_wasm)),
-    all(target_arch = "wasm64", not(tokio_wasm)),
-    all(target_family = "wasm", not(tokio_wasm)),
-    all(target_os = "wasi", not(tokio_wasm)),
-    all(target_os = "wasi", not(tokio_wasi)),
-    all(target_os = "wasi", tokio_wasm_not_wasi),
-    all(tokio_wasm, not(any(target_arch = "wasm32", target_arch = "wasm64"))),
-    all(tokio_wasm_not_wasi, not(tokio_wasm)),
-    all(tokio_wasi, not(tokio_wasm))
-))]
-compile_error!("Tokio's build script has incorrectly detected wasm.");
-
 #[cfg(all(
     not(tokio_unstable),
-    tokio_wasm,
+    target_family = "wasm",
     any(
         feature = "fs",
         feature = "io-std",

tokio/src/loom/mocked.rs

@@ -15,6 +15,7 @@ pub(crate) mod sync {
         }
 
         #[inline]
+        #[track_caller]
         pub(crate) fn lock(&self) -> MutexGuard<'_, T> {
             self.0.lock().unwrap()
         }

tokio/src/loom/std/mod.rs

@@ -6,7 +6,7 @@ mod atomic_u64;
 mod atomic_usize;
 mod barrier;
 mod mutex;
-#[cfg(feature = "parking_lot")]
+#[cfg(all(feature = "parking_lot", not(miri)))]
 mod parking_lot;
 mod unsafe_cell;
 
@@ -56,17 +56,17 @@ pub(crate) mod sync {
     // internal use. Note however that some are not _currently_ named by
     // consuming code.
 
-    #[cfg(feature = "parking_lot")]
+    #[cfg(all(feature = "parking_lot", not(miri)))]
     #[allow(unused_imports)]
     pub(crate) use crate::loom::std::parking_lot::{
         Condvar, Mutex, MutexGuard, RwLock, RwLockReadGuard, WaitTimeoutResult,
     };
 
-    #[cfg(not(feature = "parking_lot"))]
+    #[cfg(not(all(feature = "parking_lot", not(miri))))]
     #[allow(unused_imports)]
     pub(crate) use std::sync::{Condvar, MutexGuard, RwLock, RwLockReadGuard, WaitTimeoutResult};
 
-    #[cfg(not(feature = "parking_lot"))]
+    #[cfg(not(all(feature = "parking_lot", not(miri))))]
     pub(crate) use crate::loom::std::mutex::Mutex;
 
     pub(crate) mod atomic {

tokio/src/loom/std/mutex.rs

@@ -13,7 +13,6 @@ impl<T> Mutex<T> {
     }
 
     #[inline]
-    #[cfg(not(tokio_no_const_mutex_new))]
     pub(crate) const fn const_new(t: T) -> Mutex<T> {
         Mutex(sync::Mutex::new(t))
     }

tokio/src/loom/std/parking_lot.rs

@@ -52,8 +52,7 @@ impl<T> Mutex<T> {
     }
 
     #[inline]
-    #[cfg(all(feature = "parking_lot", not(all(loom, test))))]
-    #[cfg_attr(docsrs, doc(cfg(all(feature = "parking_lot",))))]
+    #[cfg(not(all(loom, test)))]
     pub(crate) const fn const_new(t: T) -> Mutex<T> {
         Mutex(PhantomData, parking_lot::const_mutex(t))
     }

tokio/src/loom/std/unsafe_cell.rs

@@ -6,10 +6,12 @@ impl<T> UnsafeCell<T> {
         UnsafeCell(std::cell::UnsafeCell::new(data))
     }
 
+    #[inline(always)]
     pub(crate) fn with<R>(&self, f: impl FnOnce(*const T) -> R) -> R {
         f(self.0.get())
     }
 
+    #[inline(always)]
     pub(crate) fn with_mut<R>(&self, f: impl FnOnce(*mut T) -> R) -> R {
         f(self.0.get())
     }

tokio/src/macros/cfg.rs

@@ -25,6 +25,18 @@ macro_rules! cfg_windows {
     }
 }
 
+/// Enables unstable Windows-specific code.
+/// Use this macro instead of `cfg(windows)` to generate docs properly.
+macro_rules! cfg_unstable_windows {
+    ($($item:item)*) => {
+        $(
+            #[cfg(all(any(all(doc, docsrs), windows), tokio_unstable))]
+            #[cfg_attr(docsrs, doc(cfg(all(windows, tokio_unstable))))]
+            $item
+        )*
+    }
+}
+
 /// Enables enter::block_on.
 macro_rules! cfg_block_on {
     ($($item:item)*) => {
@@ -73,7 +85,7 @@ macro_rules! cfg_fs {
     ($($item:item)*) => {
         $(
             #[cfg(feature = "fs")]
-            #[cfg(not(tokio_wasi))]
+            #[cfg(not(target_os = "wasi"))]
             #[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
             $item
         )*
@@ -264,7 +276,7 @@ macro_rules! cfg_process {
             #[cfg(feature = "process")]
             #[cfg_attr(docsrs, doc(cfg(feature = "process")))]
             #[cfg(not(loom))]
-            #[cfg(not(tokio_wasi))]
+            #[cfg(not(target_os = "wasi"))]
             $item
         )*
     }
@@ -293,7 +305,7 @@ macro_rules! cfg_signal {
             #[cfg(feature = "signal")]
             #[cfg_attr(docsrs, doc(cfg(feature = "signal")))]
             #[cfg(not(loom))]
-            #[cfg(not(tokio_wasi))]
+            #[cfg(not(target_os = "wasi"))]
             $item
         )*
     }
@@ -360,7 +372,7 @@ macro_rules! cfg_not_rt {
 macro_rules! cfg_rt_multi_thread {
     ($($item:item)*) => {
         $(
-            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
             $item
         )*
@@ -511,14 +523,7 @@ macro_rules! cfg_not_coop {
 macro_rules! cfg_has_atomic_u64 {
     ($($item:item)*) => {
         $(
-            #[cfg_attr(
-                not(tokio_no_target_has_atomic),
-                cfg(all(target_has_atomic = "64", not(tokio_no_atomic_u64))
-            ))]
-            #[cfg_attr(
-                tokio_no_target_has_atomic,
-                cfg(not(tokio_no_atomic_u64))
-            )]
+            #[cfg(target_has_atomic = "64")]
             $item
         )*
     }
@@ -527,14 +532,7 @@ macro_rules! cfg_has_atomic_u64 {
 macro_rules! cfg_not_has_atomic_u64 {
     ($($item:item)*) => {
         $(
-            #[cfg_attr(
-                not(tokio_no_target_has_atomic),
-                cfg(any(not(target_has_atomic = "64"), tokio_no_atomic_u64)
-            ))]
-            #[cfg_attr(
-                tokio_no_target_has_atomic,
-                cfg(tokio_no_atomic_u64)
-            )]
+            #[cfg(not(target_has_atomic = "64"))]
             $item
         )*
     }
@@ -543,13 +541,7 @@ macro_rules! cfg_not_has_atomic_u64 {
 macro_rules! cfg_has_const_mutex_new {
     ($($item:item)*) => {
         $(
-            #[cfg(all(
-                not(all(loom, test)),
-                any(
-                    feature = "parking_lot",
-                    not(tokio_no_const_mutex_new)
-                )
-            ))]
+            #[cfg(not(all(loom, test)))]
             $item
         )*
     }
@@ -558,13 +550,7 @@ macro_rules! cfg_has_const_mutex_new {
 macro_rules! cfg_not_has_const_mutex_new {
     ($($item:item)*) => {
         $(
-            #[cfg(not(all(
-                not(all(loom, test)),
-                any(
-                    feature = "parking_lot",
-                    not(tokio_no_const_mutex_new)
-                )
-            )))]
+            #[cfg(all(loom, test))]
             $item
         )*
     }
@@ -573,7 +559,7 @@ macro_rules! cfg_not_has_const_mutex_new {
 macro_rules! cfg_not_wasi {
     ($($item:item)*) => {
         $(
-            #[cfg(not(tokio_wasi))]
+            #[cfg(not(target_os = "wasi"))]
             $item
         )*
     }
@@ -582,7 +568,7 @@ macro_rules! cfg_not_wasi {
 macro_rules! cfg_is_wasm_not_wasi {
     ($($item:item)*) => {
         $(
-            #[cfg(tokio_wasm_not_wasi)]
+            #[cfg(all(target_family = "wasm", not(target_os = "wasi")))]
             $item
         )*
     }

tokio/src/macros/thread_local.rs

@@ -10,23 +10,9 @@ macro_rules! tokio_thread_local {
     ($($tts:tt)+) => { loom::thread_local!{ $($tts)+ } }
 }
 
-#[cfg(not(tokio_no_const_thread_local))]
 #[cfg(not(all(loom, test)))]
 macro_rules! tokio_thread_local {
     ($($tts:tt)+) => {
         ::std::thread_local!{ $($tts)+ }
     }
 }
-
-#[cfg(tokio_no_const_thread_local)]
-#[cfg(not(all(loom, test)))]
-macro_rules! tokio_thread_local {
-    ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => {
-        ::std::thread_local! {
-            $(#[$attrs])*
-            $vis static $name: $ty = $expr;
-        }
-    };
-
-    ($($tts:tt)+) => { ::std::thread_local!{ $($tts)+ } }
-}

tokio/src/net/tcp/listener.rs

@@ -281,7 +281,7 @@ impl TcpListener {
                 .map(|raw_socket| unsafe { std::net::TcpListener::from_raw_socket(raw_socket) })
         }
 
-        #[cfg(tokio_wasi)]
+        #[cfg(target_os = "wasi")]
         {
             use std::os::wasi::io::{FromRawFd, IntoRawFd};
             self.io
@@ -407,7 +407,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for TcpListener {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -416,7 +415,7 @@ mod sys {
 }
 
 cfg_unstable! {
-    #[cfg(tokio_wasi)]
+    #[cfg(target_os = "wasi")]
     mod sys {
         use super::TcpListener;
         use std::os::wasi::prelude::*;
@@ -427,7 +426,6 @@ cfg_unstable! {
             }
         }
 
-        #[cfg(not(tokio_no_as_fd))]
         impl AsFd for TcpListener {
             fn as_fd(&self) -> BorrowedFd<'_> {
                 unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -437,9 +435,7 @@ cfg_unstable! {
 }
 
 cfg_windows! {
-    use crate::os::windows::io::{AsRawSocket, RawSocket};
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsSocket, BorrowedSocket};
+    use crate::os::windows::io::{AsRawSocket, RawSocket, AsSocket, BorrowedSocket};
 
     impl AsRawSocket for TcpListener {
         fn as_raw_socket(&self) -> RawSocket {
@@ -447,7 +443,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsSocket for TcpListener {
         fn as_socket(&self) -> BorrowedSocket<'_> {
             unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) }

tokio/src/net/tcp/socket.rs

@@ -4,16 +4,12 @@ use std::fmt;
 use std::io;
 use std::net::SocketAddr;
 
-#[cfg(all(unix, not(tokio_no_as_fd)))]
-use std::os::unix::io::{AsFd, BorrowedFd};
 #[cfg(unix)]
-use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd};
 use std::time::Duration;
 
 cfg_windows! {
-    use crate::os::windows::io::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket};
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsSocket, BorrowedSocket};
+    use crate::os::windows::io::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket, AsSocket, BorrowedSocket};
 }
 
 cfg_net! {
@@ -457,7 +453,7 @@ impl TcpSocket {
     /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
     ///
     /// [`set_tos`]: Self::set_tos
-    // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+    // https://docs.rs/socket2/0.5.3/src/socket2/socket.rs.html#1464
     #[cfg(not(any(
         target_os = "fuchsia",
         target_os = "redox",
@@ -484,7 +480,7 @@ impl TcpSocket {
     ///
     /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or
     /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
-    // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+    // https://docs.rs/socket2/0.5.3/src/socket2/socket.rs.html#1446
     #[cfg(not(any(
         target_os = "fuchsia",
         target_os = "redox",
@@ -523,7 +519,7 @@ impl TcpSocket {
     /// works for some socket types, particularly `AF_INET` sockets.
     ///
     /// If `interface` is `None` or an empty string it removes the binding.
-    #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))]
+    #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))]
     #[cfg_attr(
         docsrs,
         doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))))
@@ -788,7 +784,7 @@ impl AsRawFd for TcpSocket {
     }
 }
 
-#[cfg(all(unix, not(tokio_no_as_fd)))]
+#[cfg(unix)]
 impl AsFd for TcpSocket {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -829,7 +825,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsSocket for TcpSocket {
         fn as_socket(&self) -> BorrowedSocket<'_> {
             unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) }

tokio/src/net/tcp/stream.rs

@@ -258,7 +258,7 @@ impl TcpStream {
                 .map(|raw_socket| unsafe { std::net::TcpStream::from_raw_socket(raw_socket) })
         }
 
-        #[cfg(tokio_wasi)]
+        #[cfg(target_os = "wasi")]
         {
             use std::os::wasi::io::{FromRawFd, IntoRawFd};
             self.io
@@ -1378,7 +1378,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for TcpStream {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -1387,9 +1386,7 @@ mod sys {
 }
 
 cfg_windows! {
-    use crate::os::windows::io::{AsRawSocket, RawSocket};
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsSocket, BorrowedSocket};
+    use crate::os::windows::io::{AsRawSocket, RawSocket, AsSocket, BorrowedSocket};
 
     impl AsRawSocket for TcpStream {
         fn as_raw_socket(&self) -> RawSocket {
@@ -1397,7 +1394,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsSocket for TcpStream {
         fn as_socket(&self) -> BorrowedSocket<'_> {
             unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) }
@@ -1405,7 +1401,7 @@ cfg_windows! {
     }
 }
 
-#[cfg(all(tokio_unstable, tokio_wasi))]
+#[cfg(all(tokio_unstable, target_os = "wasi"))]
 mod sys {
     use super::TcpStream;
     use std::os::wasi::prelude::*;
@@ -1416,7 +1412,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for TcpStream {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }

tokio/src/net/udp.rs

@@ -1855,7 +1855,7 @@ impl UdpSocket {
     /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
     ///
     /// [`set_tos`]: Self::set_tos
-    // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+    // https://docs.rs/socket2/0.5.3/src/socket2/socket.rs.html#1464
     #[cfg(not(any(
         target_os = "fuchsia",
         target_os = "redox",
@@ -1882,7 +1882,7 @@ impl UdpSocket {
     ///
     /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or
     /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
-    // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+    // https://docs.rs/socket2/0.5.3/src/socket2/socket.rs.html#1446
     #[cfg(not(any(
         target_os = "fuchsia",
         target_os = "redox",
@@ -1921,7 +1921,7 @@ impl UdpSocket {
     /// works for some socket types, particularly `AF_INET` sockets.
     ///
     /// If `interface` is `None` or an empty string it removes the binding.
-    #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))]
+    #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))]
     #[cfg_attr(
         docsrs,
         doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))))
@@ -2021,7 +2021,6 @@ mod sys {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsFd for UdpSocket {
         fn as_fd(&self) -> BorrowedFd<'_> {
             unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -2031,7 +2030,6 @@ mod sys {
 
 cfg_windows! {
     use crate::os::windows::io::{AsRawSocket, RawSocket};
-    #[cfg(not(tokio_no_as_fd))]
     use crate::os::windows::io::{AsSocket, BorrowedSocket};
 
     impl AsRawSocket for UdpSocket {
@@ -2040,7 +2038,6 @@ cfg_windows! {
         }
     }
 
-    #[cfg(not(tokio_no_as_fd))]
     impl AsSocket for UdpSocket {
         fn as_socket(&self) -> BorrowedSocket<'_> {
             unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) }

tokio/src/net/unix/datagram/socket.rs

@@ -4,9 +4,7 @@ use crate::net::unix::SocketAddr;
 use std::fmt;
 use std::io;
 use std::net::Shutdown;
-#[cfg(not(tokio_no_as_fd))]
-use std::os::unix::io::{AsFd, BorrowedFd};
-use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd};
 use std::os::unix::net;
 use std::path::Path;
 use std::task::{Context, Poll};
@@ -1578,7 +1576,6 @@ impl AsRawFd for UnixDatagram {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for UnixDatagram {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }

tokio/src/net/unix/listener.rs

@@ -3,9 +3,7 @@ use crate::net::unix::{SocketAddr, UnixStream};
 
 use std::fmt;
 use std::io;
-#[cfg(not(tokio_no_as_fd))]
-use std::os::unix::io::{AsFd, BorrowedFd};
-use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd};
 use std::os::unix::net;
 use std::path::Path;
 use std::task::{Context, Poll};
@@ -211,7 +209,6 @@ impl AsRawFd for UnixListener {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for UnixListener {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }

tokio/src/net/unix/pipe.rs

@@ -7,9 +7,7 @@ use mio::unix::pipe as mio_pipe;
 use std::fs::File;
 use std::io::{self, Read, Write};
 use std::os::unix::fs::{FileTypeExt, OpenOptionsExt};
-#[cfg(not(tokio_no_as_fd))]
-use std::os::unix::io::{AsFd, BorrowedFd};
-use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd};
 use std::path::Path;
 use std::pin::Pin;
 use std::task::{Context, Poll};
@@ -664,7 +662,6 @@ impl AsRawFd for Sender {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for Sender {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
@@ -1170,7 +1167,6 @@ impl AsRawFd for Receiver {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for Receiver {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }

tokio/src/net/unix/stream.rs

@@ -8,9 +8,7 @@ use crate::net::unix::SocketAddr;
 use std::fmt;
 use std::io::{self, Read, Write};
 use std::net::Shutdown;
-#[cfg(not(tokio_no_as_fd))]
-use std::os::unix::io::{AsFd, BorrowedFd};
-use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd};
 use std::os::unix::net;
 use std::path::Path;
 use std::pin::Pin;
@@ -1039,7 +1037,6 @@ impl AsRawFd for UnixStream {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for UnixStream {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }

tokio/src/net/unix/ucred.rs

@@ -39,7 +39,7 @@ impl UCred {
 ))]
 pub(crate) use self::impl_linux::get_peer_cred;
 
-#[cfg(any(target_os = "netbsd"))]
+#[cfg(target_os = "netbsd")]
 pub(crate) use self::impl_netbsd::get_peer_cred;
 
 #[cfg(any(target_os = "dragonfly", target_os = "freebsd"))]
@@ -54,6 +54,9 @@ pub(crate) use self::impl_solaris::get_peer_cred;
 #[cfg(target_os = "aix")]
 pub(crate) use self::impl_aix::get_peer_cred;
 
+#[cfg(target_os = "espidf")]
+pub(crate) use self::impl_noproc::get_peer_cred;
+
 #[cfg(any(
     target_os = "linux",
     target_os = "redox",
@@ -111,7 +114,7 @@ pub(crate) mod impl_linux {
     }
 }
 
-#[cfg(any(target_os = "netbsd"))]
+#[cfg(target_os = "netbsd")]
 pub(crate) mod impl_netbsd {
     use crate::net::unix::{self, UnixStream};
 
@@ -291,3 +294,17 @@ pub(crate) mod impl_aix {
         }
     }
 }
+
+#[cfg(target_os = "espidf")]
+pub(crate) mod impl_noproc {
+    use crate::net::unix::UnixStream;
+    use std::io;
+
+    pub(crate) fn get_peer_cred(_sock: &UnixStream) -> io::Result<super::UCred> {
+        Ok(super::UCred {
+            uid: 0,
+            gid: 0,
+            pid: None,
+        })
+    }
+}

tokio/src/net/windows/named_pipe.rs

@@ -10,9 +10,7 @@ use std::ptr;
 use std::task::{Context, Poll};
 
 use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready};
-#[cfg(not(tokio_no_as_fd))]
-use crate::os::windows::io::{AsHandle, BorrowedHandle};
-use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle};
+use crate::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle, FromRawHandle, RawHandle};
 
 cfg_io_util! {
     use bytes::BufMut;
@@ -930,7 +928,6 @@ impl AsRawHandle for NamedPipeServer {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsHandle for NamedPipeServer {
     fn as_handle(&self) -> BorrowedHandle<'_> {
         unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }
@@ -1720,7 +1717,6 @@ impl AsRawHandle for NamedPipeClient {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsHandle for NamedPipeClient {
     fn as_handle(&self) -> BorrowedHandle<'_> {
         unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }

tokio/src/process/mod.rs

@@ -260,8 +260,6 @@ use std::os::windows::process::CommandExt;
 
 cfg_windows! {
     use crate::os::windows::io::{AsRawHandle, RawHandle};
-    #[cfg(not(tokio_no_as_fd))]
-    use crate::os::windows::io::{AsHandle, BorrowedHandle};
 }
 
 /// This structure mimics the API of [`std::process::Command`] found in the standard library, but
@@ -400,20 +398,15 @@ impl Command {
         self
     }
 
-    /// Append literal text to the command line without any quoting or escaping.
-    ///
-    /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow
-    /// `CommandLineToArgvW` escaping rules.
-    ///
-    /// **Note**: This is an [unstable API][unstable] but will be stabilised once
-    /// tokio's MSRV is sufficiently new. See [the documentation on
-    /// unstable features][unstable] for details about using unstable features.
-    #[cfg(windows)]
-    #[cfg(tokio_unstable)]
-    #[cfg_attr(docsrs, doc(cfg(all(windows, tokio_unstable))))]
-    pub fn raw_arg<S: AsRef<OsStr>>(&mut self, text_to_append_as_is: S) -> &mut Command {
-        self.std.raw_arg(text_to_append_as_is);
-        self
+    cfg_windows! {
+        /// Append literal text to the command line without any quoting or escaping.
+        ///
+        /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow
+        /// `CommandLineToArgvW` escaping rules.
+        pub fn raw_arg<S: AsRef<OsStr>>(&mut self, text_to_append_as_is: S) -> &mut Command {
+            self.std.raw_arg(text_to_append_as_is);
+            self
+        }
     }
 
     /// Inserts or updates an environment variable mapping.
@@ -1448,92 +1441,100 @@ impl TryInto<Stdio> for ChildStderr {
 }
 
 #[cfg(unix)]
+#[cfg_attr(docsrs, doc(cfg(unix)))]
 mod sys {
-    #[cfg(not(tokio_no_as_fd))]
-    use std::os::unix::io::{AsFd, BorrowedFd};
-    use std::os::unix::io::{AsRawFd, RawFd};
+    use std::{
+        io,
+        os::unix::io::{AsFd, AsRawFd, BorrowedFd, OwnedFd, RawFd},
+    };
 
     use super::{ChildStderr, ChildStdin, ChildStdout};
 
-    impl AsRawFd for ChildStdin {
-        fn as_raw_fd(&self) -> RawFd {
-            self.inner.as_raw_fd()
-        }
+    macro_rules! impl_traits {
+        ($type:ty) => {
+            impl $type {
+                /// Convert into [`OwnedFd`].
+                pub fn into_owned_fd(self) -> io::Result<OwnedFd> {
+                    self.inner.into_owned_fd()
+                }
+            }
+
+            impl AsRawFd for $type {
+                fn as_raw_fd(&self) -> RawFd {
+                    self.inner.as_raw_fd()
+                }
+            }
+
+            impl AsFd for $type {
+                fn as_fd(&self) -> BorrowedFd<'_> {
+                    unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
+                }
+            }
+        };
     }
 
-    #[cfg(not(tokio_no_as_fd))]
-    impl AsFd for ChildStdin {
-        fn as_fd(&self) -> BorrowedFd<'_> {
-            unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
-        }
-    }
-
-    impl AsRawFd for ChildStdout {
-        fn as_raw_fd(&self) -> RawFd {
-            self.inner.as_raw_fd()
-        }
-    }
-
-    #[cfg(not(tokio_no_as_fd))]
-    impl AsFd for ChildStdout {
-        fn as_fd(&self) -> BorrowedFd<'_> {
-            unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
-        }
-    }
-
-    impl AsRawFd for ChildStderr {
-        fn as_raw_fd(&self) -> RawFd {
-            self.inner.as_raw_fd()
-        }
-    }
-
-    #[cfg(not(tokio_no_as_fd))]
-    impl AsFd for ChildStderr {
-        fn as_fd(&self) -> BorrowedFd<'_> {
-            unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
-        }
-    }
+    impl_traits!(ChildStdin);
+    impl_traits!(ChildStdout);
+    impl_traits!(ChildStderr);
 }
 
-cfg_windows! {
-    impl AsRawHandle for ChildStdin {
-        fn as_raw_handle(&self) -> RawHandle {
-            self.inner.as_raw_handle()
-        }
+#[cfg(any(windows, docsrs))]
+#[cfg_attr(docsrs, doc(cfg(windows)))]
+mod windows {
+    use super::*;
+    use crate::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle, OwnedHandle, RawHandle};
+
+    #[cfg(not(docsrs))]
+    macro_rules! impl_traits {
+        ($type:ty) => {
+            impl $type {
+                /// Convert into [`OwnedHandle`].
+                pub fn into_owned_handle(self) -> io::Result<OwnedHandle> {
+                    self.inner.into_owned_handle()
+                }
+            }
+
+            impl AsRawHandle for $type {
+                fn as_raw_handle(&self) -> RawHandle {
+                    self.inner.as_raw_handle()
+                }
+            }
+
+            impl AsHandle for $type {
+                fn as_handle(&self) -> BorrowedHandle<'_> {
+                    unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }
+                }
+            }
+        };
     }
 
-    #[cfg(not(tokio_no_as_fd))]
-    impl AsHandle for ChildStdin {
-        fn as_handle(&self) -> BorrowedHandle<'_> {
-            unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }
-        }
+    #[cfg(docsrs)]
+    macro_rules! impl_traits {
+        ($type:ty) => {
+            impl $type {
+                /// Convert into [`OwnedHandle`].
+                pub fn into_owned_handle(self) -> io::Result<OwnedHandle> {
+                    todo!("For doc generation only")
+                }
+            }
+
+            impl AsRawHandle for $type {
+                fn as_raw_handle(&self) -> RawHandle {
+                    todo!("For doc generation only")
+                }
+            }
+
+            impl AsHandle for $type {
+                fn as_handle(&self) -> BorrowedHandle<'_> {
+                    todo!("For doc generation only")
+                }
+            }
+        };
     }
 
-    impl AsRawHandle for ChildStdout {
-        fn as_raw_handle(&self) -> RawHandle {
-            self.inner.as_raw_handle()
-        }
-    }
-
-    #[cfg(not(tokio_no_as_fd))]
-    impl AsHandle for ChildStdout {
-        fn as_handle(&self) -> BorrowedHandle<'_> {
-            unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }
-        }
-    }
-
-    impl AsRawHandle for ChildStderr {
-        fn as_raw_handle(&self) -> RawHandle {
-            self.inner.as_raw_handle()
-        }
-    }
-
-    #[cfg(not(tokio_no_as_fd))]
-    impl AsHandle for ChildStderr {
-        fn as_handle(&self) -> BorrowedHandle<'_> {
-            unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) }
-        }
-    }
+    impl_traits!(ChildStdin);
+    impl_traits!(ChildStdout);
+    impl_traits!(ChildStderr);
 }
 
 #[cfg(all(test, not(loom)))]

tokio/src/process/unix/mod.rs

@@ -39,9 +39,7 @@ use std::fmt;
 use std::fs::File;
 use std::future::Future;
 use std::io;
-#[cfg(not(tokio_no_as_fd))]
-use std::os::unix::io::{AsFd, BorrowedFd};
-use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd};
 use std::pin::Pin;
 use std::process::{Child as StdChild, ExitStatus, Stdio};
 use std::task::Context;
@@ -196,14 +194,13 @@ impl AsRawFd for Pipe {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for Pipe {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }
     }
 }
 
-pub(crate) fn convert_to_stdio(io: ChildStdio) -> io::Result<Stdio> {
+fn convert_to_blocking_file(io: ChildStdio) -> io::Result<File> {
     let mut fd = io.inner.into_inner()?.fd;
 
     // Ensure that the fd to be inherited is set to *blocking* mode, as this
@@ -212,7 +209,11 @@ pub(crate) fn convert_to_stdio(io: ChildStdio) -> io::Result<Stdio> {
     // change it to nonblocking mode.
     set_nonblocking(&mut fd, false)?;
 
-    Ok(Stdio::from(fd))
+    Ok(fd)
+}
+
+pub(crate) fn convert_to_stdio(io: ChildStdio) -> io::Result<Stdio> {
+    convert_to_blocking_file(io).map(Stdio::from)
 }
 
 impl Source for Pipe {
@@ -243,6 +244,12 @@ pub(crate) struct ChildStdio {
     inner: PollEvented<Pipe>,
 }
 
+impl ChildStdio {
+    pub(super) fn into_owned_fd(self) -> io::Result<OwnedFd> {
+        convert_to_blocking_file(self).map(OwnedFd::from)
+    }
+}
+
 impl fmt::Debug for ChildStdio {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         self.inner.fmt(fmt)
@@ -255,7 +262,6 @@ impl AsRawFd for ChildStdio {
     }
 }
 
-#[cfg(not(tokio_no_as_fd))]
 impl AsFd for ChildStdio {
     fn as_fd(&self) -> BorrowedFd<'_> {
         unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) }

tokio/src/process/windows.rs

@@ -24,7 +24,7 @@ use std::fmt;
 use std::fs::File as StdFile;
 use std::future::Future;
 use std::io;
-use std::os::windows::prelude::{AsRawHandle, IntoRawHandle, RawHandle};
+use std::os::windows::prelude::{AsRawHandle, IntoRawHandle, OwnedHandle, RawHandle};
 use std::pin::Pin;
 use std::process::Stdio;
 use std::process::{Child as StdChild, Command as StdCommand, ExitStatus};
@@ -195,6 +195,12 @@ pub(crate) struct ChildStdio {
     io: Blocking<ArcFile>,
 }
 
+impl ChildStdio {
+    pub(super) fn into_owned_handle(self) -> io::Result<OwnedHandle> {
+        convert_to_file(self).map(OwnedHandle::from)
+    }
+}
+
 impl AsRawHandle for ChildStdio {
     fn as_raw_handle(&self) -> RawHandle {
         self.raw.as_raw_handle()
@@ -240,13 +246,15 @@ where
     Ok(ChildStdio { raw, io })
 }
 
-pub(crate) fn convert_to_stdio(child_stdio: ChildStdio) -> io::Result<Stdio> {
+fn convert_to_file(child_stdio: ChildStdio) -> io::Result<StdFile> {
     let ChildStdio { raw, io } = child_stdio;
     drop(io); // Try to drop the Arc count here
 
-    Arc::try_unwrap(raw)
-        .or_else(|raw| duplicate_handle(&*raw))
-        .map(Stdio::from)
+    Arc::try_unwrap(raw).or_else(|raw| duplicate_handle(&*raw))
+}
+
+pub(crate) fn convert_to_stdio(child_stdio: ChildStdio) -> io::Result<Stdio> {
+    convert_to_file(child_stdio).map(Stdio::from)
 }
 
 fn duplicate_handle<T: AsRawHandle>(io: &T) -> io::Result<StdFile> {

tokio/src/runtime/blocking/pool.rs

@@ -173,7 +173,7 @@ const KEEP_ALIVE: Duration = Duration::from_secs(10);
 /// Tasks will be scheduled as non-mandatory, meaning they may not get executed
 /// in case of runtime shutdown.
 #[track_caller]
-#[cfg_attr(tokio_wasi, allow(dead_code))]
+#[cfg_attr(target_os = "wasi", allow(dead_code))]
 pub(crate) fn spawn_blocking<F, R>(func: F) -> JoinHandle<R>
 where
     F: FnOnce() -> R + Send + 'static,

tokio/src/runtime/blocking/schedule.rs

@@ -23,8 +23,10 @@ impl BlockingSchedule {
                 scheduler::Handle::CurrentThread(handle) => {
                     handle.driver.clock.inhibit_auto_advance();
                 }
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 scheduler::Handle::MultiThread(_) => {}
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                scheduler::Handle::MultiThreadAlt(_) => {}
             }
         }
         BlockingSchedule {
@@ -43,8 +45,10 @@ impl task::Schedule for BlockingSchedule {
                     handle.driver.clock.allow_auto_advance();
                     handle.driver.unpark();
                 }
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 scheduler::Handle::MultiThread(_) => {}
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                scheduler::Handle::MultiThreadAlt(_) => {}
             }
         }
         None

tokio/src/runtime/builder.rs

@@ -93,6 +93,8 @@ pub struct Builder {
     /// How many ticks before yielding to the driver for timer and I/O events?
     pub(super) event_interval: u32,
 
+    pub(super) local_queue_capacity: usize,
+
     /// When true, the multi-threade scheduler LIFO slot should not be used.
     ///
     /// This option should only be exposed as unstable.
@@ -197,8 +199,10 @@ pub(crate) type ThreadNameFn = std::sync::Arc<dyn Fn() -> String + Send + Sync +
 #[derive(Clone, Copy)]
 pub(crate) enum Kind {
     CurrentThread,
-    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+    #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
     MultiThread,
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+    MultiThreadAlt,
 }
 
 impl Builder {
@@ -230,6 +234,26 @@ impl Builder {
             // The number `61` is fairly arbitrary. I believe this value was copied from golang.
             Builder::new(Kind::MultiThread, 61)
         }
+
+        cfg_unstable! {
+            /// Returns a new builder with the alternate multi thread scheduler
+            /// selected.
+            ///
+            /// The alternate multi threaded scheduler is an in-progress
+            /// candidate to replace the existing multi threaded scheduler. It
+            /// currently does not scale as well to 16+ processors.
+            ///
+            /// This runtime flavor is currently **not considered production
+            /// ready**.
+            ///
+            /// Configuration methods can be chained on the return value.
+            #[cfg(feature = "rt-multi-thread")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
+            pub fn new_multi_thread_alt() -> Builder {
+                // The number `61` is fairly arbitrary. I believe this value was copied from golang.
+                Builder::new(Kind::MultiThreadAlt, 61)
+            }
+        }
     }
 
     /// Returns a new runtime builder initialized with default configuration
@@ -275,6 +299,12 @@ impl Builder {
             global_queue_interval: None,
             event_interval,
 
+            #[cfg(not(loom))]
+            local_queue_capacity: 256,
+
+            #[cfg(loom)]
+            local_queue_capacity: 4,
+
             seed_generator: RngSeedGenerator::new(RngSeed::new()),
 
             #[cfg(tokio_unstable)]
@@ -374,9 +404,17 @@ impl Builder {
     /// Specifies the limit for additional threads spawned by the Runtime.
     ///
     /// These threads are used for blocking operations like tasks spawned
-    /// through [`spawn_blocking`]. Unlike the [`worker_threads`], they are not
-    /// always active and will exit if left idle for too long. You can change
-    /// this timeout duration with [`thread_keep_alive`].
+    /// through [`spawn_blocking`], this includes but is not limited to:
+    /// - [`fs`] operations
+    /// - dns resolution through [`ToSocketAddrs`]
+    /// - writing to [`Stdout`] or [`Stderr`]
+    /// - reading from [`Stdin`]
+    ///
+    /// Unlike the [`worker_threads`], they are not always active and will exit
+    /// if left idle for too long. You can change this timeout duration with [`thread_keep_alive`].
+    ///
+    /// It's recommended to not set this limit too low in order to avoid hanging on operations
+    /// requiring [`spawn_blocking`].
     ///
     /// The default value is 512.
     ///
@@ -390,6 +428,11 @@ impl Builder {
     /// current `max_blocking_threads` does not include async worker threads in the count.
     ///
     /// [`spawn_blocking`]: fn@crate::task::spawn_blocking
+    /// [`fs`]: mod@crate::fs
+    /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs
+    /// [`Stdout`]: struct@crate::io::Stdout
+    /// [`Stdin`]: struct@crate::io::Stdin
+    /// [`Stderr`]: struct@crate::io::Stderr
     /// [`worker_threads`]: Self::worker_threads
     /// [`thread_keep_alive`]: Self::thread_keep_alive
     #[track_caller]
@@ -654,8 +697,10 @@ impl Builder {
     pub fn build(&mut self) -> io::Result<Runtime> {
         match &self.kind {
             Kind::CurrentThread => self.build_current_thread_runtime(),
-            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             Kind::MultiThread => self.build_threaded_runtime(),
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+            Kind::MultiThreadAlt => self.build_alt_threaded_runtime(),
         }
     }
 
@@ -663,8 +708,10 @@ impl Builder {
         driver::Cfg {
             enable_pause_time: match self.kind {
                 Kind::CurrentThread => true,
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 Kind::MultiThread => false,
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                Kind::MultiThreadAlt => false,
             },
             enable_io: self.enable_io,
             enable_time: self.enable_time,
@@ -1020,6 +1067,14 @@ impl Builder {
         }
     }
 
+    cfg_loom! {
+        pub(crate) fn local_queue_capacity(&mut self, value: usize) -> &mut Self {
+            assert!(value.is_power_of_two());
+            self.local_queue_capacity = value;
+            self
+        }
+    }
+
     fn build_current_thread_runtime(&mut self) -> io::Result<Runtime> {
         use crate::runtime::scheduler::{self, CurrentThread};
         use crate::runtime::{runtime::Scheduler, Config};
@@ -1048,6 +1103,7 @@ impl Builder {
                 after_unpark: self.after_unpark.clone(),
                 global_queue_interval: self.global_queue_interval,
                 event_interval: self.event_interval,
+                local_queue_capacity: self.local_queue_capacity,
                 #[cfg(tokio_unstable)]
                 unhandled_panic: self.unhandled_panic.clone(),
                 disable_lifo_slot: self.disable_lifo_slot,
@@ -1198,6 +1254,7 @@ cfg_rt_multi_thread! {
                     after_unpark: self.after_unpark.clone(),
                     global_queue_interval: self.global_queue_interval,
                     event_interval: self.event_interval,
+                    local_queue_capacity: self.local_queue_capacity,
                     #[cfg(tokio_unstable)]
                     unhandled_panic: self.unhandled_panic.clone(),
                     disable_lifo_slot: self.disable_lifo_slot,
@@ -1214,6 +1271,49 @@ cfg_rt_multi_thread! {
 
             Ok(Runtime::from_parts(Scheduler::MultiThread(scheduler), handle, blocking_pool))
         }
+
+        cfg_unstable! {
+            fn build_alt_threaded_runtime(&mut self) -> io::Result<Runtime> {
+                use crate::loom::sys::num_cpus;
+                use crate::runtime::{Config, runtime::Scheduler};
+                use crate::runtime::scheduler::MultiThreadAlt;
+
+                let core_threads = self.worker_threads.unwrap_or_else(num_cpus);
+
+                let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?;
+
+                // Create the blocking pool
+                let blocking_pool =
+                    blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads);
+                let blocking_spawner = blocking_pool.spawner().clone();
+
+                // Generate a rng seed for this runtime.
+                let seed_generator_1 = self.seed_generator.next_generator();
+                let seed_generator_2 = self.seed_generator.next_generator();
+
+                let (scheduler, handle) = MultiThreadAlt::new(
+                    core_threads,
+                    driver,
+                    driver_handle,
+                    blocking_spawner,
+                    seed_generator_2,
+                    Config {
+                        before_park: self.before_park.clone(),
+                        after_unpark: self.after_unpark.clone(),
+                        global_queue_interval: self.global_queue_interval,
+                        event_interval: self.event_interval,
+                        local_queue_capacity: self.local_queue_capacity,
+                        #[cfg(tokio_unstable)]
+                        unhandled_panic: self.unhandled_panic.clone(),
+                        disable_lifo_slot: self.disable_lifo_slot,
+                        seed_generator: seed_generator_1,
+                        metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
+                    },
+                );
+
+                Ok(Runtime::from_parts(Scheduler::MultiThreadAlt(scheduler), handle, blocking_pool))
+            }
+        }
     }
 }
 

tokio/src/runtime/config.rs

@@ -1,4 +1,7 @@
-#![cfg_attr(any(not(feature = "full"), tokio_wasm), allow(dead_code))]
+#![cfg_attr(
+    any(not(all(tokio_unstable, feature = "full")), target_family = "wasm"),
+    allow(dead_code)
+)]
 use crate::runtime::Callback;
 use crate::util::RngSeedGenerator;
 
@@ -9,6 +12,9 @@ pub(crate) struct Config {
     /// How many ticks before yielding to the driver for timer and I/O events?
     pub(crate) event_interval: u32,
 
+    /// How big to make each worker's local queue
+    pub(crate) local_queue_capacity: usize,
+
     /// Callback for a worker parking itself
     pub(crate) before_park: Option<Callback>,
 

tokio/src/runtime/coop.rs

@@ -246,7 +246,7 @@ cfg_coop! {
 mod test {
     use super::*;
 
-    #[cfg(tokio_wasm_not_wasi)]
+    #[cfg(all(target_family = "wasm", not(target_os = "wasi")))]
     use wasm_bindgen_test::wasm_bindgen_test as test;
 
     fn get() -> Budget {

tokio/src/runtime/driver.rs

@@ -2,7 +2,10 @@
 
 // Eventually, this file will see significant refactoring / cleanup. For now, we
 // don't need to worry much about dead code with certain feature permutations.
-#![cfg_attr(not(feature = "full"), allow(dead_code))]
+#![cfg_attr(
+    any(not(all(tokio_unstable, feature = "full")), target_family = "wasm"),
+    allow(dead_code)
+)]
 
 use crate::runtime::park::{ParkThread, UnparkThread};
 
@@ -58,6 +61,10 @@ impl Driver {
         ))
     }
 
+    pub(crate) fn is_enabled(&self) -> bool {
+        self.inner.is_enabled()
+    }
+
     pub(crate) fn park(&mut self, handle: &Handle) {
         self.inner.park(handle)
     }
@@ -154,6 +161,13 @@ cfg_io_driver! {
     }
 
     impl IoStack {
+        pub(crate) fn is_enabled(&self) -> bool {
+            match self {
+                IoStack::Enabled(..) => true,
+                IoStack::Disabled(..) => false,
+            }
+        }
+
         pub(crate) fn park(&mut self, handle: &Handle) {
             match self {
                 IoStack::Enabled(v) => v.park(handle),
@@ -217,6 +231,11 @@ cfg_not_io_driver! {
         pub(crate) fn shutdown(&mut self, _handle: &Handle) {
             self.0.shutdown();
         }
+
+        /// This is not a "real" driver, so it is not considered enabled.
+        pub(crate) fn is_enabled(&self) -> bool {
+            false
+        }
     }
 }
 
@@ -298,6 +317,13 @@ cfg_time! {
     }
 
     impl TimeDriver {
+        pub(crate) fn is_enabled(&self) -> bool {
+            match self {
+                TimeDriver::Enabled { .. } => true,
+                TimeDriver::Disabled(inner) => inner.is_enabled(),
+            }
+        }
+
         pub(crate) fn park(&mut self, handle: &Handle) {
             match self {
                 TimeDriver::Enabled { driver, .. } => driver.park(handle),

tokio/src/runtime/handle.rs

@@ -1,3 +1,5 @@
+#[cfg(tokio_unstable)]
+use crate::runtime;
 use crate::runtime::{context, scheduler, RuntimeFlavor};
 
 /// Handle to the runtime.
@@ -353,8 +355,42 @@ impl Handle {
     pub fn runtime_flavor(&self) -> RuntimeFlavor {
         match self.inner {
             scheduler::Handle::CurrentThread(_) => RuntimeFlavor::CurrentThread,
-            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             scheduler::Handle::MultiThread(_) => RuntimeFlavor::MultiThread,
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+            scheduler::Handle::MultiThreadAlt(_) => RuntimeFlavor::MultiThreadAlt,
+        }
+    }
+
+    cfg_unstable! {
+        /// Returns the [`Id`] of the current `Runtime`.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::Handle;
+        ///
+        /// #[tokio::main(flavor = "current_thread")]
+        /// async fn main() {
+        ///   println!("Current runtime id: {}", Handle::current().id());
+        /// }
+        /// ```
+        ///
+        /// **Note**: This is an [unstable API][unstable]. The public API of this type
+        /// may break in 1.x releases. See [the documentation on unstable
+        /// features][unstable] for details.
+        ///
+        /// [unstable]: crate#unstable-features
+        /// [`Id`]: struct@crate::runtime::Id
+        pub fn id(&self) -> runtime::Id {
+            let owned_id = match &self.inner {
+                scheduler::Handle::CurrentThread(handle) => handle.owned_id(),
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
+                scheduler::Handle::MultiThread(handle) => handle.owned_id(),
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                scheduler::Handle::MultiThreadAlt(handle) => handle.owned_id(),
+            };
+            owned_id.into()
         }
     }
 }
@@ -493,7 +529,7 @@ cfg_taskdump! {
         pub async fn dump(&self) -> crate::runtime::Dump {
             match &self.inner {
                 scheduler::Handle::CurrentThread(handle) => handle.dump(),
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 scheduler::Handle::MultiThread(handle) => {
                     // perform the trace in a separate thread so that the
                     // trace itself does not appear in the taskdump.
@@ -503,6 +539,8 @@ cfg_taskdump! {
                         handle.dump().await
                     }).await
                 },
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                scheduler::Handle::MultiThreadAlt(_) => panic!("task dump not implemented for this runtime flavor"),
             }
         }
     }

tokio/src/runtime/id.rs

@@ -0,0 +1,46 @@
+use std::fmt;
+use std::num::NonZeroU64;
+
+/// An opaque ID that uniquely identifies a runtime relative to all other currently
+/// running runtimes.
+///
+/// # Notes
+///
+/// - Runtime IDs are unique relative to other *currently running* runtimes.
+///   When a runtime completes, the same ID may be used for another runtime.
+/// - Runtime IDs are *not* sequential, and do not indicate the order in which
+///   runtimes are started or any other data.
+/// - The runtime ID of the currently running task can be obtained from the
+///   Handle.
+///
+/// # Examples
+///
+/// ```
+/// use tokio::runtime::Handle;
+///
+/// #[tokio::main(flavor = "multi_thread", worker_threads = 4)]
+/// async fn main() {
+///   println!("Current runtime id: {}", Handle::current().id());
+/// }
+/// ```
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [unstable]: crate#unstable-features
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
+pub struct Id(NonZeroU64);
+
+impl From<NonZeroU64> for Id {
+    fn from(value: NonZeroU64) -> Self {
+        Id(value)
+    }
+}
+
+impl fmt::Display for Id {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}

tokio/src/runtime/io/driver.rs

@@ -0,0 +1,280 @@
+// Signal handling
+cfg_signal_internal_and_unix! {
+    mod signal;
+}
+
+use crate::io::interest::Interest;
+use crate::io::ready::Ready;
+use crate::loom::sync::Mutex;
+use crate::runtime::driver;
+use crate::runtime::io::registration_set;
+use crate::runtime::io::{IoDriverMetrics, RegistrationSet, ScheduledIo};
+
+use mio::event::Source;
+use std::fmt;
+use std::io;
+use std::sync::Arc;
+use std::time::Duration;
+
+/// I/O driver, backed by Mio.
+pub(crate) struct Driver {
+    /// Tracks the number of times `turn` is called. It is safe for this to wrap
+    /// as it is mostly used to determine when to call `compact()`.
+    tick: u8,
+
+    /// True when an event with the signal token is received
+    signal_ready: bool,
+
+    /// Reuse the `mio::Events` value across calls to poll.
+    events: mio::Events,
+
+    /// The system event queue.
+    poll: mio::Poll,
+}
+
+/// A reference to an I/O driver.
+pub(crate) struct Handle {
+    /// Registers I/O resources.
+    registry: mio::Registry,
+
+    /// Tracks all registrations
+    registrations: RegistrationSet,
+
+    /// State that should be synchronized
+    synced: Mutex<registration_set::Synced>,
+
+    /// Used to wake up the reactor from a call to `turn`.
+    /// Not supported on Wasi due to lack of threading support.
+    #[cfg(not(target_os = "wasi"))]
+    waker: mio::Waker,
+
+    pub(crate) metrics: IoDriverMetrics,
+}
+
+#[derive(Debug)]
+pub(crate) struct ReadyEvent {
+    pub(super) tick: u8,
+    pub(crate) ready: Ready,
+    pub(super) is_shutdown: bool,
+}
+
+cfg_net_unix!(
+    impl ReadyEvent {
+        pub(crate) fn with_ready(&self, ready: Ready) -> Self {
+            Self {
+                ready,
+                tick: self.tick,
+                is_shutdown: self.is_shutdown,
+            }
+        }
+    }
+);
+
+#[derive(Debug, Eq, PartialEq, Clone, Copy)]
+pub(super) enum Direction {
+    Read,
+    Write,
+}
+
+pub(super) enum Tick {
+    Set(u8),
+    Clear(u8),
+}
+
+const TOKEN_WAKEUP: mio::Token = mio::Token(0);
+const TOKEN_SIGNAL: mio::Token = mio::Token(1);
+
+fn _assert_kinds() {
+    fn _assert<T: Send + Sync>() {}
+
+    _assert::<Handle>();
+}
+
+// ===== impl Driver =====
+
+impl Driver {
+    /// Creates a new event loop, returning any error that happened during the
+    /// creation.
+    pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> {
+        let poll = mio::Poll::new()?;
+        #[cfg(not(target_os = "wasi"))]
+        let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
+        let registry = poll.registry().try_clone()?;
+
+        let driver = Driver {
+            tick: 0,
+            signal_ready: false,
+            events: mio::Events::with_capacity(nevents),
+            poll,
+        };
+
+        let (registrations, synced) = RegistrationSet::new();
+
+        let handle = Handle {
+            registry,
+            registrations,
+            synced: Mutex::new(synced),
+            #[cfg(not(target_os = "wasi"))]
+            waker,
+            metrics: IoDriverMetrics::default(),
+        };
+
+        Ok((driver, handle))
+    }
+
+    pub(crate) fn park(&mut self, rt_handle: &driver::Handle) {
+        let handle = rt_handle.io();
+        self.turn(handle, None);
+    }
+
+    pub(crate) fn park_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) {
+        let handle = rt_handle.io();
+        self.turn(handle, Some(duration));
+    }
+
+    pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) {
+        let handle = rt_handle.io();
+        let ios = handle.registrations.shutdown(&mut handle.synced.lock());
+
+        // `shutdown()` must be called without holding the lock.
+        for io in ios {
+            io.shutdown();
+        }
+    }
+
+    fn turn(&mut self, handle: &Handle, max_wait: Option<Duration>) {
+        debug_assert!(!handle.registrations.is_shutdown(&handle.synced.lock()));
+
+        self.tick = self.tick.wrapping_add(1);
+
+        handle.release_pending_registrations();
+
+        let events = &mut self.events;
+
+        // Block waiting for an event to happen, peeling out how many events
+        // happened.
+        match self.poll.poll(events, max_wait) {
+            Ok(_) => {}
+            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
+            #[cfg(target_os = "wasi")]
+            Err(e) if e.kind() == io::ErrorKind::InvalidInput => {
+                // In case of wasm32_wasi this error happens, when trying to poll without subscriptions
+                // just return from the park, as there would be nothing, which wakes us up.
+            }
+            Err(e) => panic!("unexpected error when polling the I/O driver: {:?}", e),
+        }
+
+        // Process all the events that came in, dispatching appropriately
+        let mut ready_count = 0;
+        for event in events.iter() {
+            let token = event.token();
+
+            if token == TOKEN_WAKEUP {
+                // Nothing to do, the event is used to unblock the I/O driver
+            } else if token == TOKEN_SIGNAL {
+                self.signal_ready = true;
+            } else {
+                let ready = Ready::from_mio(event);
+                // Use std::ptr::from_exposed_addr when stable
+                let ptr: *const ScheduledIo = token.0 as *const _;
+
+                // Safety: we ensure that the pointers used as tokens are not freed
+                // until they are both deregistered from mio **and** we know the I/O
+                // driver is not concurrently polling. The I/O driver holds ownership of
+                // an `Arc<ScheduledIo>` so we can safely cast this to a ref.
+                let io: &ScheduledIo = unsafe { &*ptr };
+
+                io.set_readiness(Tick::Set(self.tick), |curr| curr | ready);
+                io.wake(ready);
+
+                ready_count += 1;
+            }
+        }
+
+        handle.metrics.incr_ready_count_by(ready_count);
+    }
+}
+
+impl fmt::Debug for Driver {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Driver")
+    }
+}
+
+impl Handle {
+    /// Forces a reactor blocked in a call to `turn` to wakeup, or otherwise
+    /// makes the next call to `turn` return immediately.
+    ///
+    /// This method is intended to be used in situations where a notification
+    /// needs to otherwise be sent to the main reactor. If the reactor is
+    /// currently blocked inside of `turn` then it will wake up and soon return
+    /// after this method has been called. If the reactor is not currently
+    /// blocked in `turn`, then the next call to `turn` will not block and
+    /// return immediately.
+    pub(crate) fn unpark(&self) {
+        #[cfg(not(target_os = "wasi"))]
+        self.waker.wake().expect("failed to wake I/O driver");
+    }
+
+    /// Registers an I/O resource with the reactor for a given `mio::Ready` state.
+    ///
+    /// The registration token is returned.
+    pub(super) fn add_source(
+        &self,
+        source: &mut impl mio::event::Source,
+        interest: Interest,
+    ) -> io::Result<Arc<ScheduledIo>> {
+        let scheduled_io = self.registrations.allocate(&mut self.synced.lock())?;
+        let token = scheduled_io.token();
+
+        // TODO: if this returns an err, the `ScheduledIo` leaks...
+        self.registry.register(source, token, interest.to_mio())?;
+
+        // TODO: move this logic to `RegistrationSet` and use a `CountedLinkedList`
+        self.metrics.incr_fd_count();
+
+        Ok(scheduled_io)
+    }
+
+    /// Deregisters an I/O resource from the reactor.
+    pub(super) fn deregister_source(
+        &self,
+        registration: &Arc<ScheduledIo>,
+        source: &mut impl Source,
+    ) -> io::Result<()> {
+        // Deregister the source with the OS poller **first**
+        self.registry.deregister(source)?;
+
+        if self
+            .registrations
+            .deregister(&mut self.synced.lock(), registration)
+        {
+            self.unpark();
+        }
+
+        self.metrics.dec_fd_count();
+
+        Ok(())
+    }
+
+    fn release_pending_registrations(&self) {
+        if self.registrations.needs_release() {
+            self.registrations.release(&mut self.synced.lock());
+        }
+    }
+}
+
+impl fmt::Debug for Handle {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Handle")
+    }
+}
+
+impl Direction {
+    pub(super) fn mask(self) -> Ready {
+        match self {
+            Direction::Read => Ready::READABLE | Ready::READ_CLOSED,
+            Direction::Write => Ready::WRITABLE | Ready::WRITE_CLOSED,
+        }
+    }
+}

tokio/src/runtime/io/driver/signal.rs

@@ -0,0 +1,22 @@
+use super::{Driver, Handle, TOKEN_SIGNAL};
+
+use std::io;
+
+impl Handle {
+    pub(crate) fn register_signal_receiver(
+        &self,
+        receiver: &mut mio::net::UnixStream,
+    ) -> io::Result<()> {
+        self.registry
+            .register(receiver, TOKEN_SIGNAL, mio::Interest::READABLE)?;
+        Ok(())
+    }
+}
+
+impl Driver {
+    pub(crate) fn consume_signal_ready(&mut self) -> bool {
+        let ret = self.signal_ready;
+        self.signal_ready = false;
+        ret
+    }
+}

tokio/src/runtime/io/mod.rs

@@ -1,356 +1,16 @@
 #![cfg_attr(not(all(feature = "rt", feature = "net")), allow(dead_code))]
+mod driver;
+use driver::{Direction, Tick};
+pub(crate) use driver::{Driver, Handle, ReadyEvent};
 
 mod registration;
 pub(crate) use registration::Registration;
 
+mod registration_set;
+use registration_set::RegistrationSet;
+
 mod scheduled_io;
 use scheduled_io::ScheduledIo;
 
 mod metrics;
-
-use crate::io::interest::Interest;
-use crate::io::ready::Ready;
-use crate::runtime::driver;
-use crate::util::slab::{self, Slab};
-use crate::{loom::sync::RwLock, util::bit};
-
 use metrics::IoDriverMetrics;
-
-use std::fmt;
-use std::io;
-use std::time::Duration;
-
-/// I/O driver, backed by Mio.
-pub(crate) struct Driver {
-    /// Tracks the number of times `turn` is called. It is safe for this to wrap
-    /// as it is mostly used to determine when to call `compact()`.
-    tick: u8,
-
-    /// True when an event with the signal token is received
-    signal_ready: bool,
-
-    /// Reuse the `mio::Events` value across calls to poll.
-    events: mio::Events,
-
-    /// Primary slab handle containing the state for each resource registered
-    /// with this driver.
-    resources: Slab<ScheduledIo>,
-
-    /// The system event queue.
-    poll: mio::Poll,
-}
-
-/// A reference to an I/O driver.
-pub(crate) struct Handle {
-    /// Registers I/O resources.
-    registry: mio::Registry,
-
-    /// Allocates `ScheduledIo` handles when creating new resources.
-    io_dispatch: RwLock<IoDispatcher>,
-
-    /// Used to wake up the reactor from a call to `turn`.
-    /// Not supported on Wasi due to lack of threading support.
-    #[cfg(not(tokio_wasi))]
-    waker: mio::Waker,
-
-    pub(crate) metrics: IoDriverMetrics,
-}
-
-#[derive(Debug)]
-pub(crate) struct ReadyEvent {
-    tick: u8,
-    pub(crate) ready: Ready,
-    is_shutdown: bool,
-}
-
-cfg_net_unix!(
-    impl ReadyEvent {
-        pub(crate) fn with_ready(&self, ready: Ready) -> Self {
-            Self {
-                ready,
-                tick: self.tick,
-                is_shutdown: self.is_shutdown,
-            }
-        }
-    }
-);
-
-struct IoDispatcher {
-    allocator: slab::Allocator<ScheduledIo>,
-    is_shutdown: bool,
-}
-
-#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-enum Direction {
-    Read,
-    Write,
-}
-
-enum Tick {
-    Set(u8),
-    Clear(u8),
-}
-
-// TODO: Don't use a fake token. Instead, reserve a slot entry for the wakeup
-// token.
-const TOKEN_WAKEUP: mio::Token = mio::Token(1 << 31);
-const TOKEN_SIGNAL: mio::Token = mio::Token(1 + (1 << 31));
-
-const ADDRESS: bit::Pack = bit::Pack::least_significant(24);
-
-// Packs the generation value in the `readiness` field.
-//
-// The generation prevents a race condition where a slab slot is reused for a
-// new socket while the I/O driver is about to apply a readiness event. The
-// generation value is checked when setting new readiness. If the generation do
-// not match, then the readiness event is discarded.
-const GENERATION: bit::Pack = ADDRESS.then(7);
-
-fn _assert_kinds() {
-    fn _assert<T: Send + Sync>() {}
-
-    _assert::<Handle>();
-}
-
-// ===== impl Driver =====
-
-impl Driver {
-    /// Creates a new event loop, returning any error that happened during the
-    /// creation.
-    pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> {
-        let poll = mio::Poll::new()?;
-        #[cfg(not(tokio_wasi))]
-        let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
-        let registry = poll.registry().try_clone()?;
-
-        let slab = Slab::new();
-        let allocator = slab.allocator();
-
-        let driver = Driver {
-            tick: 0,
-            signal_ready: false,
-            events: mio::Events::with_capacity(nevents),
-            poll,
-            resources: slab,
-        };
-
-        let handle = Handle {
-            registry,
-            io_dispatch: RwLock::new(IoDispatcher::new(allocator)),
-            #[cfg(not(tokio_wasi))]
-            waker,
-            metrics: IoDriverMetrics::default(),
-        };
-
-        Ok((driver, handle))
-    }
-
-    pub(crate) fn park(&mut self, rt_handle: &driver::Handle) {
-        let handle = rt_handle.io();
-        self.turn(handle, None);
-    }
-
-    pub(crate) fn park_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) {
-        let handle = rt_handle.io();
-        self.turn(handle, Some(duration));
-    }
-
-    pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) {
-        let handle = rt_handle.io();
-
-        if handle.shutdown() {
-            self.resources.for_each(|io| {
-                // If a task is waiting on the I/O resource, notify it that the
-                // runtime is being shutdown. And shutdown will clear all wakers.
-                io.shutdown();
-            });
-        }
-    }
-
-    fn turn(&mut self, handle: &Handle, max_wait: Option<Duration>) {
-        // How often to call `compact()` on the resource slab
-        const COMPACT_INTERVAL: u8 = 255;
-
-        self.tick = self.tick.wrapping_add(1);
-
-        if self.tick == COMPACT_INTERVAL {
-            self.resources.compact()
-        }
-
-        let events = &mut self.events;
-
-        // Block waiting for an event to happen, peeling out how many events
-        // happened.
-        match self.poll.poll(events, max_wait) {
-            Ok(_) => {}
-            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
-            #[cfg(tokio_wasi)]
-            Err(e) if e.kind() == io::ErrorKind::InvalidInput => {
-                // In case of wasm32_wasi this error happens, when trying to poll without subscriptions
-                // just return from the park, as there would be nothing, which wakes us up.
-            }
-            Err(e) => panic!("unexpected error when polling the I/O driver: {:?}", e),
-        }
-
-        // Process all the events that came in, dispatching appropriately
-        let mut ready_count = 0;
-        for event in events.iter() {
-            let token = event.token();
-
-            if token == TOKEN_WAKEUP {
-                // Nothing to do, the event is used to unblock the I/O driver
-            } else if token == TOKEN_SIGNAL {
-                self.signal_ready = true;
-            } else {
-                Self::dispatch(
-                    &mut self.resources,
-                    self.tick,
-                    token,
-                    Ready::from_mio(event),
-                );
-                ready_count += 1;
-            }
-        }
-
-        handle.metrics.incr_ready_count_by(ready_count);
-    }
-
-    fn dispatch(resources: &mut Slab<ScheduledIo>, tick: u8, token: mio::Token, ready: Ready) {
-        let addr = slab::Address::from_usize(ADDRESS.unpack(token.0));
-
-        let io = match resources.get(addr) {
-            Some(io) => io,
-            None => return,
-        };
-
-        let res = io.set_readiness(Some(token.0), Tick::Set(tick), |curr| curr | ready);
-
-        if res.is_err() {
-            // token no longer valid!
-            return;
-        }
-
-        io.wake(ready);
-    }
-}
-
-impl fmt::Debug for Driver {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "Driver")
-    }
-}
-
-impl Handle {
-    /// Forces a reactor blocked in a call to `turn` to wakeup, or otherwise
-    /// makes the next call to `turn` return immediately.
-    ///
-    /// This method is intended to be used in situations where a notification
-    /// needs to otherwise be sent to the main reactor. If the reactor is
-    /// currently blocked inside of `turn` then it will wake up and soon return
-    /// after this method has been called. If the reactor is not currently
-    /// blocked in `turn`, then the next call to `turn` will not block and
-    /// return immediately.
-    pub(crate) fn unpark(&self) {
-        #[cfg(not(tokio_wasi))]
-        self.waker.wake().expect("failed to wake I/O driver");
-    }
-
-    /// Registers an I/O resource with the reactor for a given `mio::Ready` state.
-    ///
-    /// The registration token is returned.
-    pub(super) fn add_source(
-        &self,
-        source: &mut impl mio::event::Source,
-        interest: Interest,
-    ) -> io::Result<slab::Ref<ScheduledIo>> {
-        let (address, shared) = self.allocate()?;
-
-        let token = GENERATION.pack(shared.generation(), ADDRESS.pack(address.as_usize(), 0));
-
-        self.registry
-            .register(source, mio::Token(token), interest.to_mio())?;
-
-        self.metrics.incr_fd_count();
-
-        Ok(shared)
-    }
-
-    /// Deregisters an I/O resource from the reactor.
-    pub(super) fn deregister_source(&self, source: &mut impl mio::event::Source) -> io::Result<()> {
-        self.registry.deregister(source)?;
-
-        self.metrics.dec_fd_count();
-
-        Ok(())
-    }
-
-    /// shutdown the dispatcher.
-    fn shutdown(&self) -> bool {
-        let mut io = self.io_dispatch.write().unwrap();
-        if io.is_shutdown {
-            return false;
-        }
-        io.is_shutdown = true;
-        true
-    }
-
-    fn allocate(&self) -> io::Result<(slab::Address, slab::Ref<ScheduledIo>)> {
-        let io = self.io_dispatch.read().unwrap();
-        if io.is_shutdown {
-            return Err(io::Error::new(
-                io::ErrorKind::Other,
-                crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
-            ));
-        }
-        io.allocator.allocate().ok_or_else(|| {
-            io::Error::new(
-                io::ErrorKind::Other,
-                "reactor at max registered I/O resources",
-            )
-        })
-    }
-}
-
-impl fmt::Debug for Handle {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "Handle")
-    }
-}
-
-// ===== impl IoDispatcher =====
-
-impl IoDispatcher {
-    fn new(allocator: slab::Allocator<ScheduledIo>) -> Self {
-        Self {
-            allocator,
-            is_shutdown: false,
-        }
-    }
-}
-
-impl Direction {
-    pub(super) fn mask(self) -> Ready {
-        match self {
-            Direction::Read => Ready::READABLE | Ready::READ_CLOSED,
-            Direction::Write => Ready::WRITABLE | Ready::WRITE_CLOSED,
-        }
-    }
-}
-
-// Signal handling
-cfg_signal_internal_and_unix! {
-    impl Handle {
-        pub(crate) fn register_signal_receiver(&self, receiver: &mut mio::net::UnixStream) -> io::Result<()> {
-            self.registry.register(receiver, TOKEN_SIGNAL, mio::Interest::READABLE)?;
-            Ok(())
-        }
-    }
-
-    impl Driver {
-        pub(crate) fn consume_signal_ready(&mut self) -> bool {
-            let ret = self.signal_ready;
-            self.signal_ready = false;
-            ret
-        }
-    }
-}

tokio/src/runtime/io/registration.rs

@@ -3,10 +3,10 @@
 use crate::io::interest::Interest;
 use crate::runtime::io::{Direction, Handle, ReadyEvent, ScheduledIo};
 use crate::runtime::scheduler;
-use crate::util::slab;
 
 use mio::event::Source;
 use std::io;
+use std::sync::Arc;
 use std::task::{Context, Poll};
 
 cfg_io_driver! {
@@ -45,10 +45,12 @@ cfg_io_driver! {
     #[derive(Debug)]
     pub(crate) struct Registration {
         /// Handle to the associated runtime.
+        ///
+        /// TODO: this can probably be moved into `ScheduledIo`.
         handle: scheduler::Handle,
 
         /// Reference to state stored by the driver.
-        shared: slab::Ref<ScheduledIo>,
+        shared: Arc<ScheduledIo>,
     }
 }
 
@@ -95,7 +97,7 @@ impl Registration {
     ///
     /// `Err` is returned if an error is encountered.
     pub(crate) fn deregister(&mut self, io: &mut impl Source) -> io::Result<()> {
-        self.handle().deregister_source(io)
+        self.handle().deregister_source(&self.shared, io)
     }
 
     pub(crate) fn clear_readiness(&self, event: ReadyEvent) {
@@ -116,7 +118,7 @@ impl Registration {
 
     // Uses the poll path, requiring the caller to ensure mutual exclusion for
     // correctness. Only the last task to call this function is notified.
-    #[cfg(not(tokio_wasi))]
+    #[cfg(not(target_os = "wasi"))]
     pub(crate) fn poll_read_io<R>(
         &self,
         cx: &mut Context<'_>,
@@ -199,6 +201,33 @@ impl Registration {
         }
     }
 
+    pub(crate) async fn readiness(&self, interest: Interest) -> io::Result<ReadyEvent> {
+        let ev = self.shared.readiness(interest).await;
+
+        if ev.is_shutdown {
+            return Err(gone());
+        }
+
+        Ok(ev)
+    }
+
+    pub(crate) async fn async_io<R>(
+        &self,
+        interest: Interest,
+        mut f: impl FnMut() -> io::Result<R>,
+    ) -> io::Result<R> {
+        loop {
+            let event = self.readiness(interest).await?;
+
+            match f() {
+                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+                    self.clear_readiness(event);
+                }
+                x => return x,
+            }
+        }
+    }
+
     fn handle(&self) -> &Handle {
         self.handle.driver().io()
     }
@@ -223,30 +252,3 @@ fn gone() -> io::Error {
         crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
     )
 }
-
-cfg_io_readiness! {
-    impl Registration {
-        pub(crate) async fn readiness(&self, interest: Interest) -> io::Result<ReadyEvent> {
-            let ev = self.shared.readiness(interest).await;
-
-            if ev.is_shutdown {
-                return Err(gone())
-            }
-
-            Ok(ev)
-        }
-
-        pub(crate) async fn async_io<R>(&self, interest: Interest, mut f: impl FnMut() -> io::Result<R>) -> io::Result<R> {
-            loop {
-                let event = self.readiness(interest).await?;
-
-                match f() {
-                    Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
-                        self.clear_readiness(event);
-                    }
-                    x => return x,
-                }
-            }
-        }
-    }
-}

tokio/src/runtime/io/registration_set.rs

@@ -0,0 +1,134 @@
+use crate::loom::sync::atomic::AtomicUsize;
+use crate::runtime::io::ScheduledIo;
+use crate::util::linked_list::{self, LinkedList};
+
+use std::io;
+use std::ptr::NonNull;
+use std::sync::atomic::Ordering::{Acquire, Release};
+use std::sync::Arc;
+
+pub(super) struct RegistrationSet {
+    num_pending_release: AtomicUsize,
+}
+
+pub(super) struct Synced {
+    // True when the I/O driver shutdown. At this point, no more registrations
+    // should be added to the set.
+    is_shutdown: bool,
+
+    // List of all registrations tracked by the set
+    registrations: LinkedList<Arc<ScheduledIo>, ScheduledIo>,
+
+    // Registrations that are pending drop. When a `Registration` is dropped, it
+    // stores its `ScheduledIo` in this list. The I/O driver is responsible for
+    // dropping it. This ensures the `ScheduledIo` is not freed while it can
+    // still be included in an I/O event.
+    pending_release: Vec<Arc<ScheduledIo>>,
+}
+
+impl RegistrationSet {
+    pub(super) fn new() -> (RegistrationSet, Synced) {
+        let set = RegistrationSet {
+            num_pending_release: AtomicUsize::new(0),
+        };
+
+        let synced = Synced {
+            is_shutdown: false,
+            registrations: LinkedList::new(),
+            pending_release: Vec::with_capacity(16),
+        };
+
+        (set, synced)
+    }
+
+    pub(super) fn is_shutdown(&self, synced: &Synced) -> bool {
+        synced.is_shutdown
+    }
+
+    /// Returns `true` if there are registrations that need to be released
+    pub(super) fn needs_release(&self) -> bool {
+        self.num_pending_release.load(Acquire) != 0
+    }
+
+    pub(super) fn allocate(&self, synced: &mut Synced) -> io::Result<Arc<ScheduledIo>> {
+        if synced.is_shutdown {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
+            ));
+        }
+
+        let ret = Arc::new(ScheduledIo::default());
+
+        // Push a ref into the list of all resources.
+        synced.registrations.push_front(ret.clone());
+
+        Ok(ret)
+    }
+
+    // Returns `true` if the caller should unblock the I/O driver to purge
+    // registrations pending release.
+    pub(super) fn deregister(&self, synced: &mut Synced, registration: &Arc<ScheduledIo>) -> bool {
+        // Kind of arbitrary, but buffering 16 `ScheduledIo`s doesn't seem like much
+        const NOTIFY_AFTER: usize = 16;
+
+        synced.pending_release.push(registration.clone());
+
+        let len = synced.pending_release.len();
+        self.num_pending_release.store(len, Release);
+
+        len == NOTIFY_AFTER
+    }
+
+    pub(super) fn shutdown(&self, synced: &mut Synced) -> Vec<Arc<ScheduledIo>> {
+        if synced.is_shutdown {
+            return vec![];
+        }
+
+        synced.is_shutdown = true;
+        synced.pending_release.clear();
+
+        // Building a vec of all outstanding I/O handles could be expensive, but
+        // this is the shutdown operation. In theory, shutdowns should be
+        // "clean" with no outstanding I/O resources. Even if it is slow, we
+        // aren't optimizing for shutdown.
+        let mut ret = vec![];
+
+        while let Some(io) = synced.registrations.pop_back() {
+            ret.push(io);
+        }
+
+        ret
+    }
+
+    pub(super) fn release(&self, synced: &mut Synced) {
+        for io in synced.pending_release.drain(..) {
+            // safety: the registration is part of our list
+            let _ = unsafe { synced.registrations.remove(io.as_ref().into()) };
+        }
+
+        self.num_pending_release.store(0, Release);
+    }
+}
+
+// Safety: `Arc` pins the inner data
+unsafe impl linked_list::Link for Arc<ScheduledIo> {
+    type Handle = Arc<ScheduledIo>;
+    type Target = ScheduledIo;
+
+    fn as_raw(handle: &Self::Handle) -> NonNull<ScheduledIo> {
+        // safety: Arc::as_ptr never returns null
+        unsafe { NonNull::new_unchecked(Arc::as_ptr(handle) as *mut _) }
+    }
+
+    unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Arc<ScheduledIo> {
+        // safety: the linked list currently owns a ref count
+        unsafe { Arc::from_raw(ptr.as_ptr() as *const _) }
+    }
+
+    unsafe fn pointers(
+        target: NonNull<Self::Target>,
+    ) -> NonNull<linked_list::Pointers<ScheduledIo>> {
+        NonNull::new_unchecked(target.as_ref().linked_list_pointers.get())
+    }
+}

tokio/src/runtime/io/scheduled_io.rs

@@ -1,43 +1,122 @@
-use super::{ReadyEvent, Tick};
 use crate::io::interest::Interest;
 use crate::io::ready::Ready;
 use crate::loom::sync::atomic::AtomicUsize;
 use crate::loom::sync::Mutex;
+use crate::runtime::io::{Direction, ReadyEvent, Tick};
 use crate::util::bit;
-use crate::util::slab::Entry;
+use crate::util::linked_list::{self, LinkedList};
 use crate::util::WakeList;
 
-use std::sync::atomic::Ordering::{AcqRel, Acquire, Release};
+use std::cell::UnsafeCell;
+use std::future::Future;
+use std::marker::PhantomPinned;
+use std::pin::Pin;
+use std::ptr::NonNull;
+use std::sync::atomic::Ordering::{AcqRel, Acquire};
 use std::task::{Context, Poll, Waker};
 
-use super::Direction;
-
-cfg_io_readiness! {
-    use crate::util::linked_list::{self, LinkedList};
-
-    use std::cell::UnsafeCell;
-    use std::future::Future;
-    use std::marker::PhantomPinned;
-    use std::pin::Pin;
-    use std::ptr::NonNull;
-}
-
 /// Stored in the I/O driver resource slab.
 #[derive(Debug)]
+// # This struct should be cache padded to avoid false sharing. The cache padding rules are copied
+// from crossbeam-utils/src/cache_padded.rs
+//
+// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache
+// lines at a time, so we have to align to 128 bytes rather than 64.
+//
+// Sources:
+// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
+// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107
+//
+// ARM's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size.
+//
+// Sources:
+// - https://www.mono-project.com/news/2016/09/12/arm64-icache/
+//
+// powerpc64 has 128-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9
+#[cfg_attr(
+    any(
+        target_arch = "x86_64",
+        target_arch = "aarch64",
+        target_arch = "powerpc64",
+    ),
+    repr(align(128))
+)]
+// arm, mips, mips64, riscv64, sparc, and hexagon have 32-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7
+// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17
+// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12
+//
+// riscv32 is assumed not to exceed the cache line size of riscv64.
+#[cfg_attr(
+    any(
+        target_arch = "arm",
+        target_arch = "mips",
+        target_arch = "mips64",
+        target_arch = "riscv32",
+        target_arch = "riscv64",
+        target_arch = "sparc",
+        target_arch = "hexagon",
+    ),
+    repr(align(32))
+)]
+// m68k has 16-byte cache line size.
+//
+// Sources:
+// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9
+#[cfg_attr(target_arch = "m68k", repr(align(16)))]
+// s390x has 256-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7
+// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13
+#[cfg_attr(target_arch = "s390x", repr(align(256)))]
+// x86, wasm, and sparc64 have 64-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7
+// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19
+//
+// All others are assumed to have 64-byte cache line size.
+#[cfg_attr(
+    not(any(
+        target_arch = "x86_64",
+        target_arch = "aarch64",
+        target_arch = "powerpc64",
+        target_arch = "arm",
+        target_arch = "mips",
+        target_arch = "mips64",
+        target_arch = "riscv32",
+        target_arch = "riscv64",
+        target_arch = "sparc",
+        target_arch = "hexagon",
+        target_arch = "m68k",
+        target_arch = "s390x",
+    )),
+    repr(align(64))
+)]
 pub(crate) struct ScheduledIo {
-    /// Packs the resource's readiness with the resource's generation.
+    pub(super) linked_list_pointers: UnsafeCell<linked_list::Pointers<Self>>,
+
+    /// Packs the resource's readiness and I/O driver latest tick.
     readiness: AtomicUsize,
 
     waiters: Mutex<Waiters>,
 }
 
-cfg_io_readiness! {
-    type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>;
-}
+type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>;
 
 #[derive(Debug, Default)]
 struct Waiters {
-    #[cfg(feature = "net")]
     /// List of all current waiters.
     list: WaitList,
 
@@ -48,83 +127,64 @@ struct Waiters {
     writer: Option<Waker>,
 }
 
-cfg_io_readiness! {
-    #[derive(Debug)]
-    struct Waiter {
-        pointers: linked_list::Pointers<Waiter>,
+#[derive(Debug)]
+struct Waiter {
+    pointers: linked_list::Pointers<Waiter>,
 
-        /// The waker for this task.
-        waker: Option<Waker>,
+    /// The waker for this task.
+    waker: Option<Waker>,
 
-        /// The interest this waiter is waiting on.
-        interest: Interest,
+    /// The interest this waiter is waiting on.
+    interest: Interest,
 
-        is_ready: bool,
+    is_ready: bool,
 
-        /// Should never be `!Unpin`.
-        _p: PhantomPinned,
-    }
+    /// Should never be `!Unpin`.
+    _p: PhantomPinned,
+}
 
-    generate_addr_of_methods! {
-        impl<> Waiter {
-            unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> {
-                &self.pointers
-            }
+generate_addr_of_methods! {
+    impl<> Waiter {
+        unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> {
+            &self.pointers
         }
     }
+}
 
-    /// Future returned by `readiness()`.
-    struct Readiness<'a> {
-        scheduled_io: &'a ScheduledIo,
+/// Future returned by `readiness()`.
+struct Readiness<'a> {
+    scheduled_io: &'a ScheduledIo,
 
-        state: State,
+    state: State,
 
-        /// Entry in the waiter `LinkedList`.
-        waiter: UnsafeCell<Waiter>,
-    }
+    /// Entry in the waiter `LinkedList`.
+    waiter: UnsafeCell<Waiter>,
+}
 
-    enum State {
-        Init,
-        Waiting,
-        Done,
-    }
+enum State {
+    Init,
+    Waiting,
+    Done,
 }
 
 // The `ScheduledIo::readiness` (`AtomicUsize`) is packed full of goodness.
 //
-// | shutdown | generation |  driver tick | readiness |
-// |----------+------------+--------------+-----------|
-// |   1 bit  |   7 bits   +    8 bits    +   16 bits |
+// | shutdown | driver tick | readiness |
+// |----------+-------------+-----------|
+// |   1 bit  |   8 bits    +   16 bits |
 
 const READINESS: bit::Pack = bit::Pack::least_significant(16);
 
 const TICK: bit::Pack = READINESS.then(8);
 
-const GENERATION: bit::Pack = TICK.then(7);
-
-const SHUTDOWN: bit::Pack = GENERATION.then(1);
-
-#[test]
-fn test_generations_assert_same() {
-    assert_eq!(super::GENERATION, GENERATION);
-}
+const SHUTDOWN: bit::Pack = TICK.then(1);
 
 // ===== impl ScheduledIo =====
 
-impl Entry for ScheduledIo {
-    fn reset(&self) {
-        let state = self.readiness.load(Acquire);
-
-        let generation = GENERATION.unpack(state);
-        let next = GENERATION.pack_lossy(generation + 1, 0);
-
-        self.readiness.store(next, Release);
-    }
-}
-
 impl Default for ScheduledIo {
     fn default() -> ScheduledIo {
         ScheduledIo {
+            linked_list_pointers: UnsafeCell::new(linked_list::Pointers::new()),
             readiness: AtomicUsize::new(0),
             waiters: Mutex::new(Default::default()),
         }
@@ -132,8 +192,9 @@ impl Default for ScheduledIo {
 }
 
 impl ScheduledIo {
-    pub(crate) fn generation(&self) -> usize {
-        GENERATION.unpack(self.readiness.load(Acquire))
+    pub(crate) fn token(&self) -> mio::Token {
+        // use `expose_addr` when stable
+        mio::Token(self as *const _ as usize)
     }
 
     /// Invoked when the IO driver is shut down; forces this ScheduledIo into a
@@ -148,61 +209,39 @@ impl ScheduledIo {
     /// the current value, returning the previous readiness value.
     ///
     /// # Arguments
-    /// - `token`: the token for this `ScheduledIo`.
     /// - `tick`: whether setting the tick or trying to clear readiness for a
     ///    specific tick.
     /// - `f`: a closure returning a new readiness value given the previous
     ///   readiness.
-    ///
-    /// # Returns
-    ///
-    /// If the given token's generation no longer matches the `ScheduledIo`'s
-    /// generation, then the corresponding IO resource has been removed and
-    /// replaced with a new resource. In that case, this method returns `Err`.
-    /// Otherwise, this returns the previous readiness.
-    pub(super) fn set_readiness(
-        &self,
-        token: Option<usize>,
-        tick: Tick,
-        f: impl Fn(Ready) -> Ready,
-    ) -> Result<(), ()> {
+    pub(super) fn set_readiness(&self, tick: Tick, f: impl Fn(Ready) -> Ready) {
         let mut current = self.readiness.load(Acquire);
 
+        // The shutdown bit should not be set
+        debug_assert_eq!(0, SHUTDOWN.unpack(current));
+
         loop {
-            let current_generation = GENERATION.unpack(current);
-
-            if let Some(token) = token {
-                // Check that the generation for this access is still the
-                // current one.
-                if GENERATION.unpack(token) != current_generation {
-                    return Err(());
-                }
-            }
-
-            // Mask out the tick/generation bits so that the modifying
-            // function doesn't see them.
+            // Mask out the tick bits so that the modifying function doesn't see
+            // them.
             let current_readiness = Ready::from_usize(current);
             let new = f(current_readiness);
 
-            let packed = match tick {
+            let next = match tick {
                 Tick::Set(t) => TICK.pack(t as usize, new.as_usize()),
                 Tick::Clear(t) => {
                     if TICK.unpack(current) as u8 != t {
                         // Trying to clear readiness with an old event!
-                        return Err(());
+                        return;
                     }
 
                     TICK.pack(t as usize, new.as_usize())
                 }
             };
 
-            let next = GENERATION.pack(current_generation, packed);
-
             match self
                 .readiness
                 .compare_exchange(current, next, AcqRel, Acquire)
             {
-                Ok(_) => return Ok(()),
+                Ok(_) => return,
                 // we lost the race, retry!
                 Err(actual) => current = actual,
             }
@@ -238,7 +277,6 @@ impl ScheduledIo {
             }
         }
 
-        #[cfg(feature = "net")]
         'outer: loop {
             let mut iter = waiters.list.drain_filter(|w| ready.satisfies(w.interest));
 
@@ -351,9 +389,7 @@ impl ScheduledIo {
         // This consumes the current readiness state **except** for closed
         // states. Closed states are excluded because they are final states.
         let mask_no_closed = event.ready - Ready::READ_CLOSED - Ready::WRITE_CLOSED;
-
-        // result isn't important
-        let _ = self.set_readiness(None, Tick::Clear(event.tick), |curr| curr - mask_no_closed);
+        self.set_readiness(Tick::Clear(event.tick), |curr| curr - mask_no_closed);
     }
 
     pub(crate) fn clear_wakers(&self) {
@@ -372,187 +408,193 @@ impl Drop for ScheduledIo {
 unsafe impl Send for ScheduledIo {}
 unsafe impl Sync for ScheduledIo {}
 
-cfg_io_readiness! {
-    impl ScheduledIo {
-        /// An async version of `poll_readiness` which uses a linked list of wakers.
-        pub(crate) async fn readiness(&self, interest: Interest) -> ReadyEvent {
-            self.readiness_fut(interest).await
-        }
-
-        // This is in a separate function so that the borrow checker doesn't think
-        // we are borrowing the `UnsafeCell` possibly over await boundaries.
-        //
-        // Go figure.
-        fn readiness_fut(&self, interest: Interest) -> Readiness<'_> {
-            Readiness {
-                scheduled_io: self,
-                state: State::Init,
-                waiter: UnsafeCell::new(Waiter {
-                    pointers: linked_list::Pointers::new(),
-                    waker: None,
-                    is_ready: false,
-                    interest,
-                    _p: PhantomPinned,
-                }),
-            }
-        }
+impl ScheduledIo {
+    /// An async version of `poll_readiness` which uses a linked list of wakers.
+    pub(crate) async fn readiness(&self, interest: Interest) -> ReadyEvent {
+        self.readiness_fut(interest).await
     }
 
-    unsafe impl linked_list::Link for Waiter {
-        type Handle = NonNull<Waiter>;
-        type Target = Waiter;
-
-        fn as_raw(handle: &NonNull<Waiter>) -> NonNull<Waiter> {
-            *handle
-        }
-
-        unsafe fn from_raw(ptr: NonNull<Waiter>) -> NonNull<Waiter> {
-            ptr
-        }
-
-        unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
-            Waiter::addr_of_pointers(target)
+    // This is in a separate function so that the borrow checker doesn't think
+    // we are borrowing the `UnsafeCell` possibly over await boundaries.
+    //
+    // Go figure.
+    fn readiness_fut(&self, interest: Interest) -> Readiness<'_> {
+        Readiness {
+            scheduled_io: self,
+            state: State::Init,
+            waiter: UnsafeCell::new(Waiter {
+                pointers: linked_list::Pointers::new(),
+                waker: None,
+                is_ready: false,
+                interest,
+                _p: PhantomPinned,
+            }),
         }
     }
+}
 
-    // ===== impl Readiness =====
+unsafe impl linked_list::Link for Waiter {
+    type Handle = NonNull<Waiter>;
+    type Target = Waiter;
 
-    impl Future for Readiness<'_> {
-        type Output = ReadyEvent;
+    fn as_raw(handle: &NonNull<Waiter>) -> NonNull<Waiter> {
+        *handle
+    }
 
-        fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-            use std::sync::atomic::Ordering::SeqCst;
+    unsafe fn from_raw(ptr: NonNull<Waiter>) -> NonNull<Waiter> {
+        ptr
+    }
 
-            let (scheduled_io, state, waiter) = unsafe {
-                let me = self.get_unchecked_mut();
-                (&me.scheduled_io, &mut me.state, &me.waiter)
-            };
+    unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
+        Waiter::addr_of_pointers(target)
+    }
+}
 
-            loop {
-                match *state {
-                    State::Init => {
-                        // Optimistically check existing readiness
-                        let curr = scheduled_io.readiness.load(SeqCst);
-                        let ready = Ready::from_usize(READINESS.unpack(curr));
-                        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+// ===== impl Readiness =====
 
-                        // Safety: `waiter.interest` never changes
-                        let interest = unsafe { (*waiter.get()).interest };
-                        let ready = ready.intersection(interest);
+impl Future for Readiness<'_> {
+    type Output = ReadyEvent;
 
-                        if !ready.is_empty() || is_shutdown {
-                            // Currently ready!
-                            let tick = TICK.unpack(curr) as u8;
-                            *state = State::Done;
-                            return Poll::Ready(ReadyEvent { tick, ready, is_shutdown });
-                        }
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        use std::sync::atomic::Ordering::SeqCst;
 
-                        // Wasn't ready, take the lock (and check again while locked).
-                        let mut waiters = scheduled_io.waiters.lock();
+        let (scheduled_io, state, waiter) = unsafe {
+            let me = self.get_unchecked_mut();
+            (&me.scheduled_io, &mut me.state, &me.waiter)
+        };
 
-                        let curr = scheduled_io.readiness.load(SeqCst);
-                        let mut ready = Ready::from_usize(READINESS.unpack(curr));
-                        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+        loop {
+            match *state {
+                State::Init => {
+                    // Optimistically check existing readiness
+                    let curr = scheduled_io.readiness.load(SeqCst);
+                    let ready = Ready::from_usize(READINESS.unpack(curr));
+                    let is_shutdown = SHUTDOWN.unpack(curr) != 0;
 
-                        if is_shutdown {
-                            ready = Ready::ALL;
-                        }
+                    // Safety: `waiter.interest` never changes
+                    let interest = unsafe { (*waiter.get()).interest };
+                    let ready = ready.intersection(interest);
 
-                        let ready = ready.intersection(interest);
-
-                        if !ready.is_empty() || is_shutdown {
-                            // Currently ready!
-                            let tick = TICK.unpack(curr) as u8;
-                            *state = State::Done;
-                            return Poll::Ready(ReadyEvent { tick, ready, is_shutdown });
-                        }
-
-                        // Not ready even after locked, insert into list...
-
-                        // Safety: called while locked
-                        unsafe {
-                            (*waiter.get()).waker = Some(cx.waker().clone());
-                        }
-
-                        // Insert the waiter into the linked list
-                        //
-                        // safety: pointers from `UnsafeCell` are never null.
-                        waiters
-                            .list
-                            .push_front(unsafe { NonNull::new_unchecked(waiter.get()) });
-                        *state = State::Waiting;
-                    }
-                    State::Waiting => {
-                        // Currently in the "Waiting" state, implying the caller has
-                        // a waiter stored in the waiter list (guarded by
-                        // `notify.waiters`). In order to access the waker fields,
-                        // we must hold the lock.
-
-                        let waiters = scheduled_io.waiters.lock();
-
-                        // Safety: called while locked
-                        let w = unsafe { &mut *waiter.get() };
-
-                        if w.is_ready {
-                            // Our waker has been notified.
-                            *state = State::Done;
-                        } else {
-                            // Update the waker, if necessary.
-                            if !w.waker.as_ref().unwrap().will_wake(cx.waker()) {
-                                w.waker = Some(cx.waker().clone());
-                            }
-
-                            return Poll::Pending;
-                        }
-
-                        // Explicit drop of the lock to indicate the scope that the
-                        // lock is held. Because holding the lock is required to
-                        // ensure safe access to fields not held within the lock, it
-                        // is helpful to visualize the scope of the critical
-                        // section.
-                        drop(waiters);
-                    }
-                    State::Done => {
-                        // Safety: State::Done means it is no longer shared
-                        let w = unsafe { &mut *waiter.get() };
-
-                        let curr = scheduled_io.readiness.load(Acquire);
-                        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
-
-                        // The returned tick might be newer than the event
-                        // which notified our waker. This is ok because the future
-                        // still didn't return `Poll::Ready`.
+                    if !ready.is_empty() || is_shutdown {
+                        // Currently ready!
                         let tick = TICK.unpack(curr) as u8;
-
-                        // The readiness state could have been cleared in the meantime,
-                        // but we allow the returned ready set to be empty.
-                        let curr_ready = Ready::from_usize(READINESS.unpack(curr));
-                        let ready = curr_ready.intersection(w.interest);
-
+                        *state = State::Done;
                         return Poll::Ready(ReadyEvent {
                             tick,
                             ready,
                             is_shutdown,
                         });
                     }
+
+                    // Wasn't ready, take the lock (and check again while locked).
+                    let mut waiters = scheduled_io.waiters.lock();
+
+                    let curr = scheduled_io.readiness.load(SeqCst);
+                    let mut ready = Ready::from_usize(READINESS.unpack(curr));
+                    let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+
+                    if is_shutdown {
+                        ready = Ready::ALL;
+                    }
+
+                    let ready = ready.intersection(interest);
+
+                    if !ready.is_empty() || is_shutdown {
+                        // Currently ready!
+                        let tick = TICK.unpack(curr) as u8;
+                        *state = State::Done;
+                        return Poll::Ready(ReadyEvent {
+                            tick,
+                            ready,
+                            is_shutdown,
+                        });
+                    }
+
+                    // Not ready even after locked, insert into list...
+
+                    // Safety: called while locked
+                    unsafe {
+                        (*waiter.get()).waker = Some(cx.waker().clone());
+                    }
+
+                    // Insert the waiter into the linked list
+                    //
+                    // safety: pointers from `UnsafeCell` are never null.
+                    waiters
+                        .list
+                        .push_front(unsafe { NonNull::new_unchecked(waiter.get()) });
+                    *state = State::Waiting;
+                }
+                State::Waiting => {
+                    // Currently in the "Waiting" state, implying the caller has
+                    // a waiter stored in the waiter list (guarded by
+                    // `notify.waiters`). In order to access the waker fields,
+                    // we must hold the lock.
+
+                    let waiters = scheduled_io.waiters.lock();
+
+                    // Safety: called while locked
+                    let w = unsafe { &mut *waiter.get() };
+
+                    if w.is_ready {
+                        // Our waker has been notified.
+                        *state = State::Done;
+                    } else {
+                        // Update the waker, if necessary.
+                        if !w.waker.as_ref().unwrap().will_wake(cx.waker()) {
+                            w.waker = Some(cx.waker().clone());
+                        }
+
+                        return Poll::Pending;
+                    }
+
+                    // Explicit drop of the lock to indicate the scope that the
+                    // lock is held. Because holding the lock is required to
+                    // ensure safe access to fields not held within the lock, it
+                    // is helpful to visualize the scope of the critical
+                    // section.
+                    drop(waiters);
+                }
+                State::Done => {
+                    // Safety: State::Done means it is no longer shared
+                    let w = unsafe { &mut *waiter.get() };
+
+                    let curr = scheduled_io.readiness.load(Acquire);
+                    let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+
+                    // The returned tick might be newer than the event
+                    // which notified our waker. This is ok because the future
+                    // still didn't return `Poll::Ready`.
+                    let tick = TICK.unpack(curr) as u8;
+
+                    // The readiness state could have been cleared in the meantime,
+                    // but we allow the returned ready set to be empty.
+                    let curr_ready = Ready::from_usize(READINESS.unpack(curr));
+                    let ready = curr_ready.intersection(w.interest);
+
+                    return Poll::Ready(ReadyEvent {
+                        tick,
+                        ready,
+                        is_shutdown,
+                    });
                 }
             }
         }
     }
-
-    impl Drop for Readiness<'_> {
-        fn drop(&mut self) {
-            let mut waiters = self.scheduled_io.waiters.lock();
-
-            // Safety: `waiter` is only ever stored in `waiters`
-            unsafe {
-                waiters
-                    .list
-                    .remove(NonNull::new_unchecked(self.waiter.get()))
-            };
-        }
-    }
-
-    unsafe impl Send for Readiness<'_> {}
-    unsafe impl Sync for Readiness<'_> {}
 }
+
+impl Drop for Readiness<'_> {
+    fn drop(&mut self) {
+        let mut waiters = self.scheduled_io.waiters.lock();
+
+        // Safety: `waiter` is only ever stored in `waiters`
+        unsafe {
+            waiters
+                .list
+                .remove(NonNull::new_unchecked(self.waiter.get()))
+        };
+    }
+}
+
+unsafe impl Send for Readiness<'_> {}
+unsafe impl Sync for Readiness<'_> {}

tokio/src/runtime/metrics/batch.rs

@@ -73,7 +73,8 @@ impl MetricsBatch {
         }
     }
 
-    pub(crate) fn submit(&mut self, worker: &WorkerMetrics) {
+    pub(crate) fn submit(&mut self, worker: &WorkerMetrics, mean_poll_time: u64) {
+        worker.mean_poll_time.store(mean_poll_time, Relaxed);
         worker.park_count.store(self.park_count, Relaxed);
         worker.noop_count.store(self.noop_count, Relaxed);
         worker.steal_count.store(self.steal_count, Relaxed);

tokio/src/runtime/metrics/mock.rs

@@ -37,7 +37,7 @@ impl MetricsBatch {
         Self {}
     }
 
-    pub(crate) fn submit(&mut self, _to: &WorkerMetrics) {}
+    pub(crate) fn submit(&mut self, _to: &WorkerMetrics, _mean_poll_time: u64) {}
     pub(crate) fn about_to_park(&mut self) {}
     pub(crate) fn inc_local_schedule_count(&mut self) {}
     pub(crate) fn start_processing_scheduled_tasks(&mut self) {}

tokio/src/runtime/metrics/runtime.rs

@@ -769,6 +769,47 @@ impl RuntimeMetrics {
             .unwrap_or_default()
     }
 
+    /// Returns the mean duration of task polls, in nanoseconds.
+    ///
+    /// This is an exponentially weighted moving average. Currently, this metric
+    /// is only provided by the multi-threaded runtime.
+    ///
+    /// # Arguments
+    ///
+    /// `worker` is the index of the worker being queried. The given value must
+    /// be between 0 and `num_workers()`. The index uniquely identifies a single
+    /// worker and will continue to identify the worker throughout the lifetime
+    /// of the runtime instance.
+    ///
+    /// # Panics
+    ///
+    /// The method panics when `worker` represents an invalid worker, i.e. is
+    /// greater than or equal to `num_workers()`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Handle;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     let metrics = Handle::current().metrics();
+    ///
+    ///     let n = metrics.worker_mean_poll_time(0);
+    ///     println!("worker 0 has a mean poll time of {:?}", n);
+    /// }
+    /// ```
+    #[track_caller]
+    pub fn worker_mean_poll_time(&self, worker: usize) -> Duration {
+        let nanos = self
+            .handle
+            .inner
+            .worker_metrics(worker)
+            .mean_poll_time
+            .load(Relaxed);
+        Duration::from_nanos(nanos)
+    }
+
     /// Returns the number of tasks currently scheduled in the blocking
     /// thread pool, spawned using `spawn_blocking`.
     ///

tokio/src/runtime/metrics/worker.rs

@@ -28,6 +28,9 @@ pub(crate) struct WorkerMetrics {
     /// Number of tasks the worker polled.
     pub(crate) poll_count: AtomicU64,
 
+    /// EWMA task poll time, in nanoseconds.
+    pub(crate) mean_poll_time: AtomicU64,
+
     /// Amount of time the worker spent doing work vs. parking.
     pub(crate) busy_duration_total: AtomicU64,
 
@@ -62,6 +65,7 @@ impl WorkerMetrics {
             steal_count: AtomicU64::new(0),
             steal_operations: AtomicU64::new(0),
             poll_count: AtomicU64::new(0),
+            mean_poll_time: AtomicU64::new(0),
             overflow_count: AtomicU64::new(0),
             busy_duration_total: AtomicU64::new(0),
             local_schedule_count: AtomicU64::new(0),

tokio/src/runtime/mod.rs

@@ -175,7 +175,7 @@
 
 // At the top due to macros
 #[cfg(test)]
-#[cfg(not(tokio_wasm))]
+#[cfg(not(target_family = "wasm"))]
 #[macro_use]
 mod tests;
 
@@ -212,7 +212,7 @@ cfg_rt! {
     use config::Config;
 
     mod blocking;
-    #[cfg_attr(tokio_wasi, allow(unused_imports))]
+    #[cfg_attr(target_os = "wasi", allow(unused_imports))]
     pub(crate) use blocking::spawn_blocking;
 
     cfg_trace! {
@@ -226,6 +226,10 @@ cfg_rt! {
     mod builder;
     pub use self::builder::Builder;
     cfg_unstable! {
+        mod id;
+        #[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+        pub use id::Id;
+
         pub use self::builder::UnhandledPanic;
         pub use crate::util::rand::RngSeed;
     }

tokio/src/runtime/park.rs

@@ -67,9 +67,9 @@ impl ParkThread {
         CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
 
         // Wasm doesn't have threads, so just sleep.
-        #[cfg(not(tokio_wasm))]
+        #[cfg(not(target_family = "wasm"))]
         self.inner.park_timeout(duration);
-        #[cfg(tokio_wasm)]
+        #[cfg(target_family = "wasm")]
         std::thread::sleep(duration);
     }
 
@@ -222,7 +222,6 @@ impl UnparkThread {
 use crate::loom::thread::AccessError;
 use std::future::Future;
 use std::marker::PhantomData;
-use std::mem;
 use std::rc::Rc;
 use std::task::{RawWaker, RawWakerVTable, Waker};
 
@@ -317,16 +316,12 @@ unsafe fn unparker_to_raw_waker(unparker: Arc<Inner>) -> RawWaker {
 }
 
 unsafe fn clone(raw: *const ()) -> RawWaker {
-    let unparker = Inner::from_raw(raw);
-
-    // Increment the ref count
-    mem::forget(unparker.clone());
-
-    unparker_to_raw_waker(unparker)
+    Arc::increment_strong_count(raw as *const Inner);
+    unparker_to_raw_waker(Inner::from_raw(raw))
 }
 
 unsafe fn drop_waker(raw: *const ()) {
-    let _ = Inner::from_raw(raw);
+    drop(Inner::from_raw(raw));
 }
 
 unsafe fn wake(raw: *const ()) {
@@ -335,11 +330,8 @@ unsafe fn wake(raw: *const ()) {
 }
 
 unsafe fn wake_by_ref(raw: *const ()) {
-    let unparker = Inner::from_raw(raw);
-    unparker.unpark();
-
-    // We don't actually own a reference to the unparker
-    mem::forget(unparker);
+    let raw = raw as *const Inner;
+    (*raw).unpark();
 }
 
 #[cfg(loom)]

tokio/src/runtime/runtime.rs

@@ -9,6 +9,10 @@ use std::time::Duration;
 cfg_rt_multi_thread! {
     use crate::runtime::Builder;
     use crate::runtime::scheduler::MultiThread;
+
+    cfg_unstable! {
+        use crate::runtime::scheduler::MultiThreadAlt;
+    }
 }
 
 /// The Tokio runtime.
@@ -25,7 +29,7 @@ cfg_rt_multi_thread! {
 /// # Shutdown
 ///
 /// Shutting down the runtime is done by dropping the value, or calling
-/// [`Runtime::shutdown_background`] or [`Runtime::shutdown_timeout`].
+/// [`shutdown_background`] or [`shutdown_timeout`].
 ///
 /// Tasks spawned through [`Runtime::spawn`] keep running until they yield.
 /// Then they are dropped. They are not *guaranteed* to run to completion, but
@@ -38,11 +42,11 @@ cfg_rt_multi_thread! {
 /// stopped. This can take an indefinite amount of time. The `Drop`
 /// implementation waits forever for this.
 ///
-/// `shutdown_background` and `shutdown_timeout` can be used if waiting forever
-/// is undesired. When the timeout is reached, spawned work that did not stop
-/// in time and threads running it are leaked. The work continues to run until
-/// one of the stopping conditions is fulfilled, but the thread initiating the
-/// shutdown is unblocked.
+/// The [`shutdown_background`] and [`shutdown_timeout`] methods can be used if
+/// waiting forever is undesired. When the timeout is reached, spawned work that
+/// did not stop in time and threads running it are leaked. The work continues
+/// to run until one of the stopping conditions is fulfilled, but the thread
+/// initiating the shutdown is unblocked.
 ///
 /// Once the runtime has been dropped, any outstanding I/O resources bound to
 /// it will no longer function. Calling any method on them will result in an
@@ -50,18 +54,43 @@ cfg_rt_multi_thread! {
 ///
 /// # Sharing
 ///
-/// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it
-/// in a `Arc`. Most fn take `&self` to allow you to call them concurrently
-/// across multiple threads.
+/// There are several ways to establish shared access to a Tokio runtime:
 ///
-/// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of
-/// the runtime type and this can be achieved via `Arc::try_unwrap` when only
-/// one strong count reference is left over.
+///  * Using an <code>[Arc]\<Runtime></code>.
+///  * Using a [`Handle`].
+///  * Entering the runtime context.
+///
+/// Using an <code>[Arc]\<Runtime></code> or [`Handle`] allows you to do various
+/// things with the runtime such as spawning new tasks or entering the runtime
+/// context. Both types can be cloned to create a new handle that allows access
+/// to the same runtime. By passing clones into different tasks or threads, you
+/// will be able to access the runtime from those tasks or threads.
+///
+/// The difference between <code>[Arc]\<Runtime></code> and [`Handle`] is that
+/// an <code>[Arc]\<Runtime></code> will prevent the runtime from shutting down,
+/// whereas a [`Handle`] does not prevent that. This is because shutdown of the
+/// runtime happens when the destructor of the `Runtime` object runs.
+///
+/// Calls to [`shutdown_background`] and [`shutdown_timeout`] require exclusive
+/// ownership of the `Runtime` type. When using an <code>[Arc]\<Runtime></code>,
+/// this can be achieved via [`Arc::try_unwrap`] when only one strong count
+/// reference is left over.
+///
+/// The runtime context is entered using the [`Runtime::enter`] or
+/// [`Handle::enter`] methods, which use a thread-local variable to store the
+/// current runtime. Whenever you are inside the runtime context, methods such
+/// as [`tokio::spawn`] will use the runtime whose context you are inside.
 ///
 /// [timer]: crate::time
 /// [mod]: index.html
 /// [`new`]: method@Self::new
 /// [`Builder`]: struct@Builder
+/// [`Handle`]: struct@Handle
+/// [`tokio::spawn`]: crate::spawn
+/// [`Arc::try_unwrap`]: std::sync::Arc::try_unwrap
+/// [Arc]: std::sync::Arc
+/// [`shutdown_background`]: method@Runtime::shutdown_background
+/// [`shutdown_timeout`]: method@Runtime::shutdown_timeout
 #[derive(Debug)]
 pub struct Runtime {
     /// Task scheduler
@@ -84,6 +113,9 @@ pub enum RuntimeFlavor {
     CurrentThread,
     /// The flavor that executes tasks across multiple threads.
     MultiThread,
+    /// The flavor that executes tasks across multiple threads.
+    #[cfg(tokio_unstable)]
+    MultiThreadAlt,
 }
 
 /// The runtime scheduler is either a multi-thread or a current-thread executor.
@@ -93,8 +125,12 @@ pub(super) enum Scheduler {
     CurrentThread(CurrentThread),
 
     /// Execute tasks across multiple threads.
-    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+    #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
     MultiThread(MultiThread),
+
+    /// Execute tasks across multiple threads.
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+    MultiThreadAlt(MultiThreadAlt),
 }
 
 impl Runtime {
@@ -309,8 +345,10 @@ impl Runtime {
 
         match &self.scheduler {
             Scheduler::CurrentThread(exec) => exec.block_on(&self.handle.inner, future),
-            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             Scheduler::MultiThread(exec) => exec.block_on(&self.handle.inner, future),
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+            Scheduler::MultiThreadAlt(exec) => exec.block_on(&self.handle.inner, future),
         }
     }
 
@@ -425,12 +463,18 @@ impl Drop for Runtime {
                 let _guard = context::try_set_current(&self.handle.inner);
                 current_thread.shutdown(&self.handle.inner);
             }
-            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             Scheduler::MultiThread(multi_thread) => {
                 // The threaded scheduler drops its tasks on its worker threads, which is
                 // already in the runtime's context.
                 multi_thread.shutdown(&self.handle.inner);
             }
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+            Scheduler::MultiThreadAlt(multi_thread) => {
+                // The threaded scheduler drops its tasks on its worker threads, which is
+                // already in the runtime's context.
+                multi_thread.shutdown(&self.handle.inner);
+            }
         }
     }
 }

tokio/src/runtime/scheduler/block_in_place.rs

@@ -0,0 +1,21 @@
+use crate::runtime::scheduler;
+
+#[track_caller]
+pub(crate) fn block_in_place<F, R>(f: F) -> R
+where
+    F: FnOnce() -> R,
+{
+    #[cfg(tokio_unstable)]
+    {
+        use crate::runtime::{Handle, RuntimeFlavor::MultiThreadAlt};
+
+        match Handle::try_current().map(|h| h.runtime_flavor()) {
+            Ok(MultiThreadAlt) => {
+                return scheduler::multi_thread_alt::block_in_place(f);
+            }
+            _ => {}
+        }
+    }
+
+    scheduler::multi_thread::block_in_place(f)
+}

tokio/src/runtime/scheduler/current_thread/mod.rs

@@ -321,7 +321,7 @@ impl Core {
     }
 
     fn submit_metrics(&mut self, handle: &Handle) {
-        self.metrics.submit(&handle.shared.worker_metrics);
+        self.metrics.submit(&handle.shared.worker_metrics, 0);
     }
 }
 
@@ -523,6 +523,10 @@ cfg_metrics! {
             &self.shared.worker_metrics
         }
 
+        pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+            self.worker_metrics(worker).queue_depth()
+        }
+
         pub(crate) fn num_blocking_threads(&self) -> usize {
             self.blocking_spawner.num_threads()
         }
@@ -541,6 +545,16 @@ cfg_metrics! {
     }
 }
 
+cfg_unstable! {
+    use std::num::NonZeroU64;
+
+    impl Handle {
+        pub(crate) fn owned_id(&self) -> NonZeroU64 {
+            self.shared.owned.id
+        }
+    }
+}
+
 impl fmt::Debug for Handle {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt.debug_struct("current_thread::Handle { ... }").finish()

tokio/src/runtime/scheduler/inject/rt_multi_thread.rs

@@ -75,6 +75,21 @@ impl<T: 'static> Shared<T> {
         debug_assert!(unsafe { batch_tail.get_queue_next().is_none() });
 
         let mut synced = shared.lock();
+
+        if synced.as_mut().is_closed {
+            drop(synced);
+
+            let mut curr = Some(batch_head);
+
+            while let Some(task) = curr {
+                curr = task.get_queue_next();
+
+                let _ = unsafe { task::Notified::<T>::from_raw(task) };
+            }
+
+            return;
+        }
+
         let synced = synced.as_mut();
 
         if let Some(tail) = synced.tail {

tokio/src/runtime/scheduler/inject/shared.rs

@@ -38,7 +38,10 @@ impl<T: 'static> Shared<T> {
     }
 
     // Kind of annoying to have to include the cfg here
-    #[cfg(any(tokio_taskdump, all(feature = "rt-multi-thread", not(tokio_wasi))))]
+    #[cfg(any(
+        tokio_taskdump,
+        all(feature = "rt-multi-thread", not(target_os = "wasi"))
+    ))]
     pub(crate) fn is_closed(&self, synced: &Synced) -> bool {
         synced.is_closed
     }
@@ -106,6 +109,8 @@ impl<T: 'static> Shared<T> {
     pub(crate) unsafe fn pop_n<'a>(&'a self, synced: &'a mut Synced, n: usize) -> Pop<'a, T> {
         use std::cmp;
 
+        debug_assert!(n > 0);
+
         // safety: All updates to the len atomic are guarded by the mutex. As
         // such, a non-atomic load followed by a store is safe.
         let len = self.len.unsync_load();

tokio/src/runtime/scheduler/inject/synced.rs

@@ -1,3 +1,8 @@
+#![cfg_attr(
+    any(not(all(tokio_unstable, feature = "full")), target_family = "wasm"),
+    allow(dead_code)
+)]
+
 use crate::runtime::task;
 
 pub(crate) struct Synced {
@@ -29,4 +34,8 @@ impl Synced {
         // safety: a `Notified` is pushed into the queue and now it is popped!
         Some(unsafe { task::Notified::from_raw(task) })
     }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.head.is_none()
+    }
 }

tokio/src/runtime/scheduler/mod.rs

@@ -10,11 +10,19 @@ cfg_rt! {
 }
 
 cfg_rt_multi_thread! {
+    mod block_in_place;
+    pub(crate) use block_in_place::block_in_place;
+
     mod lock;
     use lock::Lock;
 
     pub(crate) mod multi_thread;
     pub(crate) use multi_thread::MultiThread;
+
+    cfg_unstable! {
+        pub(crate) mod multi_thread_alt;
+        pub(crate) use multi_thread_alt::MultiThread as MultiThreadAlt;
+    }
 }
 
 use crate::runtime::driver;
@@ -24,9 +32,12 @@ pub(crate) enum Handle {
     #[cfg(feature = "rt")]
     CurrentThread(Arc<current_thread::Handle>),
 
-    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+    #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
     MultiThread(Arc<multi_thread::Handle>),
 
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+    MultiThreadAlt(Arc<multi_thread_alt::Handle>),
+
     // TODO: This is to avoid triggering "dead code" warnings many other places
     // in the codebase. Remove this during a later cleanup
     #[cfg(not(feature = "rt"))]
@@ -38,8 +49,11 @@ pub(crate) enum Handle {
 pub(super) enum Context {
     CurrentThread(current_thread::Context),
 
-    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+    #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
     MultiThread(multi_thread::Context),
+
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+    MultiThreadAlt(multi_thread_alt::Context),
 }
 
 impl Handle {
@@ -49,9 +63,12 @@ impl Handle {
             #[cfg(feature = "rt")]
             Handle::CurrentThread(ref h) => &h.driver,
 
-            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             Handle::MultiThread(ref h) => &h.driver,
 
+            #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+            Handle::MultiThreadAlt(ref h) => &h.driver,
+
             #[cfg(not(feature = "rt"))]
             Handle::Disabled => unreachable!(),
         }
@@ -67,6 +84,20 @@ cfg_rt! {
     use crate::util::RngSeedGenerator;
     use std::task::Waker;
 
+    macro_rules! match_flavor {
+        ($self:expr, $ty:ident($h:ident) => $e:expr) => {
+            match $self {
+                $ty::CurrentThread($h) => $e,
+
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
+                $ty::MultiThread($h) => $e,
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                $ty::MultiThreadAlt($h) => $e,
+            }
+        }
+    }
+
     impl Handle {
         #[track_caller]
         pub(crate) fn current() -> Handle {
@@ -77,12 +108,7 @@ cfg_rt! {
         }
 
         pub(crate) fn blocking_spawner(&self) -> &blocking::Spawner {
-            match self {
-                Handle::CurrentThread(h) => &h.blocking_spawner,
-
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                Handle::MultiThread(h) => &h.blocking_spawner,
-            }
+            match_flavor!(self, Handle(h) => &h.blocking_spawner)
         }
 
         pub(crate) fn spawn<F>(&self, future: F, id: Id) -> JoinHandle<F::Output>
@@ -93,8 +119,11 @@ cfg_rt! {
             match self {
                 Handle::CurrentThread(h) => current_thread::Handle::spawn(h, future, id),
 
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 Handle::MultiThread(h) => multi_thread::Handle::spawn(h, future, id),
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                Handle::MultiThreadAlt(h) => multi_thread_alt::Handle::spawn(h, future, id),
             }
         }
 
@@ -102,27 +131,36 @@ cfg_rt! {
             match *self {
                 Handle::CurrentThread(_) => {},
 
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 Handle::MultiThread(ref h) => h.shutdown(),
+
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                Handle::MultiThreadAlt(ref h) => h.shutdown(),
             }
         }
 
         pub(crate) fn seed_generator(&self) -> &RngSeedGenerator {
-            match self {
-                Handle::CurrentThread(h) => &h.seed_generator,
-
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                Handle::MultiThread(h) => &h.seed_generator,
-            }
+            match_flavor!(self, Handle(h) => &h.seed_generator)
         }
 
         pub(crate) fn as_current_thread(&self) -> &Arc<current_thread::Handle> {
             match self {
                 Handle::CurrentThread(handle) => handle,
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 _ => panic!("not a CurrentThread handle"),
             }
         }
+
+        cfg_rt_multi_thread! {
+            cfg_unstable! {
+                pub(crate) fn expect_multi_thread_alt(&self) -> &Arc<multi_thread_alt::Handle> {
+                    match self {
+                        Handle::MultiThreadAlt(handle) => handle,
+                        _ => panic!("not a `MultiThreadAlt` handle"),
+                    }
+                }
+            }
+        }
     }
 
     cfg_metrics! {
@@ -132,73 +170,43 @@ cfg_rt! {
             pub(crate) fn num_workers(&self) -> usize {
                 match self {
                     Handle::CurrentThread(_) => 1,
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                    #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                     Handle::MultiThread(handle) => handle.num_workers(),
+                    #[cfg(all(tokio_unstable, feature = "rt-multi-thread", not(target_os = "wasi")))]
+                    Handle::MultiThreadAlt(handle) => handle.num_workers(),
                 }
             }
 
             pub(crate) fn num_blocking_threads(&self) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.num_blocking_threads(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.num_blocking_threads(),
-                }
+                match_flavor!(self, Handle(handle) => handle.num_blocking_threads())
             }
 
             pub(crate) fn num_idle_blocking_threads(&self) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.num_idle_blocking_threads(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.num_idle_blocking_threads(),
-                }
+                match_flavor!(self, Handle(handle) => handle.num_idle_blocking_threads())
             }
 
             pub(crate) fn active_tasks_count(&self) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.active_tasks_count(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.active_tasks_count(),
-                }
+                match_flavor!(self, Handle(handle) => handle.active_tasks_count())
             }
 
             pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
-                match self {
-                    Handle::CurrentThread(handle) => handle.scheduler_metrics(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.scheduler_metrics(),
-                }
+                match_flavor!(self, Handle(handle) => handle.scheduler_metrics())
             }
 
             pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
-                match self {
-                    Handle::CurrentThread(handle) => handle.worker_metrics(worker),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.worker_metrics(worker),
-                }
+                match_flavor!(self, Handle(handle) => handle.worker_metrics(worker))
             }
 
             pub(crate) fn injection_queue_depth(&self) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.injection_queue_depth(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.injection_queue_depth(),
-                }
+                match_flavor!(self, Handle(handle) => handle.injection_queue_depth())
             }
 
             pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.worker_metrics(worker).queue_depth(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.worker_local_queue_depth(worker),
-                }
+                match_flavor!(self, Handle(handle) => handle.worker_local_queue_depth(worker))
             }
 
             pub(crate) fn blocking_queue_depth(&self) -> usize {
-                match self {
-                    Handle::CurrentThread(handle) => handle.blocking_queue_depth(),
-                    #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                    Handle::MultiThread(handle) => handle.blocking_queue_depth(),
-                }
+                match_flavor!(self, Handle(handle) => handle.blocking_queue_depth())
             }
         }
     }
@@ -208,17 +216,13 @@ cfg_rt! {
         pub(crate) fn expect_current_thread(&self) -> &current_thread::Context {
             match self {
                 Context::CurrentThread(context) => context,
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
                 _ => panic!("expected `CurrentThread::Context`")
             }
         }
 
         pub(crate) fn defer(&self, waker: &Waker) {
-            match self {
-                Context::CurrentThread(context) => context.defer(waker),
-                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
-                Context::MultiThread(context) => context.defer(waker),
-            }
+            match_flavor!(self, Context(context) => context.defer(waker))
         }
 
         cfg_rt_multi_thread! {
@@ -229,6 +233,16 @@ cfg_rt! {
                     _ => panic!("expected `MultiThread::Context`")
                 }
             }
+
+            cfg_unstable! {
+                #[track_caller]
+                pub(crate) fn expect_multi_thread_alt(&self) -> &multi_thread_alt::Context {
+                    match self {
+                        Context::MultiThreadAlt(context) => context,
+                        _ => panic!("expected `MultiThreadAlt::Context`")
+                    }
+                }
+            }
         }
     }
 }

tokio/src/runtime/scheduler/multi_thread/handle.rs

@@ -53,14 +53,22 @@ impl Handle {
     {
         let (handle, notified) = me.shared.owned.bind(future, me.clone(), id);
 
-        if let Some(notified) = notified {
-            me.schedule_task(notified, false);
-        }
+        me.schedule_option_task_without_yield(notified);
 
         handle
     }
 }
 
+cfg_unstable! {
+    use std::num::NonZeroU64;
+
+    impl Handle {
+        pub(crate) fn owned_id(&self) -> NonZeroU64 {
+            self.shared.owned.id
+        }
+    }
+}
+
 impl fmt::Debug for Handle {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt.debug_struct("multi_thread::Handle { ... }").finish()

tokio/src/runtime/scheduler/multi_thread/stats.rs

@@ -74,7 +74,7 @@ impl Stats {
     }
 
     pub(crate) fn submit(&mut self, to: &WorkerMetrics) {
-        self.batch.submit(to);
+        self.batch.submit(to, self.task_poll_time_ewma as u64);
     }
 
     pub(crate) fn about_to_park(&mut self) {

tokio/src/runtime/scheduler/multi_thread/worker.rs

@@ -158,7 +158,7 @@ pub(crate) struct Shared {
     idle: Idle,
 
     /// Collection of all active tasks spawned onto this executor.
-    pub(super) owned: OwnedTasks<Arc<Handle>>,
+    pub(crate) owned: OwnedTasks<Arc<Handle>>,
 
     /// Data synchronized by the scheduler mutex
     pub(super) synced: Mutex<Synced>,
@@ -323,26 +323,32 @@ where
     F: FnOnce() -> R,
 {
     // Try to steal the worker core back
-    struct Reset(coop::Budget);
+    struct Reset {
+        take_core: bool,
+        budget: coop::Budget,
+    }
 
     impl Drop for Reset {
         fn drop(&mut self) {
             with_current(|maybe_cx| {
                 if let Some(cx) = maybe_cx {
-                    let core = cx.worker.core.take();
-                    let mut cx_core = cx.core.borrow_mut();
-                    assert!(cx_core.is_none());
-                    *cx_core = core;
+                    if self.take_core {
+                        let core = cx.worker.core.take();
+                        let mut cx_core = cx.core.borrow_mut();
+                        assert!(cx_core.is_none());
+                        *cx_core = core;
+                    }
 
                     // Reset the task budget as we are re-entering the
                     // runtime.
-                    coop::set(self.0);
+                    coop::set(self.budget);
                 }
             });
         }
     }
 
     let mut had_entered = false;
+    let mut take_core = false;
 
     let setup_result = with_current(|maybe_cx| {
         match (
@@ -394,6 +400,10 @@ where
             None => return Ok(()),
         };
 
+        // We are taking the core from the context and sending it to another
+        // thread.
+        take_core = true;
+
         // The parker should be set here
         assert!(core.park.is_some());
 
@@ -420,7 +430,10 @@ where
     if had_entered {
         // Unset the current task's budget. Blocking sections are not
         // constrained by task budgets.
-        let _reset = Reset(coop::stop());
+        let _reset = Reset {
+            take_core,
+            budget: coop::stop(),
+        };
 
         crate::runtime::context::exit_runtime(f)
     } else {
@@ -774,6 +787,10 @@ impl Core {
                 cap,
             );
 
+            // Take at least one task since the first task is returned directly
+            // and nto pushed onto the local queue.
+            let n = usize::max(1, n);
+
             let mut synced = worker.handle.shared.synced.lock();
             // safety: passing in the correct `inject::Synced`.
             let mut tasks = unsafe { worker.inject().pop_n(&mut synced.inject, n) };
@@ -1011,6 +1028,12 @@ impl Handle {
         })
     }
 
+    pub(super) fn schedule_option_task_without_yield(&self, task: Option<Notified>) {
+        if let Some(task) = task {
+            self.schedule_task(task, false);
+        }
+    }
+
     fn schedule_local(&self, core: &mut Core, task: Notified, is_yield: bool) {
         core.stats.inc_local_schedule_count();
 

tokio/src/runtime/scheduler/multi_thread_alt/counters.rs

@@ -0,0 +1,208 @@
+#[cfg(tokio_internal_mt_counters)]
+mod imp {
+    use std::sync::atomic::AtomicUsize;
+    use std::sync::atomic::Ordering::Relaxed;
+    use std::time::Duration;
+    use crate::runtime::WorkerMetrics;
+
+    static NUM_MAINTENANCE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_NOTIFY_LOCAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_NOTIFY_REMOTE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_UNPARKS_LOCAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_UNPARKS_REMOTE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_SCHEDULES: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_CAPPED: AtomicUsize = AtomicUsize::new(0);
+    static NUM_STEALS: AtomicUsize = AtomicUsize::new(0);
+    static NUM_OVERFLOW: AtomicUsize = AtomicUsize::new(0);
+    static NUM_PARK: AtomicUsize = AtomicUsize::new(0);
+    static NUM_POLLS: AtomicUsize = AtomicUsize::new(0);
+    static NUM_LIFO_POLLS: AtomicUsize = AtomicUsize::new(0);
+    static NUM_REMOTE_BATCH: AtomicUsize = AtomicUsize::new(0);
+    static NUM_GLOBAL_QUEUE_INTERVAL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_NO_AVAIL_CORE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_RELAY_SEARCH: AtomicUsize = AtomicUsize::new(0);
+    static NUM_SPIN_STALL: AtomicUsize = AtomicUsize::new(0);
+    static NUM_NO_LOCAL_WORK: AtomicUsize = AtomicUsize::new(0);
+    static NUM_DRIVER_WAKE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_DRIVER_WAKE_NO_CORE: AtomicUsize = AtomicUsize::new(0);
+    static NUM_SPAWNS: AtomicUsize = AtomicUsize::new(0);
+
+    impl super::Counters {
+        pub(crate) fn dump(&self, worker_metrics: &[WorkerMetrics]) {
+            let notifies_local = NUM_NOTIFY_LOCAL.load(Relaxed);
+            let notifies_remote = NUM_NOTIFY_REMOTE.load(Relaxed);
+            let unparks_local = NUM_UNPARKS_LOCAL.load(Relaxed);
+            let unparks_remote = NUM_UNPARKS_REMOTE.load(Relaxed);
+            let maintenance = NUM_MAINTENANCE.load(Relaxed);
+            let lifo_scheds = NUM_LIFO_SCHEDULES.load(Relaxed);
+            let lifo_capped = NUM_LIFO_CAPPED.load(Relaxed);
+            let num_steals = NUM_STEALS.load(Relaxed);
+            let num_overflow = NUM_OVERFLOW.load(Relaxed);
+            let num_park = NUM_PARK.load(Relaxed);
+            let num_polls = NUM_POLLS.load(Relaxed);
+            let num_lifo_polls = NUM_LIFO_POLLS.load(Relaxed);
+            let num_remote_batch = NUM_REMOTE_BATCH.load(Relaxed);
+            let num_global_queue_interval = NUM_GLOBAL_QUEUE_INTERVAL.load(Relaxed);
+            let num_no_avail_core = NUM_NO_AVAIL_CORE.load(Relaxed);
+            let num_relay_search = NUM_RELAY_SEARCH.load(Relaxed);
+            let num_spin_stall = NUM_SPIN_STALL.load(Relaxed);
+            let num_no_local_work = NUM_NO_LOCAL_WORK.load(Relaxed);
+            let num_driver_wake = NUM_DRIVER_WAKE.load(Relaxed);
+            let num_driver_wake_no_core = NUM_DRIVER_WAKE_NO_CORE.load(Relaxed);
+            let num_spawns = NUM_SPAWNS.load(Relaxed);
+
+            println!("---");
+            println!("notifies (remote): {}", notifies_remote);
+            println!(" notifies (local): {}", notifies_local);
+            println!("  unparks (local): {}", unparks_local);
+            println!(" unparks (remote): {}", unparks_remote);
+            println!("  notify, no core: {}", num_no_avail_core);
+            println!("      maintenance: {}", maintenance);
+            println!("   LIFO schedules: {}", lifo_scheds);
+            println!("      LIFO capped: {}", lifo_capped);
+            println!("           steals: {}", num_steals);
+            println!("  queue overflows: {}", num_overflow);
+            println!("            parks: {}", num_park);
+            println!("            polls: {}", num_polls);
+            println!("     polls (LIFO): {}", num_lifo_polls);
+            println!("remote task batch: {}", num_remote_batch);
+            println!("global Q interval: {}", num_global_queue_interval);
+            println!("     relay search: {}", num_relay_search);
+            println!("       spin stall: {}", num_spin_stall);
+            println!("    no local work: {}", num_no_local_work);
+            println!("     driver wakes: {}", num_driver_wake);
+            println!("        (no core): {}", num_driver_wake_no_core);
+            println!("       num spawns: {}", num_spawns);
+            println!("");
+            println!("worker metrics:");
+
+            for (i, worker) in worker_metrics.iter().enumerate() {
+                let mean_poll_time = Duration::from_nanos(worker.mean_poll_time.load(Relaxed));
+
+                println!("");
+                println!("{}:", i);
+                println!("  mean poll time: {:?}", mean_poll_time);
+            }
+        }
+    }
+
+    pub(crate) fn inc_num_inc_notify_local() {
+        NUM_NOTIFY_LOCAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_notify_remote() {
+        NUM_NOTIFY_REMOTE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_unparks_local() {
+        NUM_UNPARKS_LOCAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_unparks_remote() {
+        NUM_UNPARKS_REMOTE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_maintenance() {
+        NUM_MAINTENANCE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_lifo_schedules() {
+        NUM_LIFO_SCHEDULES.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_lifo_capped() {
+        NUM_LIFO_CAPPED.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_steals() {
+        NUM_STEALS.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_overflows() {
+        NUM_OVERFLOW.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_parks() {
+        NUM_PARK.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_polls() {
+        NUM_POLLS.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_lifo_polls() {
+        NUM_LIFO_POLLS.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_remote_batch() {
+        NUM_REMOTE_BATCH.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_global_queue_interval() {
+        NUM_GLOBAL_QUEUE_INTERVAL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_notify_no_core() {
+        NUM_NO_AVAIL_CORE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_relay_search() {
+        NUM_RELAY_SEARCH.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_spin_stall() {
+        NUM_SPIN_STALL.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_no_local_work() {
+        NUM_NO_LOCAL_WORK.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_driver_wakes() {
+        NUM_DRIVER_WAKE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_driver_wakes_no_core() {
+        NUM_DRIVER_WAKE_NO_CORE.fetch_add(1, Relaxed);
+    }
+
+    pub(crate) fn inc_num_spawns() {
+        NUM_SPAWNS.fetch_add(1, Relaxed);
+    }
+}
+
+#[cfg(not(tokio_internal_mt_counters))]
+mod imp {
+    use crate::runtime::WorkerMetrics;
+
+    pub(crate) fn inc_num_inc_notify_local() {}
+    pub(crate) fn inc_num_notify_remote() {}
+    pub(crate) fn inc_num_unparks_local() {}
+    pub(crate) fn inc_num_unparks_remote() {}
+    pub(crate) fn inc_num_maintenance() {}
+    pub(crate) fn inc_lifo_schedules() {}
+    pub(crate) fn inc_lifo_capped() {}
+    pub(crate) fn inc_num_steals() {}
+    pub(crate) fn inc_num_overflows() {}
+    pub(crate) fn inc_num_parks() {}
+    pub(crate) fn inc_num_polls() {}
+    pub(crate) fn inc_num_lifo_polls() {}
+    pub(crate) fn inc_num_remote_batch() {}
+    pub(crate) fn inc_global_queue_interval() {}
+    pub(crate) fn inc_notify_no_core() {}
+    pub(crate) fn inc_num_relay_search() {}
+    pub(crate) fn inc_num_spin_stall() {}
+    pub(crate) fn inc_num_no_local_work() {}
+    pub(crate) fn inc_num_driver_wakes() {}
+    pub(crate) fn inc_num_driver_wakes_no_core() {}
+    pub(crate) fn inc_num_spawns() {}
+
+    impl super::Counters {
+        pub(crate) fn dump(&self, _: &[WorkerMetrics]) {}
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Counters;
+
+pub(super) use imp::*;

tokio/src/runtime/scheduler/multi_thread_alt/handle.rs

@@ -0,0 +1,77 @@
+use crate::future::Future;
+use crate::loom::sync::Arc;
+use crate::runtime::scheduler::multi_thread_alt::worker;
+use crate::runtime::{
+    blocking, driver,
+    task::{self, JoinHandle},
+};
+use crate::util::RngSeedGenerator;
+
+use std::fmt;
+
+cfg_metrics! {
+    mod metrics;
+}
+
+/// Handle to the multi thread scheduler
+pub(crate) struct Handle {
+    /// Task spawner
+    pub(super) shared: worker::Shared,
+
+    /// Resource driver handles
+    pub(crate) driver: driver::Handle,
+
+    /// Blocking pool spawner
+    pub(crate) blocking_spawner: blocking::Spawner,
+
+    /// Current random number generator seed
+    pub(crate) seed_generator: RngSeedGenerator,
+}
+
+impl Handle {
+    /// Spawns a future onto the thread pool
+    pub(crate) fn spawn<F>(me: &Arc<Self>, future: F, id: task::Id) -> JoinHandle<F::Output>
+    where
+        F: crate::future::Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        Self::bind_new_task(me, future, id)
+    }
+
+    pub(crate) fn shutdown(&self) {
+        self.shared.close(self);
+        self.driver.unpark();
+    }
+
+    pub(super) fn bind_new_task<T>(me: &Arc<Self>, future: T, id: task::Id) -> JoinHandle<T::Output>
+    where
+        T: Future + Send + 'static,
+        T::Output: Send + 'static,
+    {
+        let (handle, notified) = me.shared.owned.bind(future, me.clone(), id);
+
+        super::counters::inc_num_spawns();
+
+        if let Some(notified) = notified {
+            me.shared.schedule_task(notified, false);
+        }
+
+        handle
+    }
+}
+
+cfg_unstable! {
+    use std::num::NonZeroU64;
+
+    impl Handle {
+        pub(crate) fn owned_id(&self) -> NonZeroU64 {
+            self.shared.owned.id
+        }
+    }
+}
+
+impl fmt::Debug for Handle {
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt.debug_struct("multi_thread::Handle { ... }").finish()
+    }
+}

tokio/src/runtime/scheduler/multi_thread_alt/handle/metrics.rs

@@ -0,0 +1,41 @@
+use super::Handle;
+
+use crate::runtime::{SchedulerMetrics, WorkerMetrics};
+
+impl Handle {
+    pub(crate) fn num_workers(&self) -> usize {
+        self.shared.worker_metrics.len()
+    }
+
+    pub(crate) fn num_blocking_threads(&self) -> usize {
+        self.blocking_spawner.num_threads()
+    }
+
+    pub(crate) fn num_idle_blocking_threads(&self) -> usize {
+        self.blocking_spawner.num_idle_threads()
+    }
+
+    pub(crate) fn active_tasks_count(&self) -> usize {
+        self.shared.owned.active_tasks_count()
+    }
+
+    pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
+        &self.shared.scheduler_metrics
+    }
+
+    pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
+        &self.shared.worker_metrics[worker]
+    }
+
+    pub(crate) fn injection_queue_depth(&self) -> usize {
+        self.shared.injection_queue_depth()
+    }
+
+    pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+        self.shared.worker_local_queue_depth(worker)
+    }
+
+    pub(crate) fn blocking_queue_depth(&self) -> usize {
+        self.blocking_spawner.queue_depth()
+    }
+}

tokio/src/runtime/scheduler/multi_thread_alt/handle/taskdump.rs

@@ -0,0 +1,26 @@
+use super::Handle;
+
+use crate::runtime::Dump;
+
+impl Handle {
+    pub(crate) async fn dump(&self) -> Dump {
+        let trace_status = &self.shared.trace_status;
+
+        // If a dump is in progress, block.
+        trace_status.start_trace_request(&self).await;
+
+        let result = loop {
+            if let Some(result) = trace_status.take_result() {
+                break result;
+            } else {
+                self.notify_all();
+                trace_status.result_ready.notified().await;
+            }
+        };
+
+        // Allow other queued dumps to proceed.
+        trace_status.end_trace_request(&self).await;
+
+        result
+    }
+}

tokio/src/runtime/scheduler/multi_thread_alt/idle.rs

@@ -0,0 +1,423 @@
+//! Coordinates idling workers
+
+#![allow(dead_code)]
+
+use crate::loom::sync::atomic::{AtomicBool, AtomicUsize};
+use crate::loom::sync::MutexGuard;
+use crate::runtime::scheduler::multi_thread_alt::{worker, Core, Handle, Shared};
+
+use std::sync::atomic::Ordering::{AcqRel, Acquire, Release};
+
+pub(super) struct Idle {
+    /// Number of searching cores
+    num_searching: AtomicUsize,
+
+    /// Number of idle cores
+    num_idle: AtomicUsize,
+
+    /// Map of idle cores
+    idle_map: IdleMap,
+
+    /// Used to catch false-negatives when waking workers
+    needs_searching: AtomicBool,
+
+    /// Total number of cores
+    num_cores: usize,
+}
+
+pub(super) struct IdleMap {
+    chunks: Vec<AtomicUsize>,
+}
+
+pub(super) struct Snapshot {
+    chunks: Vec<usize>,
+}
+
+/// Data synchronized by the scheduler mutex
+pub(super) struct Synced {
+    /// Worker IDs that are currently sleeping
+    sleepers: Vec<usize>,
+
+    /// Cores available for workers
+    available_cores: Vec<Box<Core>>,
+}
+
+impl Idle {
+    pub(super) fn new(cores: Vec<Box<Core>>, num_workers: usize) -> (Idle, Synced) {
+        let idle = Idle {
+            num_searching: AtomicUsize::new(0),
+            num_idle: AtomicUsize::new(cores.len()),
+            idle_map: IdleMap::new(&cores),
+            needs_searching: AtomicBool::new(false),
+            num_cores: cores.len(),
+        };
+
+        let synced = Synced {
+            sleepers: Vec::with_capacity(num_workers),
+            available_cores: cores,
+        };
+
+        (idle, synced)
+    }
+
+    pub(super) fn needs_searching(&self) -> bool {
+        self.needs_searching.load(Acquire)
+    }
+
+    pub(super) fn num_idle(&self, synced: &Synced) -> usize {
+        #[cfg(not(loom))]
+        debug_assert_eq!(synced.available_cores.len(), self.num_idle.load(Acquire));
+        synced.available_cores.len()
+    }
+
+    pub(super) fn num_searching(&self) -> usize {
+        self.num_searching.load(Acquire)
+    }
+
+    pub(super) fn snapshot(&self, snapshot: &mut Snapshot) {
+        snapshot.update(&self.idle_map)
+    }
+
+    /// Try to acquire an available core
+    pub(super) fn try_acquire_available_core(&self, synced: &mut Synced) -> Option<Box<Core>> {
+        let ret = synced.available_cores.pop();
+
+        if let Some(core) = &ret {
+            // Decrement the number of idle cores
+            let num_idle = self.num_idle.load(Acquire) - 1;
+            debug_assert_eq!(num_idle, synced.available_cores.len());
+            self.num_idle.store(num_idle, Release);
+
+            self.idle_map.unset(core.index);
+            debug_assert!(self.idle_map.matches(&synced.available_cores));
+        }
+
+        ret
+    }
+
+    /// We need at least one searching worker
+    pub(super) fn notify_local(&self, shared: &Shared) {
+        if self.num_searching.load(Acquire) != 0 {
+            // There already is a searching worker. Note, that this could be a
+            // false positive. However, because this method is called **from** a
+            // worker, we know that there is at least one worker currently
+            // awake, so the scheduler won't deadlock.
+            return;
+        }
+
+        if self.num_idle.load(Acquire) == 0 {
+            self.needs_searching.store(true, Release);
+            return;
+        }
+
+        // There aren't any searching workers. Try to initialize one
+        if self
+            .num_searching
+            .compare_exchange(0, 1, AcqRel, Acquire)
+            .is_err()
+        {
+            // Failing the compare_exchange means another thread concurrently
+            // launched a searching worker.
+            return;
+        }
+
+        super::counters::inc_num_unparks_local();
+
+        // Acquire the lock
+        let synced = shared.synced.lock();
+        self.notify_synced(synced, shared);
+    }
+
+    /// Notifies a single worker
+    pub(super) fn notify_remote(&self, synced: MutexGuard<'_, worker::Synced>, shared: &Shared) {
+        if synced.idle.sleepers.is_empty() {
+            self.needs_searching.store(true, Release);
+            return;
+        }
+
+        // We need to establish a stronger barrier than with `notify_local`
+        self.num_searching.fetch_add(1, AcqRel);
+
+        self.notify_synced(synced, shared);
+    }
+
+    /// Notify a worker while synced
+    fn notify_synced(&self, mut synced: MutexGuard<'_, worker::Synced>, shared: &Shared) {
+        // Find a sleeping worker
+        if let Some(worker) = synced.idle.sleepers.pop() {
+            // Find an available core
+            if let Some(mut core) = self.try_acquire_available_core(&mut synced.idle) {
+                debug_assert!(!core.is_searching);
+                core.is_searching = true;
+
+                // Assign the core to the worker
+                synced.assigned_cores[worker] = Some(core);
+
+                // Drop the lock before notifying the condvar.
+                drop(synced);
+
+                super::counters::inc_num_unparks_remote();
+
+                // Notify the worker
+                shared.condvars[worker].notify_one();
+                return;
+            } else {
+                synced.idle.sleepers.push(worker);
+            }
+        }
+
+        super::counters::inc_notify_no_core();
+
+        // Set the `needs_searching` flag, this happens *while* the lock is held.
+        self.needs_searching.store(true, Release);
+        self.num_searching.fetch_sub(1, Release);
+
+        // Explicit mutex guard drop to show that holding the guard to this
+        // point is significant. `needs_searching` and `num_searching` must be
+        // updated in the critical section.
+        drop(synced);
+    }
+
+    pub(super) fn notify_mult(
+        &self,
+        synced: &mut worker::Synced,
+        workers: &mut Vec<usize>,
+        num: usize,
+    ) {
+        debug_assert!(workers.is_empty());
+
+        for _ in 0..num {
+            if let Some(worker) = synced.idle.sleepers.pop() {
+                // TODO: can this be switched to use next_available_core?
+                if let Some(core) = synced.idle.available_cores.pop() {
+                    debug_assert!(!core.is_searching);
+
+                    self.idle_map.unset(core.index);
+
+                    synced.assigned_cores[worker] = Some(core);
+
+                    workers.push(worker);
+
+                    continue;
+                } else {
+                    synced.idle.sleepers.push(worker);
+                }
+            }
+
+            break;
+        }
+
+        if !workers.is_empty() {
+            debug_assert!(self.idle_map.matches(&synced.idle.available_cores));
+            let num_idle = synced.idle.available_cores.len();
+            self.num_idle.store(num_idle, Release);
+        } else {
+            #[cfg(not(loom))]
+            debug_assert_eq!(
+                synced.idle.available_cores.len(),
+                self.num_idle.load(Acquire)
+            );
+            self.needs_searching.store(true, Release);
+        }
+    }
+
+    pub(super) fn shutdown(&self, synced: &mut worker::Synced, shared: &Shared) {
+        // Wake every sleeping worker and assign a core to it. There may not be
+        // enough sleeping workers for all cores, but other workers will
+        // eventually find the cores and shut them down.
+        while !synced.idle.sleepers.is_empty() && !synced.idle.available_cores.is_empty() {
+            let worker = synced.idle.sleepers.pop().unwrap();
+            let core = self.try_acquire_available_core(&mut synced.idle).unwrap();
+
+            synced.assigned_cores[worker] = Some(core);
+            shared.condvars[worker].notify_one();
+        }
+
+        debug_assert!(self.idle_map.matches(&synced.idle.available_cores));
+
+        // Wake up any other workers
+        while let Some(index) = synced.idle.sleepers.pop() {
+            shared.condvars[index].notify_one();
+        }
+    }
+
+    pub(super) fn shutdown_unassigned_cores(&self, handle: &Handle, shared: &Shared) {
+        // If there are any remaining cores, shut them down here.
+        //
+        // This code is a bit convoluted to avoid lock-reentry.
+        while let Some(core) = {
+            let mut synced = shared.synced.lock();
+            self.try_acquire_available_core(&mut synced.idle)
+        } {
+            shared.shutdown_core(handle, core);
+        }
+    }
+
+    /// The worker releases the given core, making it available to other workers
+    /// that are waiting.
+    pub(super) fn release_core(&self, synced: &mut worker::Synced, core: Box<Core>) {
+        // The core should not be searching at this point
+        debug_assert!(!core.is_searching);
+
+        // Check that there are no pending tasks in the global queue
+        debug_assert!(synced.inject.is_empty());
+
+        let num_idle = synced.idle.available_cores.len();
+        #[cfg(not(loom))]
+        debug_assert_eq!(num_idle, self.num_idle.load(Acquire));
+
+        self.idle_map.set(core.index);
+
+        // Store the core in the list of available cores
+        synced.idle.available_cores.push(core);
+
+        debug_assert!(self.idle_map.matches(&synced.idle.available_cores));
+
+        // Update `num_idle`
+        self.num_idle.store(num_idle + 1, Release);
+    }
+
+    pub(super) fn transition_worker_to_parked(&self, synced: &mut worker::Synced, index: usize) {
+        // Store the worker index in the list of sleepers
+        synced.idle.sleepers.push(index);
+
+        // The worker's assigned core slot should be empty
+        debug_assert!(synced.assigned_cores[index].is_none());
+    }
+
+    pub(super) fn try_transition_worker_to_searching(&self, core: &mut Core) {
+        debug_assert!(!core.is_searching);
+
+        let num_searching = self.num_searching.load(Acquire);
+        let num_idle = self.num_idle.load(Acquire);
+
+        if 2 * num_searching >= self.num_cores - num_idle {
+            return;
+        }
+
+        self.transition_worker_to_searching(core);
+    }
+
+    /// Needs to happen while synchronized in order to avoid races
+    pub(super) fn transition_worker_to_searching_if_needed(
+        &self,
+        _synced: &mut Synced,
+        core: &mut Core,
+    ) -> bool {
+        if self.needs_searching.load(Acquire) {
+            // Needs to be called while holding the lock
+            self.transition_worker_to_searching(core);
+            true
+        } else {
+            false
+        }
+    }
+
+    pub(super) fn transition_worker_to_searching(&self, core: &mut Core) {
+        core.is_searching = true;
+        self.num_searching.fetch_add(1, AcqRel);
+        self.needs_searching.store(false, Release);
+    }
+
+    /// A lightweight transition from searching -> running.
+    ///
+    /// Returns `true` if this is the final searching worker. The caller
+    /// **must** notify a new worker.
+    pub(super) fn transition_worker_from_searching(&self) -> bool {
+        let prev = self.num_searching.fetch_sub(1, AcqRel);
+        debug_assert!(prev > 0);
+
+        prev == 1
+    }
+}
+
+const BITS: usize = usize::BITS as usize;
+const BIT_MASK: usize = (usize::BITS - 1) as usize;
+
+impl IdleMap {
+    fn new(cores: &[Box<Core>]) -> IdleMap {
+        let ret = IdleMap::new_n(num_chunks(cores.len()));
+        ret.set_all(cores);
+
+        ret
+    }
+
+    fn new_n(n: usize) -> IdleMap {
+        let chunks = (0..n).map(|_| AtomicUsize::new(0)).collect();
+        IdleMap { chunks }
+    }
+
+    fn set(&self, index: usize) {
+        let (chunk, mask) = index_to_mask(index);
+        let prev = self.chunks[chunk].load(Acquire);
+        let next = prev | mask;
+        self.chunks[chunk].store(next, Release);
+    }
+
+    fn set_all(&self, cores: &[Box<Core>]) {
+        for core in cores {
+            self.set(core.index);
+        }
+    }
+
+    fn unset(&self, index: usize) {
+        let (chunk, mask) = index_to_mask(index);
+        let prev = self.chunks[chunk].load(Acquire);
+        let next = prev & !mask;
+        self.chunks[chunk].store(next, Release);
+    }
+
+    fn matches(&self, idle_cores: &[Box<Core>]) -> bool {
+        let expect = IdleMap::new_n(self.chunks.len());
+        expect.set_all(idle_cores);
+
+        for (i, chunk) in expect.chunks.iter().enumerate() {
+            if chunk.load(Acquire) != self.chunks[i].load(Acquire) {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+impl Snapshot {
+    pub(crate) fn new(idle: &Idle) -> Snapshot {
+        let chunks = vec![0; idle.idle_map.chunks.len()];
+        let mut ret = Snapshot { chunks };
+        ret.update(&idle.idle_map);
+        ret
+    }
+
+    fn update(&mut self, idle_map: &IdleMap) {
+        for i in 0..self.chunks.len() {
+            self.chunks[i] = idle_map.chunks[i].load(Acquire);
+        }
+    }
+
+    pub(super) fn is_idle(&self, index: usize) -> bool {
+        let (chunk, mask) = index_to_mask(index);
+        debug_assert!(
+            chunk < self.chunks.len(),
+            "index={}; chunks={}",
+            index,
+            self.chunks.len()
+        );
+        self.chunks[chunk] & mask == mask
+    }
+}
+
+fn num_chunks(max_cores: usize) -> usize {
+    (max_cores / BITS) + 1
+}
+
+fn index_to_mask(index: usize) -> (usize, usize) {
+    let mask = 1 << (index & BIT_MASK);
+    let chunk = index / BITS;
+
+    (chunk, mask)
+}
+
+fn num_active_workers(synced: &Synced) -> usize {
+    synced.available_cores.capacity() - synced.available_cores.len()
+}