chore: prepare Tokio v1.24.1 (#5357)

This should let CI to pass.

.cirrus.yml

@@ -1,5 +1,5 @@
 freebsd_instance:
-  image: freebsd-12-3-release-amd64
+  image: freebsd-12-4-release-amd64
 env:
   RUST_STABLE: stable
   RUST_NIGHTLY: nightly-2022-10-25

.github/workflows/audit.yml

@@ -15,15 +15,16 @@ permissions:
 jobs:
   security-audit:
     permissions:
-      checks: write  # for actions-rs/audit-check to create check
+      checks: write  # for rustsec/audit-check to create check
       contents: read  # for actions/checkout to fetch code
-      issues: write  # for actions-rs/audit-check to create issues
+      issues: write  # for rustsec/audit-check to create issues
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
       - uses: actions/checkout@v3
 
       - name: Audit Check
-        uses: actions-rs/audit-check@v1
+        # https://github.com/rustsec/audit-check/issues/2
+        uses: rustsec/audit-check@master
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ env:
   # Change to specific Rust release to pin
   rust_stable: stable
   rust_nightly: nightly-2022-11-03
-  rust_clippy: 1.60.0
+  rust_clippy: 1.65.0
   # When updating this, also update:
   # - README.md
   # - tokio/README.md
@@ -73,13 +73,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-            override: true
       - name: Install Rust
         run: rustup update stable
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
 
@@ -123,11 +122,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-            override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Enable parking_lot send_guard feature
         # Inserts the line "plsend = ["parking_lot/send_guard"]" right after [features]
         run: sed -i '/\[features\]/a plsend = ["parking_lot/send_guard"]' tokio/Cargo.toml
@@ -140,11 +138,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-            override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
 
       - name: Install Valgrind
         uses: taiki-e/install-action@valgrind
@@ -179,11 +176,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-            override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       # Run `tokio` with "unstable" cfg flag.
       - name: test tokio full --cfg unstable
         run: cargo test --all-features
@@ -200,12 +196,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
           components: miri
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: miri
         # Many of tests in tokio/tests and doctests use #[tokio::test] or
         # #[tokio::main] that calls epoll_create1 that Miri does not support.
@@ -224,11 +219,10 @@ jobs:
         # Required to resolve symbols in sanitizer output
         run: sudo apt-get install -y llvm
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: asan
         run: cargo test --workspace --all-features --target x86_64-unknown-linux-gnu --tests -- --test-threads 1
         env:
@@ -250,16 +244,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
           target: ${{ matrix.target }}
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: check
-          args: --workspace --all-features --target ${{ matrix.target }}
+      - name: Install cross
+        uses: taiki-e/install-action@cross
+      - run: cross check --workspace --all-features --target ${{ matrix.target }}
         env:
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings
 
@@ -268,26 +259,35 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target:
-          - i686-unknown-linux-gnu
-          - arm-unknown-linux-gnueabihf
-          - armv7-unknown-linux-gnueabihf
-          - aarch64-unknown-linux-gnu
+        include:
+          - target: i686-unknown-linux-gnu
+          - target: arm-unknown-linux-gnueabihf
+          - target: armv7-unknown-linux-gnueabihf
+          - target: aarch64-unknown-linux-gnu
+
+          # Run a platform without AtomicU64 and no const Mutex::new
+          - target: arm-unknown-linux-gnueabihf
+            rustflags: --cfg tokio_no_const_mutex_new
     steps:
       - uses: actions/checkout@v3
-      - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+      - name: Install Rust stable
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
           target: ${{ matrix.target }}
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: test
-          args: -p tokio --all-features --target ${{ matrix.target }} --tests
+      - name: Install cross
+        uses: taiki-e/install-action@cross
+      # First run with all features (including parking_lot)
+      - run: cross test -p tokio --all-features --target ${{ matrix.target }} --tests
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 ${{ matrix.rustflags }}
+      # Now run without parking_lot
+      - name: Remove `parking_lot` from `full` feature
+        run: sed -i '0,/parking_lot/{/parking_lot/d;}' tokio/Cargo.toml
+      # The `tokio_no_parking_lot` cfg is here to ensure the `sed` above does not silently break.
+      - run: cross test -p tokio --features full,test-util --target ${{ matrix.target }} --tests
+        env:
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_parking_lot ${{ matrix.rustflags }}
 
   # See https://github.com/tokio-rs/tokio/issues/5187
   no-atomic-u64:
@@ -296,11 +296,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
           components: rust-src
-          override: true
       # Install linker and libraries for i686-unknown-linux-gnu
       - uses: taiki-e/setup-cross-toolchain-action@v1
         with:
@@ -315,12 +314,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
           target: ${{ matrix.target }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
       - name: check --feature-powerset
@@ -337,11 +335,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_min }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_min }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       # First compile just the main tokio crate with minrust and newest version
       # of all dependencies, then pin once_cell and compile the rest of the
       # crates with the pinned once_cell version.
@@ -351,10 +348,14 @@ jobs:
       # release.
       - name: "check -p tokio --all-features"
         run: cargo check -p tokio --all-features
+        env:
+          RUSTFLAGS: "" # remove -Dwarnings
       - name: "pin once_cell version"
         run: cargo update -p once_cell --precise 1.14.0
       - name: "check --workspace --all-features"
         run: cargo check --workspace --all-features
+        env:
+          RUSTFLAGS: "" # remove -Dwarnings
 
   minimal-versions:
     name: minimal-versions
@@ -362,11 +363,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
       - name: "check --all-features -Z minimal-versions"
@@ -394,12 +394,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
-          override: true
           components: rustfmt
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       # Check fmt
       - name: "rustfmt --check"
         # Workaround for rust-lang/cargo#7732
@@ -415,12 +414,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_clippy }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_clippy }}
-          override: true
           components: clippy
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       # Run clippy
       - name: "clippy --all"
         run: cargo clippy --all --tests --all-features
@@ -431,11 +429,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: "doc --lib --all-features"
         run: cargo doc --lib --no-deps --all-features --document-private-items
         env:
@@ -448,11 +445,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: build --cfg loom
         run: cargo test --no-run --lib --features full
         working-directory: tokio
@@ -483,11 +479,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Test hyper
         run: |
           set -x
@@ -511,12 +506,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_nightly }}
           target: x86_64-fortanix-unknown-sgx
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       # NOTE: Currently the only test we can run is to build tokio with rt and sync features.
       - name: build tokio
         run: cargo build --target x86_64-fortanix-unknown-sgx --features rt,sync
@@ -528,11 +522,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: test tokio
@@ -545,11 +538,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.rust_stable }}
-          override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
 
       # Install dependencies
       - name: Install cargo-hack
@@ -597,17 +589,16 @@ jobs:
           - ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Install Rust nightly-2022-07-25
-        uses: actions-rs/toolchain@v1
+      - name: Install Rust nightly-2022-11-16
+        uses: dtolnay/rust-toolchain@master
         with:
           # `check-external-types` requires a specific Rust nightly version. See
           # the README for details: https://github.com/awslabs/cargo-check-external-types
-          toolchain: nightly-2022-07-25
-          override: true
-      - uses: Swatinem/rust-cache@v1
+          toolchain: nightly-2022-11-16
+      - uses: Swatinem/rust-cache@v2
       - name: check-external-types
         run: |
           set -x
-          cargo install cargo-check-external-types --locked --version 0.1.5
+          cargo install cargo-check-external-types --locked --version 0.1.6
           cargo check-external-types --all-features --config external-types.toml
         working-directory: tokio

.github/workflows/loom.yml

@@ -34,15 +34,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-            override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: loom ${{ matrix.scope }}
         run: cargo test --lib --release --features full -- --nocapture $SCOPE
         working-directory: tokio
         env:
           RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings
           LOOM_MAX_PREEMPTIONS: 2
+          LOOM_MAX_BRANCHES: 10000
           SCOPE: ${{ matrix.scope }}

.github/workflows/pr-audit.yml

@@ -19,17 +19,10 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install cargo-audit
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: cargo-audit
+        run: cargo install cargo-audit
 
       - name: Generate lockfile
-        uses: actions-rs/cargo@v1
-        with:
-          command: generate-lockfile
+        run: cargo generate-lockfile
 
       - name: Audit dependencies
-        uses: actions-rs/cargo@v1
-        with:
-          command: audit
+        run: cargo audit

.github/workflows/stress-test.yml

@@ -25,11 +25,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
             toolchain: ${{ env.rust_stable }}
-            override: true
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Install Valgrind
         uses: taiki-e/install-action@valgrind
 

Cross.toml

@@ -1,4 +1,5 @@
 [build.env]
 passthrough = [
     "RUSTFLAGS",
+    "RUST_BACKTRACE",
 ]

README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.22.0", features = ["full"] }
+tokio = { version = "1.24.1", features = ["full"] }
 ```
 Then, on your main.rs:
 

tokio-macros/src/select.rs

@@ -100,10 +100,10 @@ fn clean_pattern(pat: &mut syn::Pat) {
         }
         syn::Pat::Reference(reference) => {
             reference.mutability = None;
-            clean_pattern(&mut *reference.pat);
+            clean_pattern(&mut reference.pat);
         }
         syn::Pat::Type(type_pat) => {
-            clean_pattern(&mut *type_pat.pat);
+            clean_pattern(&mut type_pat.pat);
         }
         _ => {}
     }

tokio-stream/src/stream_ext/then.rs

@@ -72,7 +72,7 @@ where
     }
 
     fn size_hint(&self) -> (usize, Option<usize>) {
-        let future_len = if self.future.is_some() { 1 } else { 0 };
+        let future_len = usize::from(self.future.is_some());
         let (lower, upper) = self.stream.size_hint();
 
         let lower = lower.saturating_add(future_len);

tokio-test/Cargo.toml

@@ -19,7 +19,7 @@ categories = ["asynchronous", "testing"]
 [dependencies]
 tokio = { version = "1.2.0", path = "../tokio", features = ["rt", "sync", "time", "test-util"] }
 tokio-stream = { version = "0.1.1", path = "../tokio-stream" }
-async-stream = "0.3"
+async-stream = "0.3.3"
 
 bytes = "1.0.0"
 futures-core = "0.3.0"

tokio-util/src/codec/framed.rs

@@ -368,10 +368,7 @@ pub struct FramedParts<T, U> {
 
 impl<T, U> FramedParts<T, U> {
     /// Create a new, default, `FramedParts`
-    pub fn new<I>(io: T, codec: U) -> FramedParts<T, U>
-    where
-        U: Encoder<I>,
-    {
+    pub fn new(io: T, codec: U) -> FramedParts<T, U> {
         FramedParts {
             io,
             codec,

tokio-util/src/io/sync_bridge.rs

@@ -1,5 +1,7 @@
-use std::io::{Read, Write};
-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use std::io::{BufRead, Read, Write};
+use tokio::io::{
+    AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt,
+};
 
 /// Use a [`tokio::io::AsyncRead`] synchronously as a [`std::io::Read`] or
 /// a [`tokio::io::AsyncWrite`] as a [`std::io::Write`].
@@ -9,6 +11,28 @@ pub struct SyncIoBridge<T> {
     rt: tokio::runtime::Handle,
 }
 
+impl<T: AsyncBufRead + Unpin> BufRead for SyncIoBridge<T> {
+    fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
+        let src = &mut self.src;
+        self.rt.block_on(AsyncBufReadExt::fill_buf(src))
+    }
+
+    fn consume(&mut self, amt: usize) {
+        let src = &mut self.src;
+        AsyncBufReadExt::consume(src, amt)
+    }
+
+    fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> std::io::Result<usize> {
+        let src = &mut self.src;
+        self.rt
+            .block_on(AsyncBufReadExt::read_until(src, byte, buf))
+    }
+    fn read_line(&mut self, buf: &mut String) -> std::io::Result<usize> {
+        let src = &mut self.src;
+        self.rt.block_on(AsyncBufReadExt::read_line(src, buf))
+    }
+}
+
 impl<T: AsyncRead + Unpin> Read for SyncIoBridge<T> {
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
         let src = &mut self.src;

tokio-util/src/sync/poll_semaphore.rs

@@ -166,6 +166,6 @@ impl fmt::Debug for PollSemaphore {
 
 impl AsRef<Semaphore> for PollSemaphore {
     fn as_ref(&self) -> &Semaphore {
-        &*self.semaphore
+        &self.semaphore
     }
 }

tokio-util/tests/spawn_pinned.rs

@@ -82,8 +82,8 @@ async fn task_panic_propagates() {
     assert!(result.is_err());
     let error = result.unwrap_err();
     assert!(error.is_panic());
-    let panic_str: &str = *error.into_panic().downcast().unwrap();
-    assert_eq!(panic_str, "Test panic");
+    let panic_str = error.into_panic().downcast::<&'static str>().unwrap();
+    assert_eq!(*panic_str, "Test panic");
 
     // Trying again with a "safe" task still works
     let join_handle = pool.spawn_pinned(|| async { "test" });
@@ -108,8 +108,8 @@ async fn callback_panic_does_not_kill_worker() {
     assert!(result.is_err());
     let error = result.unwrap_err();
     assert!(error.is_panic());
-    let panic_str: &str = *error.into_panic().downcast().unwrap();
-    assert_eq!(panic_str, "Test panic");
+    let panic_str = error.into_panic().downcast::<&'static str>().unwrap();
+    assert_eq!(*panic_str, "Test panic");
 
     // Trying again with a "safe" callback works
     let join_handle = pool.spawn_pinned(|| async { "test" });

tokio-util/tests/time_delay_queue.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::blacklisted_name)]
+#![allow(clippy::disallowed_names)]
 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
 

tokio/CHANGELOG.md

@@ -1,3 +1,67 @@
+# 1.24.1 (January 6, 2022)
+
+This release fixes a compilation failure on targets without `AtomicU64` when using rustc older than 1.63. ([#5356])
+
+[#5356]: https://github.com/tokio-rs/tokio/pull/5356
+
+# 1.24.0 (January 5, 2022)
+
+### Fixed
+ - rt: improve native `AtomicU64` support detection ([#5284])
+
+### Added
+ - rt: add configuration option for max number of I/O events polled from the OS
+   per tick ([#5186])
+ - rt: add an environment variable for configuring the default number of worker
+   threads per runtime instance ([#4250])
+
+### Changed
+ - sync: reduce MPSC channel stack usage ([#5294])
+ - io: reduce lock contention in I/O operations  ([#5300])
+ - fs: speed up `read_dir()` by chunking operations ([#5309])
+ - rt: use internal `ThreadId` implementation ([#5329])
+ - test: don't auto-advance time when a `spawn_blocking` task is running ([#5115])
+
+[#5186]: https://github.com/tokio-rs/tokio/pull/5186
+[#5294]: https://github.com/tokio-rs/tokio/pull/5294
+[#5284]: https://github.com/tokio-rs/tokio/pull/5284
+[#4250]: https://github.com/tokio-rs/tokio/pull/4250
+[#5300]: https://github.com/tokio-rs/tokio/pull/5300
+[#5329]: https://github.com/tokio-rs/tokio/pull/5329
+[#5115]: https://github.com/tokio-rs/tokio/pull/5115
+[#5309]: https://github.com/tokio-rs/tokio/pull/5309
+
+# 1.23.1 (January 4, 2022)
+
+This release forward ports changes from 1.18.4.
+
+### Fixed
+
+- net: fix Windows named pipe server builder to maintain option when toggling
+  pipe mode ([#5336]).
+
+[#5336]: https://github.com/tokio-rs/tokio/pull/5336
+
+# 1.23.0 (December 5, 2022)
+
+### Fixed
+
+ - net: fix Windows named pipe connect ([#5208])
+ - io: support vectored writes for `ChildStdin` ([#5216])
+ - io: fix `async fn ready()` false positive for OS-specific events ([#5231])
+
+ ### Changed
+ - runtime: `yield_now` defers task until after driver poll ([#5223])
+ - runtime: reduce amount of codegen needed per spawned task ([#5213])
+ - windows: replace `winapi` dependency with `windows-sys` ([#5204])
+
+ [#5208]: https://github.com/tokio-rs/tokio/pull/5208
+ [#5216]: https://github.com/tokio-rs/tokio/pull/5216
+ [#5213]: https://github.com/tokio-rs/tokio/pull/5213
+ [#5204]: https://github.com/tokio-rs/tokio/pull/5204
+ [#5223]: https://github.com/tokio-rs/tokio/pull/5223
+ [#5231]: https://github.com/tokio-rs/tokio/pull/5231
+
 # 1.22.0 (November 17, 2022)
 
 ### Added
@@ -242,6 +306,17 @@ wasm32-wasi target is given unstable support for the `net` feature.
 [#4956]: https://github.com/tokio-rs/tokio/pull/4956
 [#4959]: https://github.com/tokio-rs/tokio/pull/4959
 
+# 1.20.3 (January 3, 2022)
+
+This release forward ports changes from 1.18.4.
+
+### Fixed
+
+- net: fix Windows named pipe server builder to maintain option when toggling
+  pipe mode ([#5336]).
+
+[#5336]: https://github.com/tokio-rs/tokio/pull/5336
+
 # 1.20.2 (September 27, 2022)
 
 This release removes the dependency on the `once_cell` crate to restore the MSRV
@@ -367,6 +442,15 @@ This release fixes a bug in `Notified::enable`. ([#4747])
 [#4729]: https://github.com/tokio-rs/tokio/pull/4729
 [#4739]: https://github.com/tokio-rs/tokio/pull/4739
 
+# 1.18.4 (January 3, 2022)
+
+### Fixed
+
+- net: fix Windows named pipe server builder to maintain option when toggling
+  pipe mode ([#5336]).
+
+[#5336]: https://github.com/tokio-rs/tokio/pull/5336
+
 # 1.18.3 (September 27, 2022)
 
 This release removes the dependency on the `once_cell` crate to restore the MSRV

tokio/Cargo.toml

@@ -6,7 +6,7 @@ name = "tokio"
 #   - README.md
 # - Update CHANGELOG.md.
 # - Create "v1.x.y" git tag.
-version = "1.22.0"
+version = "1.24.1"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -157,7 +157,7 @@ rand = "0.8.0"
 wasm-bindgen-test = "0.3.0"
 
 [target.'cfg(target_os = "freebsd")'.dev-dependencies]
-mio-aio = { version = "0.6.0", features = ["tokio"] }
+mio-aio = { version = "0.7.0", features = ["tokio"] }
 
 [target.'cfg(loom)'.dev-dependencies]
 loom = { version = "0.5.2", features = ["futures", "checkpoint"] }

tokio/README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.22.0", features = ["full"] }
+tokio = { version = "1.24.1", features = ["full"] }
 ```
 Then, on your main.rs:
 

tokio/build.rs

@@ -24,10 +24,25 @@ const CONST_MUTEX_NEW_PROBE: &str = r#"
 }
 "#;
 
+const TARGET_HAS_ATOMIC_PROBE: &str = r#"
+{
+    #[cfg(target_has_atomic = "ptr")]
+    let _ = (); 
+}
+"#;
+
+const TARGET_ATOMIC_U64_PROBE: &str = r#"
+{
+    use std::sync::atomic::AtomicU64 as _;
+}
+"#;
+
 fn main() {
     let mut enable_const_thread_local = false;
     let mut enable_addr_of = false;
+    let mut enable_target_has_atomic = false;
     let mut enable_const_mutex_new = false;
+    let mut target_needs_atomic_u64_fallback = false;
 
     match AutoCfg::new() {
         Ok(ac) => {
@@ -66,6 +81,27 @@ fn main() {
                 }
             }
 
+            // The `target_has_atomic` cfg was stabilized in 1.60.
+            if ac.probe_rustc_version(1, 61) {
+                enable_target_has_atomic = true;
+            } else if ac.probe_rustc_version(1, 60) {
+                // This compiler claims to be 1.60, but there are some nightly
+                // compilers that claim to be 1.60 without supporting the
+                // feature. Explicitly probe to check if code using them
+                // compiles.
+                //
+                // The oldest nightly that supports the feature is 2022-02-11.
+                if ac.probe_expression(TARGET_HAS_ATOMIC_PROBE) {
+                    enable_target_has_atomic = true;
+                }
+            }
+
+            // If we can't tell using `target_has_atomic`, tell if the target
+            // has `AtomicU64` by trying to use it.
+            if !enable_target_has_atomic && !ac.probe_expression(TARGET_ATOMIC_U64_PROBE) {
+                target_needs_atomic_u64_fallback = true;
+            }
+
             // The `Mutex::new` method was made const in 1.63.
             if ac.probe_rustc_version(1, 64) {
                 enable_const_mutex_new = true;
@@ -109,6 +145,14 @@ fn main() {
         autocfg::emit("tokio_no_addr_of")
     }
 
+    if !enable_target_has_atomic {
+        // To disable this feature on compilers that support it, you can
+        // explicitly pass this flag with the following environment variable:
+        //
+        // RUSTFLAGS="--cfg tokio_no_target_has_atomic"
+        autocfg::emit("tokio_no_target_has_atomic")
+    }
+
     if !enable_const_mutex_new {
         // To disable this feature on compilers that support it, you can
         // explicitly pass this flag with the following environment variable:
@@ -117,6 +161,14 @@ fn main() {
         autocfg::emit("tokio_no_const_mutex_new")
     }
 
+    if target_needs_atomic_u64_fallback {
+        // To disable this feature on compilers that support it, you can
+        // explicitly pass this flag with the following environment variable:
+        //
+        // RUSTFLAGS="--cfg tokio_no_atomic_u64"
+        autocfg::emit("tokio_no_atomic_u64")
+    }
+
     let target = ::std::env::var("TARGET").unwrap_or_default();
 
     // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54.

tokio/docs/reactor-refactor.md

@@ -188,12 +188,12 @@ readiness, the driver's tick is packed into the atomic `usize`.
 The `ScheduledIo` readiness `AtomicUsize` is structured as:
 
 ```
-| reserved | generation |  driver tick | readiness |
+| shutdown | generation |  driver tick | readiness |
 |----------+------------+--------------+-----------|
 |   1 bit  |   7 bits   +    8 bits    +  16 bits  |
 ```
 
-The `reserved` and `generation` components exist today.
+The `shutdown` and `generation` components exist today.
 
 The `readiness()` function returns a `ReadyEvent` value. This value includes the
 `tick` component read with the resource's readiness value. When

tokio/src/fs/read_dir.rs

@@ -1,5 +1,6 @@
 use crate::fs::asyncify;
 
+use std::collections::VecDeque;
 use std::ffi::OsString;
 use std::fs::{FileType, Metadata};
 use std::future::Future;
@@ -19,6 +20,8 @@ use crate::blocking::spawn_blocking;
 #[cfg(not(test))]
 use crate::blocking::JoinHandle;
 
+const CHUNK_SIZE: usize = 32;
+
 /// Returns a stream over the entries within a directory.
 ///
 /// This is an async version of [`std::fs::read_dir`](std::fs::read_dir)
@@ -29,9 +32,14 @@ use crate::blocking::JoinHandle;
 /// [`spawn_blocking`]: crate::task::spawn_blocking
 pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
     let path = path.as_ref().to_owned();
-    let std = asyncify(|| std::fs::read_dir(path)).await?;
+    asyncify(|| -> io::Result<ReadDir> {
+        let mut std = std::fs::read_dir(path)?;
+        let mut buf = VecDeque::with_capacity(CHUNK_SIZE);
+        ReadDir::next_chunk(&mut buf, &mut std);
 
-    Ok(ReadDir(State::Idle(Some(std))))
+        Ok(ReadDir(State::Idle(Some((buf, std)))))
+    })
+    .await
 }
 
 /// Reads the entries in a directory.
@@ -58,8 +66,8 @@ pub struct ReadDir(State);
 
 #[derive(Debug)]
 enum State {
-    Idle(Option<std::fs::ReadDir>),
-    Pending(JoinHandle<(Option<io::Result<std::fs::DirEntry>>, std::fs::ReadDir)>),
+    Idle(Option<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir)>),
+    Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir)>),
 }
 
 impl ReadDir {
@@ -94,29 +102,57 @@ impl ReadDir {
     pub fn poll_next_entry(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<DirEntry>>> {
         loop {
             match self.0 {
-                State::Idle(ref mut std) => {
-                    let mut std = std.take().unwrap();
+                State::Idle(ref mut data) => {
+                    let (buf, _) = data.as_mut().unwrap();
+
+                    if let Some(ent) = buf.pop_front() {
+                        return Poll::Ready(ent.map(Some));
+                    };
+
+                    let (mut buf, mut std) = data.take().unwrap();
 
                     self.0 = State::Pending(spawn_blocking(move || {
-                        let ret = std.next();
-                        (ret, std)
+                        ReadDir::next_chunk(&mut buf, &mut std);
+                        (buf, std)
                     }));
                 }
                 State::Pending(ref mut rx) => {
-                    let (ret, std) = ready!(Pin::new(rx).poll(cx))?;
-                    self.0 = State::Idle(Some(std));
+                    let (mut buf, std) = ready!(Pin::new(rx).poll(cx))?;
 
-                    let ret = match ret {
-                        Some(Ok(std)) => Ok(Some(DirEntry(Arc::new(std)))),
+                    let ret = match buf.pop_front() {
+                        Some(Ok(x)) => Ok(Some(x)),
                         Some(Err(e)) => Err(e),
                         None => Ok(None),
                     };
 
+                    self.0 = State::Idle(Some((buf, std)));
+
                     return Poll::Ready(ret);
                 }
             }
         }
     }
+
+    fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut std::fs::ReadDir) {
+        for ret in std.by_ref().take(CHUNK_SIZE) {
+            let success = ret.is_ok();
+
+            buf.push_back(ret.map(|std| DirEntry {
+                #[cfg(not(any(
+                    target_os = "solaris",
+                    target_os = "illumos",
+                    target_os = "haiku",
+                    target_os = "vxworks"
+                )))]
+                file_type: std.file_type().ok(),
+                std: Arc::new(std),
+            }));
+
+            if !success {
+                break;
+            }
+        }
+    }
 }
 
 feature! {
@@ -160,7 +196,16 @@ feature! {
 /// filesystem. Each entry can be inspected via methods to learn about the full
 /// path or possibly other metadata through per-platform extension traits.
 #[derive(Debug)]
-pub struct DirEntry(Arc<std::fs::DirEntry>);
+pub struct DirEntry {
+    #[cfg(not(any(
+        target_os = "solaris",
+        target_os = "illumos",
+        target_os = "haiku",
+        target_os = "vxworks"
+    )))]
+    file_type: Option<FileType>,
+    std: Arc<std::fs::DirEntry>,
+}
 
 impl DirEntry {
     /// Returns the full path to the file that this entry represents.
@@ -193,7 +238,7 @@ impl DirEntry {
     ///
     /// The exact text, of course, depends on what files you have in `.`.
     pub fn path(&self) -> PathBuf {
-        self.0.path()
+        self.std.path()
     }
 
     /// Returns the bare file name of this directory entry without any other
@@ -214,7 +259,7 @@ impl DirEntry {
     /// # }
     /// ```
     pub fn file_name(&self) -> OsString {
-        self.0.file_name()
+        self.std.file_name()
     }
 
     /// Returns the metadata for the file that this entry points at.
@@ -248,7 +293,7 @@ impl DirEntry {
     /// # }
     /// ```
     pub async fn metadata(&self) -> io::Result<Metadata> {
-        let std = self.0.clone();
+        let std = self.std.clone();
         asyncify(move || std.metadata()).await
     }
 
@@ -283,13 +328,23 @@ impl DirEntry {
     /// # }
     /// ```
     pub async fn file_type(&self) -> io::Result<FileType> {
-        let std = self.0.clone();
+        #[cfg(not(any(
+            target_os = "solaris",
+            target_os = "illumos",
+            target_os = "haiku",
+            target_os = "vxworks"
+        )))]
+        if let Some(file_type) = self.file_type {
+            return Ok(file_type);
+        }
+
+        let std = self.std.clone();
         asyncify(move || std.file_type()).await
     }
 
     /// Returns a reference to the underlying `std::fs::DirEntry`.
     #[cfg(unix)]
     pub(super) fn as_inner(&self) -> &std::fs::DirEntry {
-        &self.0
+        &self.std
     }
 }

tokio/src/io/util/read.rs

@@ -48,7 +48,7 @@ where
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
         let me = self.project();
-        let mut buf = ReadBuf::new(*me.buf);
+        let mut buf = ReadBuf::new(me.buf);
         ready!(Pin::new(me.reader).poll_read(cx, &mut buf))?;
         Poll::Ready(Ok(buf.filled().len()))
     }

tokio/src/lib.rs

@@ -174,12 +174,15 @@
 //! swapping the currently running task on each thread. However, this kind of
 //! swapping can only happen at `.await` points, so code that spends a long time
 //! without reaching an `.await` will prevent other tasks from running. To
-//! combat this, Tokio provides two kinds of threads: Core threads and blocking
-//! threads. The core threads are where all asynchronous code runs, and Tokio
-//! will by default spawn one for each CPU core. The blocking threads are
-//! spawned on demand, can be used to run blocking code that would otherwise
-//! block other tasks from running and are kept alive when not used for a certain
-//! amount of time which can be configured with [`thread_keep_alive`].
+//! combat this, Tokio provides two kinds of threads: Core threads and blocking threads.
+//!
+//! The core threads are where all asynchronous code runs, and Tokio will by default
+//! spawn one for each CPU core. You can use the environment variable `TOKIO_WORKER_THREADS`
+//! to override the default value.
+//!
+//! The blocking threads are spawned on demand, can be used to run blocking code
+//! that would otherwise block other tasks from running and are kept alive when
+//! not used for a certain amount of time which can be configured with [`thread_keep_alive`].
 //! Since it is not possible for Tokio to swap out blocking tasks, like it
 //! can do with asynchronous code, the upper limit on the number of blocking
 //! threads is very large. These limits can be configured on the [`Builder`].

tokio/src/loom/mocked.rs

@@ -25,6 +25,13 @@ pub(crate) mod sync {
         }
     }
     pub(crate) use loom::sync::*;
+
+    pub(crate) mod atomic {
+        pub(crate) use loom::sync::atomic::*;
+
+        // TODO: implement a loom version
+        pub(crate) type StaticAtomicU64 = std::sync::atomic::AtomicU64;
+    }
 }
 
 pub(crate) mod rand {

tokio/src/loom/std/atomic_u64.rs

@@ -7,80 +7,13 @@
 // `#[cfg(target_has_atomic = "64")]`.
 // Refs: https://github.com/rust-lang/rust/tree/master/src/librustc_target
 cfg_has_atomic_u64! {
-    pub(crate) use std::sync::atomic::AtomicU64;
+    #[path = "atomic_u64_native.rs"]
+    mod imp;
 }
 
 cfg_not_has_atomic_u64! {
-    use crate::loom::sync::Mutex;
-    use std::sync::atomic::Ordering;
-
-    #[derive(Debug)]
-    pub(crate) struct AtomicU64 {
-        inner: Mutex<u64>,
-    }
-
-    impl AtomicU64 {
-        pub(crate) fn new(val: u64) -> Self {
-            Self {
-                inner: Mutex::new(val),
-            }
-        }
-
-        pub(crate) fn load(&self, _: Ordering) -> u64 {
-            *self.inner.lock()
-        }
-
-        pub(crate) fn store(&self, val: u64, _: Ordering) {
-            *self.inner.lock() = val;
-        }
-
-        pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 {
-            let mut lock = self.inner.lock();
-            let prev = *lock;
-            *lock = prev + val;
-            prev
-        }
-
-        pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
-            let mut lock = self.inner.lock();
-            let prev = *lock;
-            *lock = prev | val;
-            prev
-        }
-
-        pub(crate) fn compare_exchange(
-            &self,
-            current: u64,
-            new: u64,
-            _success: Ordering,
-            _failure: Ordering,
-        ) -> Result<u64, u64> {
-            let mut lock = self.inner.lock();
-
-            if *lock == current {
-                *lock = new;
-                Ok(current)
-            } else {
-                Err(*lock)
-            }
-        }
-
-        pub(crate) fn compare_exchange_weak(
-            &self,
-            current: u64,
-            new: u64,
-            success: Ordering,
-            failure: Ordering,
-        ) -> Result<u64, u64> {
-            self.compare_exchange(current, new, success, failure)
-        }
-    }
-
-    impl Default for AtomicU64 {
-        fn default() -> AtomicU64 {
-            Self {
-                inner: Mutex::new(0),
-            }
-        }
-    }
+    #[path = "atomic_u64_as_mutex.rs"]
+    mod imp;
 }
+
+pub(crate) use imp::{AtomicU64, StaticAtomicU64};

tokio/src/loom/std/atomic_u64_as_mutex.rs

@@ -0,0 +1,76 @@
+use crate::loom::sync::Mutex;
+use std::sync::atomic::Ordering;
+
+cfg_has_const_mutex_new! {
+    #[path = "atomic_u64_static_const_new.rs"]
+    mod static_macro;
+}
+
+cfg_not_has_const_mutex_new! {
+    #[path = "atomic_u64_static_once_cell.rs"]
+    mod static_macro;
+}
+
+pub(crate) use static_macro::StaticAtomicU64;
+
+#[derive(Debug)]
+pub(crate) struct AtomicU64 {
+    inner: Mutex<u64>,
+}
+
+impl AtomicU64 {
+    pub(crate) fn load(&self, _: Ordering) -> u64 {
+        *self.inner.lock()
+    }
+
+    pub(crate) fn store(&self, val: u64, _: Ordering) {
+        *self.inner.lock() = val;
+    }
+
+    pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 {
+        let mut lock = self.inner.lock();
+        let prev = *lock;
+        *lock = prev + val;
+        prev
+    }
+
+    pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
+        let mut lock = self.inner.lock();
+        let prev = *lock;
+        *lock = prev | val;
+        prev
+    }
+
+    pub(crate) fn compare_exchange(
+        &self,
+        current: u64,
+        new: u64,
+        _success: Ordering,
+        _failure: Ordering,
+    ) -> Result<u64, u64> {
+        let mut lock = self.inner.lock();
+
+        if *lock == current {
+            *lock = new;
+            Ok(current)
+        } else {
+            Err(*lock)
+        }
+    }
+
+    pub(crate) fn compare_exchange_weak(
+        &self,
+        current: u64,
+        new: u64,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<u64, u64> {
+        self.compare_exchange(current, new, success, failure)
+    }
+}
+
+impl Default for AtomicU64 {
+    fn default() -> AtomicU64 {
+        AtomicU64::new(u64::default())
+    }
+}

tokio/src/loom/std/atomic_u64_native.rs

@@ -0,0 +1,4 @@
+pub(crate) use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Alias `AtomicU64` to `StaticAtomicU64`
+pub(crate) type StaticAtomicU64 = AtomicU64;

tokio/src/loom/std/atomic_u64_static_const_new.rs

@@ -0,0 +1,12 @@
+use super::AtomicU64;
+use crate::loom::sync::Mutex;
+
+pub(crate) type StaticAtomicU64 = AtomicU64;
+
+impl AtomicU64 {
+    pub(crate) const fn new(val: u64) -> Self {
+        Self {
+            inner: Mutex::const_new(val),
+        }
+    }
+}

tokio/src/loom/std/atomic_u64_static_once_cell.rs

@@ -0,0 +1,57 @@
+use super::AtomicU64;
+use crate::loom::sync::{atomic::Ordering, Mutex};
+use crate::util::once_cell::OnceCell;
+
+pub(crate) struct StaticAtomicU64 {
+    init: u64,
+    cell: OnceCell<Mutex<u64>>,
+}
+
+impl AtomicU64 {
+    pub(crate) fn new(val: u64) -> Self {
+        Self {
+            inner: Mutex::new(val),
+        }
+    }
+}
+
+impl StaticAtomicU64 {
+    pub(crate) const fn new(val: u64) -> StaticAtomicU64 {
+        StaticAtomicU64 {
+            init: val,
+            cell: OnceCell::new(),
+        }
+    }
+
+    pub(crate) fn load(&self, order: Ordering) -> u64 {
+        *self.inner().lock()
+    }
+
+    pub(crate) fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+        let mut lock = self.inner().lock();
+        let prev = *lock;
+        *lock = prev + val;
+        prev
+    }
+
+    pub(crate) fn compare_exchange_weak(
+        &self,
+        current: u64,
+        new: u64,
+        _success: Ordering,
+        _failure: Ordering,
+    ) -> Result<u64, u64> {
+        let mut lock = self.inner().lock();
+
+        if *lock == current {
+            *lock = new;
+            Ok(current)
+        } else {
+            Err(*lock)
+        }
+    }
+
+    fn inner(&self) -> &Mutex<u64> {
+        self.cell.get(|| Mutex::new(self.init))
+    }
+}

tokio/src/loom/std/mod.rs

@@ -71,7 +71,7 @@ pub(crate) mod sync {
     pub(crate) mod atomic {
         pub(crate) use crate::loom::std::atomic_u16::AtomicU16;
         pub(crate) use crate::loom::std::atomic_u32::AtomicU32;
-        pub(crate) use crate::loom::std::atomic_u64::AtomicU64;
+        pub(crate) use crate::loom::std::atomic_u64::{AtomicU64, StaticAtomicU64};
         pub(crate) use crate::loom::std::atomic_usize::AtomicUsize;
 
         pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering};
@@ -81,7 +81,27 @@ pub(crate) mod sync {
 pub(crate) mod sys {
     #[cfg(feature = "rt-multi-thread")]
     pub(crate) fn num_cpus() -> usize {
-        usize::max(1, num_cpus::get())
+        const ENV_WORKER_THREADS: &str = "TOKIO_WORKER_THREADS";
+
+        match std::env::var(ENV_WORKER_THREADS) {
+            Ok(s) => {
+                let n = s.parse().unwrap_or_else(|e| {
+                    panic!(
+                        "\"{}\" must be usize, error: {}, value: {}",
+                        ENV_WORKER_THREADS, e, s
+                    )
+                });
+                assert!(n > 0, "\"{}\" cannot be set to 0", ENV_WORKER_THREADS);
+                n
+            }
+            Err(std::env::VarError::NotPresent) => usize::max(1, num_cpus::get()),
+            Err(std::env::VarError::NotUnicode(e)) => {
+                panic!(
+                    "\"{}\" must be valid unicode, error: {:?}",
+                    ENV_WORKER_THREADS, e
+                )
+            }
+        }
     }
 
     #[cfg(not(feature = "rt-multi-thread"))]

tokio/src/macros/cfg.rs

@@ -461,14 +461,14 @@ macro_rules! cfg_not_coop {
 macro_rules! cfg_has_atomic_u64 {
     ($($item:item)*) => {
         $(
-            #[cfg(not(any(
-                target_arch = "arm",
-                target_arch = "mips",
-                target_arch = "powerpc",
-                target_arch = "riscv32",
-                tokio_wasm,
-                tokio_no_atomic_u64,
-            )))]
+            #[cfg_attr(
+                not(tokio_no_target_has_atomic),
+                cfg(all(target_has_atomic = "64", not(tokio_no_atomic_u64))
+            ))]
+            #[cfg_attr(
+                tokio_no_target_has_atomic,
+                cfg(not(tokio_no_atomic_u64))
+            )]
             $item
         )*
     }
@@ -477,14 +477,14 @@ macro_rules! cfg_has_atomic_u64 {
 macro_rules! cfg_not_has_atomic_u64 {
     ($($item:item)*) => {
         $(
-            #[cfg(any(
-                target_arch = "arm",
-                target_arch = "mips",
-                target_arch = "powerpc",
-                target_arch = "riscv32",
-                tokio_wasm,
-                tokio_no_atomic_u64,
+            #[cfg_attr(
+                not(tokio_no_target_has_atomic),
+                cfg(any(not(target_has_atomic = "64"), tokio_no_atomic_u64)
             ))]
+            #[cfg_attr(
+                tokio_no_target_has_atomic,
+                cfg(tokio_no_atomic_u64)
+            )]
             $item
         )*
     }

tokio/src/net/tcp/split.rs

@@ -145,6 +145,12 @@ impl ReadHalf<'_> {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety
@@ -273,6 +279,12 @@ impl WriteHalf<'_> {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety

tokio/src/net/tcp/split_owned.rs

@@ -200,6 +200,12 @@ impl OwnedReadHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety
@@ -355,6 +361,12 @@ impl OwnedWriteHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety
@@ -478,12 +490,12 @@ impl AsyncWrite for OwnedWriteHalf {
 
 impl AsRef<TcpStream> for OwnedReadHalf {
     fn as_ref(&self) -> &TcpStream {
-        &*self.inner
+        &self.inner
     }
 }
 
 impl AsRef<TcpStream> for OwnedWriteHalf {
     fn as_ref(&self) -> &TcpStream {
-        &*self.inner
+        &self.inner
     }
 }

tokio/src/net/tcp/stream.rs

@@ -377,6 +377,12 @@ impl TcpStream {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method

tokio/src/net/udp.rs

@@ -357,7 +357,9 @@ impl UdpSocket {
     ///
     /// The function may complete without the socket being ready. This is a
     /// false-positive and attempting an operation will return with
-    /// `io::ErrorKind::WouldBlock`.
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
     ///
     /// # Cancel safety
     ///

tokio/src/net/unix/datagram/socket.rs

@@ -104,7 +104,9 @@ impl UnixDatagram {
     ///
     /// The function may complete without the socket being ready. This is a
     /// false-positive and attempting an operation will return with
-    /// `io::ErrorKind::WouldBlock`.
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
     ///
     /// # Cancel safety
     ///

tokio/src/net/unix/split.rs

@@ -182,6 +182,12 @@ impl WriteHalf<'_> {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method

tokio/src/net/unix/split_owned.rs

@@ -114,6 +114,12 @@ impl OwnedReadHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method
@@ -265,6 +271,12 @@ impl OwnedWriteHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method
@@ -386,12 +398,12 @@ impl AsyncWrite for OwnedWriteHalf {
 
 impl AsRef<UnixStream> for OwnedReadHalf {
     fn as_ref(&self) -> &UnixStream {
-        &*self.inner
+        &self.inner
     }
 }
 
 impl AsRef<UnixStream> for OwnedWriteHalf {
     fn as_ref(&self) -> &UnixStream {
-        &*self.inner
+        &self.inner
     }
 }

tokio/src/net/unix/stream.rs

@@ -66,6 +66,12 @@ impl UnixStream {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method

tokio/src/net/windows/named_pipe.rs

@@ -238,6 +238,12 @@ impl NamedPipeServer {
     /// can be used to concurrently read / write to the same pipe on a single
     /// task without splitting the pipe.
     ///
+    /// The function may complete without the pipe being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Examples
     ///
     /// Concurrently read and write to the pipe on the same task without
@@ -989,6 +995,12 @@ impl NamedPipeClient {
     /// can be used to concurrently read / write to the same pipe on a single
     /// task without splitting the pipe.
     ///
+    /// The function may complete without the pipe being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Examples
     ///
     /// Concurrently read and write to the pipe on the same task without
@@ -1693,11 +1705,10 @@ impl ServerOptions {
     ///
     /// [`dwPipeMode`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn pipe_mode(&mut self, pipe_mode: PipeMode) -> &mut Self {
-        self.pipe_mode = match pipe_mode {
-            PipeMode::Byte => windows_sys::PIPE_TYPE_BYTE,
-            PipeMode::Message => windows_sys::PIPE_TYPE_MESSAGE,
-        };
-
+        let is_msg = matches!(pipe_mode, PipeMode::Message);
+        // Pipe mode is implemented as a bit flag 0x4. Set is message and unset
+        // is byte.
+        bool_flag!(self.pipe_mode, is_msg, windows_sys::PIPE_TYPE_MESSAGE);
         self
     }
 
@@ -2542,3 +2553,48 @@ unsafe fn named_pipe_info(handle: RawHandle) -> io::Result<PipeInfo> {
         max_instances,
     })
 }
+
+#[cfg(test)]
+mod test {
+    use self::windows_sys::{PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_TYPE_MESSAGE};
+    use super::*;
+
+    #[test]
+    fn opts_default_pipe_mode() {
+        let opts = ServerOptions::new();
+        assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS);
+    }
+
+    #[test]
+    fn opts_unset_reject_remote() {
+        let mut opts = ServerOptions::new();
+        opts.reject_remote_clients(false);
+        assert_eq!(opts.pipe_mode & PIPE_REJECT_REMOTE_CLIENTS, 0);
+    }
+
+    #[test]
+    fn opts_set_pipe_mode_maintains_reject_remote_clients() {
+        let mut opts = ServerOptions::new();
+        opts.pipe_mode(PipeMode::Byte);
+        assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS);
+
+        opts.reject_remote_clients(false);
+        opts.pipe_mode(PipeMode::Byte);
+        assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE);
+
+        opts.reject_remote_clients(true);
+        opts.pipe_mode(PipeMode::Byte);
+        assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS);
+
+        opts.reject_remote_clients(false);
+        opts.pipe_mode(PipeMode::Message);
+        assert_eq!(opts.pipe_mode, PIPE_TYPE_MESSAGE);
+
+        opts.reject_remote_clients(true);
+        opts.pipe_mode(PipeMode::Message);
+        assert_eq!(
+            opts.pipe_mode,
+            PIPE_TYPE_MESSAGE | PIPE_REJECT_REMOTE_CLIENTS
+        );
+    }
+}

tokio/src/process/unix/mod.rs

@@ -156,7 +156,7 @@ impl Future for Child {
 
 #[derive(Debug)]
 pub(crate) struct Pipe {
-    // Actually a pipe and not a File. However, we are reusing `File` to get
+    // Actually a pipe is not a File. However, we are reusing `File` to get
     // close on drop. This is a similar trick as `mio`.
     fd: File,
 }

tokio/src/process/unix/orphan.rs

@@ -294,7 +294,7 @@ pub(crate) mod test {
     #[cfg_attr(miri, ignore)] // Miri does not support epoll.
     #[test]
     fn does_not_register_signal_if_queue_empty() {
-        let (io_driver, io_handle) = IoDriver::new().unwrap();
+        let (io_driver, io_handle) = IoDriver::new(1024).unwrap();
         let signal_driver = SignalDriver::new(io_driver, &io_handle).unwrap();
         let handle = signal_driver.handle();
 

tokio/src/runtime/blocking/mod.rs

@@ -17,8 +17,6 @@ cfg_trace! {
 mod schedule;
 mod shutdown;
 mod task;
-#[cfg(all(test, not(tokio_wasm)))]
-pub(crate) use schedule::NoopSchedule;
 pub(crate) use task::BlockingTask;
 
 use crate::runtime::Builder;

tokio/src/runtime/blocking/pool.rs

@@ -2,7 +2,7 @@
 
 use crate::loom::sync::{Arc, Condvar, Mutex};
 use crate::loom::thread;
-use crate::runtime::blocking::schedule::NoopSchedule;
+use crate::runtime::blocking::schedule::BlockingSchedule;
 use crate::runtime::blocking::{shutdown, BlockingTask};
 use crate::runtime::builder::ThreadNameFn;
 use crate::runtime::task::{self, JoinHandle};
@@ -120,7 +120,7 @@ struct Shared {
 }
 
 pub(crate) struct Task {
-    task: task::UnownedTask<NoopSchedule>,
+    task: task::UnownedTask<BlockingSchedule>,
     mandatory: Mandatory,
 }
 
@@ -151,7 +151,7 @@ impl From<SpawnError> for io::Error {
 }
 
 impl Task {
-    pub(crate) fn new(task: task::UnownedTask<NoopSchedule>, mandatory: Mandatory) -> Task {
+    pub(crate) fn new(task: task::UnownedTask<BlockingSchedule>, mandatory: Mandatory) -> Task {
         Task { task, mandatory }
     }
 
@@ -379,7 +379,8 @@ impl Spawner {
         #[cfg(not(all(tokio_unstable, feature = "tracing")))]
         let _ = name;
 
-        let (task, handle) = task::unowned(fut, NoopSchedule, id);
+        let (task, handle) = task::unowned(fut, BlockingSchedule::new(rt), id);
+
         let spawned = self.spawn_task(Task::new(task, is_mandatory), rt);
         (handle, spawned)
     }

tokio/src/runtime/blocking/schedule.rs

@@ -1,15 +1,52 @@
+#[cfg(feature = "test-util")]
+use crate::runtime::scheduler;
 use crate::runtime::task::{self, Task};
+use crate::runtime::Handle;
 
-/// `task::Schedule` implementation that does nothing. This is unique to the
-/// blocking scheduler as tasks scheduled are not really futures but blocking
-/// operations.
+/// `task::Schedule` implementation that does nothing (except some bookkeeping
+/// in test-util builds). This is unique to the blocking scheduler as tasks
+/// scheduled are not really futures but blocking operations.
 ///
 /// We avoid storing the task by forgetting it in `bind` and re-materializing it
-/// in `release.
-pub(crate) struct NoopSchedule;
+/// in `release`.
+pub(crate) struct BlockingSchedule {
+    #[cfg(feature = "test-util")]
+    handle: Handle,
+}
 
-impl task::Schedule for NoopSchedule {
+impl BlockingSchedule {
+    #[cfg_attr(not(feature = "test-util"), allow(unused_variables))]
+    pub(crate) fn new(handle: &Handle) -> Self {
+        #[cfg(feature = "test-util")]
+        {
+            match &handle.inner {
+                scheduler::Handle::CurrentThread(handle) => {
+                    handle.driver.clock.inhibit_auto_advance();
+                }
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThread(_) => {}
+            }
+        }
+        BlockingSchedule {
+            #[cfg(feature = "test-util")]
+            handle: handle.clone(),
+        }
+    }
+}
+
+impl task::Schedule for BlockingSchedule {
     fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> {
+        #[cfg(feature = "test-util")]
+        {
+            match &self.handle.inner {
+                scheduler::Handle::CurrentThread(handle) => {
+                    handle.driver.clock.allow_auto_advance();
+                    handle.driver.unpark();
+                }
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThread(_) => {}
+            }
+        }
         None
     }
 

tokio/src/runtime/builder.rs

@@ -44,6 +44,7 @@ pub struct Builder {
 
     /// Whether or not to enable the I/O driver
     enable_io: bool,
+    nevents: usize,
 
     /// Whether or not to enable the time driver
     enable_time: bool,
@@ -181,6 +182,7 @@ cfg_unstable! {
 
 pub(crate) type ThreadNameFn = std::sync::Arc<dyn Fn() -> String + Send + Sync + 'static>;
 
+#[derive(Clone, Copy)]
 pub(crate) enum Kind {
     CurrentThread,
     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
@@ -228,6 +230,7 @@ impl Builder {
 
             // I/O defaults to "off"
             enable_io: false,
+            nevents: 1024,
 
             // Time defaults to "off"
             enable_time: false,
@@ -235,6 +238,7 @@ impl Builder {
             // The clock starts not-paused
             start_paused: false,
 
+            // Read from environment variable first in multi-threaded mode.
             // Default to lazy auto-detection (one thread per CPU core)
             worker_threads: None,
 
@@ -302,6 +306,8 @@ impl Builder {
     /// This can be any number above 0 though it is advised to keep this value
     /// on the smaller side.
     ///
+    /// This will override the value read from environment variable `TOKIO_WORKER_THREADS`.
+    ///
     /// # Default
     ///
     /// The default value is the number of cores available to the system.
@@ -647,6 +653,7 @@ impl Builder {
             enable_io: self.enable_io,
             enable_time: self.enable_time,
             start_paused: self.start_paused,
+            nevents: self.nevents,
         }
     }
 
@@ -938,6 +945,25 @@ cfg_io_driver! {
             self.enable_io = true;
             self
         }
+
+        /// Enables the I/O driver and configures the max number of events to be
+        /// processed per tick.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        ///
+        /// let rt = runtime::Builder::new_current_thread()
+        ///     .enable_io()
+        ///     .max_io_events_per_tick(1024)
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self {
+            self.nevents = capacity;
+            self
+        }
     }
 }
 

tokio/src/runtime/context.rs

@@ -7,8 +7,7 @@ use std::cell::Cell;
 use crate::util::rand::{FastRand, RngSeed};
 
 cfg_rt! {
-    use crate::runtime::scheduler;
-    use crate::runtime::task::Id;
+    use crate::runtime::{scheduler, task::Id, Defer};
 
     use std::cell::RefCell;
     use std::marker::PhantomData;
@@ -16,9 +15,14 @@ cfg_rt! {
 }
 
 struct Context {
+    /// Uniquely identifies the current thread
+    #[cfg(feature = "rt")]
+    thread_id: Cell<Option<ThreadId>>,
+
     /// Handle to the runtime scheduler running on the current thread.
     #[cfg(feature = "rt")]
     handle: RefCell<Option<scheduler::Handle>>,
+
     #[cfg(feature = "rt")]
     current_task_id: Cell<Option<Id>>,
 
@@ -30,6 +34,11 @@ struct Context {
     #[cfg(feature = "rt")]
     runtime: Cell<EnterRuntime>,
 
+    /// Yielded task wakers are stored here and notified after resource drivers
+    /// are polled.
+    #[cfg(feature = "rt")]
+    defer: RefCell<Option<Defer>>,
+
     #[cfg(any(feature = "rt", feature = "macros"))]
     rng: FastRand,
 
@@ -41,6 +50,9 @@ struct Context {
 tokio_thread_local! {
     static CONTEXT: Context = {
         Context {
+            #[cfg(feature = "rt")]
+            thread_id: Cell::new(None),
+
             /// Tracks the current runtime handle to use when spawning,
             /// accessing drivers, etc...
             #[cfg(feature = "rt")]
@@ -56,6 +68,9 @@ tokio_thread_local! {
             #[cfg(feature = "rt")]
             runtime: Cell::new(EnterRuntime::NotEntered),
 
+            #[cfg(feature = "rt")]
+            defer: RefCell::new(None),
+
             #[cfg(any(feature = "rt", feature = "macros"))]
             rng: FastRand::new(RngSeed::new()),
 
@@ -74,10 +89,23 @@ pub(super) fn budget<R>(f: impl FnOnce(&Cell<coop::Budget>) -> R) -> Result<R, A
 }
 
 cfg_rt! {
-    use crate::runtime::TryCurrentError;
+    use crate::runtime::{ThreadId, TryCurrentError};
 
     use std::fmt;
 
+    pub(crate) fn thread_id() -> Result<ThreadId, AccessError> {
+        CONTEXT.try_with(|ctx| {
+            match ctx.thread_id.get() {
+                Some(id) => id,
+                None => {
+                    let id = ThreadId::next();
+                    ctx.thread_id.set(Some(id));
+                    id
+                }
+            }
+        })
+    }
+
     #[derive(Debug, Clone, Copy)]
     #[must_use]
     pub(crate) enum EnterRuntime {
@@ -99,9 +127,17 @@ cfg_rt! {
     /// Guard tracking that a caller has entered a runtime context.
     #[must_use]
     pub(crate) struct EnterRuntimeGuard {
+        /// Tracks that the current thread has entered a blocking function call.
         pub(crate) blocking: BlockingRegionGuard,
+
         #[allow(dead_code)] // Only tracking the guard.
         pub(crate) handle: SetCurrentGuard,
+
+        /// If true, then this is the root runtime guard. It is possible to nest
+        /// runtime guards by using `block_in_place` between the calls. We need
+        /// to track the root guard as this is the guard responsible for freeing
+        /// the deferred task queue.
+        is_root: bool,
     }
 
     /// Guard tracking that a caller has entered a blocking region.
@@ -159,10 +195,23 @@ cfg_rt! {
             if c.runtime.get().is_entered() {
                 None
             } else {
+                // Set the entered flag
                 c.runtime.set(EnterRuntime::Entered { allow_block_in_place });
+
+                // Initialize queue to track yielded tasks
+                let mut defer = c.defer.borrow_mut();
+
+                let is_root = if defer.is_none() {
+                    *defer = Some(Defer::new());
+                    true
+                } else {
+                    false
+                };
+
                 Some(EnterRuntimeGuard {
                     blocking: BlockingRegionGuard::new(),
                     handle: c.set_current(handle),
+                    is_root,
                 })
             }
         })
@@ -201,6 +250,13 @@ cfg_rt! {
         DisallowBlockInPlaceGuard(reset)
     }
 
+    pub(crate) fn with_defer<R>(f: impl FnOnce(&mut Defer) -> R) -> Option<R> {
+        CONTEXT.with(|c| {
+            let mut defer = c.defer.borrow_mut();
+            defer.as_mut().map(f)
+        })
+    }
+
     impl Context {
         fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard {
             let rng_seed = handle.seed_generator().next_seed();
@@ -235,6 +291,10 @@ cfg_rt! {
             CONTEXT.with(|c| {
                 assert!(c.runtime.get().is_entered());
                 c.runtime.set(EnterRuntime::NotEntered);
+
+                if self.is_root {
+                    *c.defer.borrow_mut() = None;
+                }
             });
         }
     }
@@ -286,6 +346,10 @@ cfg_rt! {
                     return Err(());
                 }
 
+                // Wake any yielded tasks before parking in order to avoid
+                // blocking.
+                with_defer(|defer| defer.wake());
+
                 park.park_timeout(when - now);
             }
         }

tokio/src/runtime/defer.rs

@@ -0,0 +1,27 @@
+use std::task::Waker;
+
+pub(crate) struct Defer {
+    deferred: Vec<Waker>,
+}
+
+impl Defer {
+    pub(crate) fn new() -> Defer {
+        Defer {
+            deferred: Default::default(),
+        }
+    }
+
+    pub(crate) fn defer(&mut self, waker: Waker) {
+        self.deferred.push(waker);
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.deferred.is_empty()
+    }
+
+    pub(crate) fn wake(&mut self) {
+        for waker in self.deferred.drain(..) {
+            waker.wake();
+        }
+    }
+}

tokio/src/runtime/driver.rs

@@ -36,11 +36,12 @@ pub(crate) struct Cfg {
     pub(crate) enable_time: bool,
     pub(crate) enable_pause_time: bool,
     pub(crate) start_paused: bool,
+    pub(crate) nevents: usize,
 }
 
 impl Driver {
     pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Handle)> {
-        let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io)?;
+        let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io, cfg.nevents)?;
 
         let clock = create_clock(cfg.enable_pause_time, cfg.start_paused);
 
@@ -135,12 +136,12 @@ cfg_io_driver! {
         Disabled(UnparkThread),
     }
 
-    fn create_io_stack(enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
+    fn create_io_stack(enabled: bool, nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
         #[cfg(loom)]
         assert!(!enabled);
 
         let ret = if enabled {
-            let (io_driver, io_handle) = crate::runtime::io::Driver::new()?;
+            let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents)?;
 
             let (signal_driver, signal_handle) = create_signal_driver(io_driver, &io_handle)?;
             let process_driver = create_process_driver(signal_driver);
@@ -201,7 +202,7 @@ cfg_not_io_driver! {
     #[derive(Debug)]
     pub(crate) struct IoStack(ParkThread);
 
-    fn create_io_stack(_enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
+    fn create_io_stack(_enabled: bool, _nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
         let park_thread = ParkThread::new();
         let unpark_thread = park_thread.unpark();
         Ok((IoStack(park_thread), unpark_thread, Default::default()))

tokio/src/runtime/io/mod.rs

@@ -60,6 +60,7 @@ pub(crate) struct Handle {
 pub(crate) struct ReadyEvent {
     tick: u8,
     pub(crate) ready: Ready,
+    is_shutdown: bool,
 }
 
 struct IoDispatcher {
@@ -104,7 +105,7 @@ fn _assert_kinds() {
 impl Driver {
     /// Creates a new event loop, returning any error that happened during the
     /// creation.
-    pub(crate) fn new() -> io::Result<(Driver, Handle)> {
+    pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> {
         let poll = mio::Poll::new()?;
         #[cfg(not(tokio_wasi))]
         let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
@@ -116,7 +117,7 @@ impl Driver {
         let driver = Driver {
             tick: 0,
             signal_ready: false,
-            events: mio::Events::with_capacity(1024),
+            events: mio::Events::with_capacity(nevents),
             poll,
             resources: slab,
         };
@@ -147,9 +148,8 @@ impl Driver {
 
         if handle.shutdown() {
             self.resources.for_each(|io| {
-                // If a task is waiting on the I/O resource, notify it. The task
-                // will then attempt to use the I/O resource and fail due to the
-                // driver being shutdown. And shutdown will clear all wakers.
+                // If a task is waiting on the I/O resource, notify it that the
+                // runtime is being shutdown. And shutdown will clear all wakers.
                 io.shutdown();
             });
         }
@@ -282,16 +282,12 @@ impl Handle {
         true
     }
 
-    fn is_shutdown(&self) -> bool {
-        return self.io_dispatch.read().unwrap().is_shutdown;
-    }
-
     fn allocate(&self) -> io::Result<(slab::Address, slab::Ref<ScheduledIo>)> {
         let io = self.io_dispatch.read().unwrap();
         if io.is_shutdown {
             return Err(io::Error::new(
                 io::ErrorKind::Other,
-                "failed to find event loop",
+                crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
             ));
         }
         io.allocator.allocate().ok_or_else(|| {

tokio/src/runtime/io/registration.rs

@@ -148,7 +148,7 @@ impl Registration {
         let coop = ready!(crate::runtime::coop::poll_proceed(cx));
         let ev = ready!(self.shared.poll_readiness(cx, direction));
 
-        if self.handle().is_shutdown() {
+        if ev.is_shutdown {
             return Poll::Ready(Err(gone()));
         }
 
@@ -217,28 +217,22 @@ impl Drop for Registration {
 }
 
 fn gone() -> io::Error {
-    io::Error::new(io::ErrorKind::Other, "IO driver has terminated")
+    io::Error::new(
+        io::ErrorKind::Other,
+        crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
+    )
 }
 
 cfg_io_readiness! {
     impl Registration {
         pub(crate) async fn readiness(&self, interest: Interest) -> io::Result<ReadyEvent> {
-            use std::future::Future;
-            use std::pin::Pin;
+            let ev = self.shared.readiness(interest).await;
 
-            let fut = self.shared.readiness(interest);
-            pin!(fut);
+            if ev.is_shutdown {
+                return Err(gone())
+            }
 
-            crate::future::poll_fn(|cx| {
-                if self.handle().is_shutdown() {
-                    return Poll::Ready(Err(io::Error::new(
-                        io::ErrorKind::Other,
-                        crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR
-                    )));
-                }
-
-                Pin::new(&mut fut).poll(cx).map(Ok)
-            }).await
+            Ok(ev)
         }
 
         pub(crate) async fn async_io<R>(&self, interest: Interest, mut f: impl FnMut() -> io::Result<R>) -> io::Result<R> {

tokio/src/runtime/io/scheduled_io.rs

@@ -46,9 +46,6 @@ struct Waiters {
 
     /// Waker used for AsyncWrite.
     writer: Option<Waker>,
-
-    /// True if this ScheduledIo has been killed due to IO driver shutdown.
-    is_shutdown: bool,
 }
 
 cfg_io_readiness! {
@@ -95,7 +92,7 @@ cfg_io_readiness! {
 
 // The `ScheduledIo::readiness` (`AtomicUsize`) is packed full of goodness.
 //
-// | reserved | generation |  driver tick | readiness |
+// | shutdown | generation |  driver tick | readiness |
 // |----------+------------+--------------+-----------|
 // |   1 bit  |   7 bits   +    8 bits    +   16 bits |
 
@@ -105,6 +102,8 @@ const TICK: bit::Pack = READINESS.then(8);
 
 const GENERATION: bit::Pack = TICK.then(7);
 
+const SHUTDOWN: bit::Pack = GENERATION.then(1);
+
 #[test]
 fn test_generations_assert_same() {
     assert_eq!(super::GENERATION, GENERATION);
@@ -138,9 +137,11 @@ impl ScheduledIo {
     }
 
     /// Invoked when the IO driver is shut down; forces this ScheduledIo into a
-    /// permanently ready state.
+    /// permanently shutdown state.
     pub(super) fn shutdown(&self) {
-        self.wake0(Ready::ALL, true)
+        let mask = SHUTDOWN.pack(1, 0);
+        self.readiness.fetch_or(mask, AcqRel);
+        self.wake(Ready::ALL);
     }
 
     /// Sets the readiness on this `ScheduledIo` by invoking the given closure on
@@ -219,16 +220,10 @@ impl ScheduledIo {
     /// than 32 wakers to notify, if the stack array fills up, the lock is
     /// released, the array is cleared, and the iteration continues.
     pub(super) fn wake(&self, ready: Ready) {
-        self.wake0(ready, false);
-    }
-
-    fn wake0(&self, ready: Ready, shutdown: bool) {
         let mut wakers = WakeList::new();
 
         let mut waiters = self.waiters.lock();
 
-        waiters.is_shutdown |= shutdown;
-
         // check for AsyncRead slot
         if ready.is_readable() {
             if let Some(waker) = waiters.reader.take() {
@@ -283,6 +278,7 @@ impl ScheduledIo {
         ReadyEvent {
             tick: TICK.unpack(curr) as u8,
             ready: interest.mask() & Ready::from_usize(READINESS.unpack(curr)),
+            is_shutdown: SHUTDOWN.unpack(curr) != 0,
         }
     }
 
@@ -299,8 +295,9 @@ impl ScheduledIo {
         let curr = self.readiness.load(Acquire);
 
         let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr));
+        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
 
-        if ready.is_empty() {
+        if ready.is_empty() && !is_shutdown {
             // Update the task info
             let mut waiters = self.waiters.lock();
             let slot = match direction {
@@ -325,10 +322,12 @@ impl ScheduledIo {
             // taking the waiters lock
             let curr = self.readiness.load(Acquire);
             let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr));
-            if waiters.is_shutdown {
+            let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+            if is_shutdown {
                 Poll::Ready(ReadyEvent {
                     tick: TICK.unpack(curr) as u8,
                     ready: direction.mask(),
+                    is_shutdown,
                 })
             } else if ready.is_empty() {
                 Poll::Pending
@@ -336,12 +335,14 @@ impl ScheduledIo {
                 Poll::Ready(ReadyEvent {
                     tick: TICK.unpack(curr) as u8,
                     ready,
+                    is_shutdown,
                 })
             }
         } else {
             Poll::Ready(ReadyEvent {
                 tick: TICK.unpack(curr) as u8,
                 ready,
+                is_shutdown,
             })
         }
     }
@@ -433,16 +434,17 @@ cfg_io_readiness! {
                         // Optimistically check existing readiness
                         let curr = scheduled_io.readiness.load(SeqCst);
                         let ready = Ready::from_usize(READINESS.unpack(curr));
+                        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
 
                         // Safety: `waiter.interest` never changes
                         let interest = unsafe { (*waiter.get()).interest };
                         let ready = ready.intersection(interest);
 
-                        if !ready.is_empty() {
+                        if !ready.is_empty() || is_shutdown {
                             // Currently ready!
                             let tick = TICK.unpack(curr) as u8;
                             *state = State::Done;
-                            return Poll::Ready(ReadyEvent { tick, ready });
+                            return Poll::Ready(ReadyEvent { tick, ready, is_shutdown });
                         }
 
                         // Wasn't ready, take the lock (and check again while locked).
@@ -450,18 +452,19 @@ cfg_io_readiness! {
 
                         let curr = scheduled_io.readiness.load(SeqCst);
                         let mut ready = Ready::from_usize(READINESS.unpack(curr));
+                        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
 
-                        if waiters.is_shutdown {
+                        if is_shutdown {
                             ready = Ready::ALL;
                         }
 
                         let ready = ready.intersection(interest);
 
-                        if !ready.is_empty() {
+                        if !ready.is_empty() || is_shutdown {
                             // Currently ready!
                             let tick = TICK.unpack(curr) as u8;
                             *state = State::Done;
-                            return Poll::Ready(ReadyEvent { tick, ready });
+                            return Poll::Ready(ReadyEvent { tick, ready, is_shutdown });
                         }
 
                         // Not ready even after locked, insert into list...
@@ -510,14 +513,26 @@ cfg_io_readiness! {
                         drop(waiters);
                     }
                     State::Done => {
-                        let tick = TICK.unpack(scheduled_io.readiness.load(Acquire)) as u8;
-
                         // Safety: State::Done means it is no longer shared
                         let w = unsafe { &mut *waiter.get() };
 
+                        let curr = scheduled_io.readiness.load(Acquire);
+                        let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+
+                        // The returned tick might be newer than the event
+                        // which notified our waker. This is ok because the future
+                        // still didn't return `Poll::Ready`.
+                        let tick = TICK.unpack(curr) as u8;
+
+                        // The readiness state could have been cleared in the meantime,
+                        // but we allow the returned ready set to be empty.
+                        let curr_ready = Ready::from_usize(READINESS.unpack(curr));
+                        let ready = curr_ready.intersection(w.interest);
+
                         return Poll::Ready(ReadyEvent {
                             tick,
-                            ready: Ready::from_interest(w.interest),
+                            ready,
+                            is_shutdown,
                         });
                     }
                 }

tokio/src/runtime/mod.rs

@@ -228,12 +228,18 @@ cfg_rt! {
         pub use crate::util::rand::RngSeed;
     }
 
+    mod defer;
+    pub(crate) use defer::Defer;
+
     mod handle;
     pub use handle::{EnterGuard, Handle, TryCurrentError};
 
     mod runtime;
     pub use runtime::{Runtime, RuntimeFlavor};
 
+    mod thread_id;
+    pub(crate) use thread_id::ThreadId;
+
     cfg_metrics! {
         mod metrics;
         pub use metrics::RuntimeMetrics;

tokio/src/runtime/park.rs

@@ -32,6 +32,12 @@ tokio_thread_local! {
     static CURRENT_PARKER: ParkThread = ParkThread::new();
 }
 
+// Bit of a hack, but it is only for loom
+#[cfg(loom)]
+tokio_thread_local! {
+    static CURRENT_THREAD_PARK_COUNT: AtomicUsize = AtomicUsize::new(0);
+}
+
 // ==== impl ParkThread ====
 
 impl ParkThread {
@@ -51,10 +57,15 @@ impl ParkThread {
     }
 
     pub(crate) fn park(&mut self) {
+        #[cfg(loom)]
+        CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
         self.inner.park();
     }
 
     pub(crate) fn park_timeout(&mut self, duration: Duration) {
+        #[cfg(loom)]
+        CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
+
         // Wasm doesn't have threads, so just sleep.
         #[cfg(not(tokio_wasm))]
         self.inner.park_timeout(duration);
@@ -273,6 +284,11 @@ impl CachedParkThread {
                 return Ok(v);
             }
 
+            // Wake any yielded tasks before parking in order to avoid
+            // blocking.
+            #[cfg(feature = "rt")]
+            crate::runtime::context::with_defer(|defer| defer.wake());
+
             self.park();
         }
     }
@@ -330,3 +346,8 @@ unsafe fn wake_by_ref(raw: *const ()) {
     // We don't actually own a reference to the unparker
     mem::forget(unparker);
 }
+
+#[cfg(loom)]
+pub(crate) fn current_thread_park_count() -> usize {
+    CURRENT_THREAD_PARK_COUNT.with(|count| count.load(SeqCst))
+}

tokio/src/runtime/runtime.rs

@@ -138,6 +138,9 @@ impl Runtime {
     /// The returned handle can be used to spawn tasks that run on this runtime, and can
     /// be cloned to allow moving the `Handle` to other threads.
     ///
+    /// Calling [`Handle::block_on`] on a handle to a `current_thread` runtime is error-prone.
+    /// Refer to the documentation of [`Handle::block_on`] for more.
+    ///
     /// # Examples
     ///
     /// ```

tokio/src/runtime/scheduler/current_thread.rs

@@ -3,7 +3,7 @@ use crate::loom::sync::atomic::AtomicBool;
 use crate::loom::sync::{Arc, Mutex};
 use crate::runtime::driver::{self, Driver};
 use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task};
-use crate::runtime::{blocking, scheduler, Config};
+use crate::runtime::{blocking, context, scheduler, Config};
 use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
 use crate::sync::notify::Notify;
 use crate::util::atomic_cell::AtomicCell;
@@ -267,6 +267,14 @@ impl Core {
     }
 }
 
+fn did_defer_tasks() -> bool {
+    context::with_defer(|deferred| !deferred.is_empty()).unwrap()
+}
+
+fn wake_deferred_tasks() {
+    context::with_defer(|deferred| deferred.wake());
+}
+
 // ===== impl Context =====
 
 impl Context {
@@ -299,6 +307,7 @@ impl Context {
 
             let (c, _) = self.enter(core, || {
                 driver.park(&handle.driver);
+                wake_deferred_tasks();
             });
 
             core = c;
@@ -324,6 +333,7 @@ impl Context {
         core.metrics.submit(&handle.shared.worker_metrics);
         let (mut core, _) = self.enter(core, || {
             driver.park_timeout(&handle.driver, Duration::from_millis(0));
+            wake_deferred_tasks();
         });
 
         core.driver = Some(driver);
@@ -557,7 +567,11 @@ impl CoreGuard<'_> {
                     let task = match entry {
                         Some(entry) => entry,
                         None => {
-                            core = context.park(core, handle);
+                            core = if did_defer_tasks() {
+                                context.park_yield(core, handle)
+                            } else {
+                                context.park(core, handle)
+                            };
 
                             // Try polling the `block_on` future next
                             continue 'outer;

tokio/src/runtime/scheduler/multi_thread/queue.rs

@@ -263,7 +263,7 @@ impl<T> Local<T> {
         // safety: The CAS above ensures that no consumer will look at these
         // values again, and we are the only producer.
         let batch_iter = BatchTaskIter {
-            buffer: &*self.inner.buffer,
+            buffer: &self.inner.buffer,
             head: head as UnsignedLong,
             i: 0,
         };

tokio/src/runtime/scheduler/multi_thread/worker.rs

@@ -368,6 +368,22 @@ impl Launch {
 }
 
 fn run(worker: Arc<Worker>) {
+    struct AbortOnPanic;
+
+    impl Drop for AbortOnPanic {
+        fn drop(&mut self) {
+            if std::thread::panicking() {
+                eprintln!("worker thread panicking; aborting process");
+                std::process::abort();
+            }
+        }
+    }
+
+    // Catching panics on worker threads in tests is quite tricky. Instead, when
+    // debug assertions are enabled, we just abort the process.
+    #[cfg(debug_assertions)]
+    let _abort_on_panic = AbortOnPanic;
+
     // Acquire a core. If this fails, then another thread is running this
     // worker and there is nothing further to do.
     let core = match worker.core.take() {
@@ -388,6 +404,11 @@ fn run(worker: Arc<Worker>) {
         // This should always be an error. It only returns a `Result` to support
         // using `?` to short circuit.
         assert!(cx.run(core).is_err());
+
+        // Check if there are any deferred tasks to notify. This can happen when
+        // the worker core is lost due to `block_in_place()` being called from
+        // within the task.
+        wake_deferred_tasks();
     });
 }
 
@@ -412,7 +433,11 @@ impl Context {
                 core = self.run_task(task, core)?;
             } else {
                 // Wait for work
-                core = self.park(core);
+                core = if did_defer_tasks() {
+                    self.park_timeout(core, Some(Duration::from_millis(0)))
+                } else {
+                    self.park(core)
+                };
             }
         }
 
@@ -535,6 +560,8 @@ impl Context {
             park.park(&self.worker.handle.driver);
         }
 
+        wake_deferred_tasks();
+
         // Remove `core` from context
         core = self.core.borrow_mut().take().expect("core missing");
 
@@ -853,6 +880,14 @@ impl Handle {
     }
 }
 
+fn did_defer_tasks() -> bool {
+    context::with_defer(|deferred| !deferred.is_empty()).unwrap()
+}
+
+fn wake_deferred_tasks() {
+    context::with_defer(|deferred| deferred.wake());
+}
+
 cfg_metrics! {
     impl Shared {
         pub(super) fn injection_queue_depth(&self) -> usize {

tokio/src/runtime/task/harness.rs

@@ -194,7 +194,7 @@ where
             TransitionToRunning::Success => {
                 let header_ptr = self.header_ptr();
                 let waker_ref = waker_ref::<T, S>(&header_ptr);
-                let cx = Context::from_waker(&*waker_ref);
+                let cx = Context::from_waker(&waker_ref);
                 let res = poll_future(self.core(), cx);
 
                 if res == Poll::Ready(()) {
@@ -315,9 +315,10 @@ where
                 // this task. It is our responsibility to drop the
                 // output.
                 self.core().drop_future_or_output();
-            } else if snapshot.has_join_waker() {
-                // Notify the join handle. The previous transition obtains the
-                // lock on the waker cell.
+            } else if snapshot.is_join_waker_set() {
+                // Notify the waker. Reading the waker field is safe per rule 4
+                // in task/mod.rs, since the JOIN_WAKER bit is set and the call
+                // to transition_to_complete() above set the COMPLETE bit.
                 self.trailer().wake_join();
             }
         }));
@@ -367,36 +368,30 @@ fn can_read_output(header: &Header, trailer: &Trailer, waker: &Waker) -> bool {
     debug_assert!(snapshot.is_join_interested());
 
     if !snapshot.is_complete() {
-        // The waker must be stored in the task struct.
-        let res = if snapshot.has_join_waker() {
-            // There already is a waker stored in the struct. If it matches
-            // the provided waker, then there is no further work to do.
-            // Otherwise, the waker must be swapped.
-            let will_wake = unsafe {
-                // Safety: when `JOIN_INTEREST` is set, only `JOIN_HANDLE`
-                // may mutate the `waker` field.
-                trailer.will_wake(waker)
-            };
+        // If the task is not complete, try storing the provided waker in the
+        // task's waker field.
 
-            if will_wake {
-                // The task is not complete **and** the waker is up to date,
-                // there is nothing further that needs to be done.
+        let res = if snapshot.is_join_waker_set() {
+            // If JOIN_WAKER is set, then JoinHandle has previously stored a
+            // waker in the waker field per step (iii) of rule 5 in task/mod.rs.
+
+            // Optimization: if the stored waker and the provided waker wake the
+            // same task, then return without touching the waker field. (Reading
+            // the waker field below is safe per rule 3 in task/mod.rs.)
+            if unsafe { trailer.will_wake(waker) } {
                 return false;
             }
 
-            // Unset the `JOIN_WAKER` to gain mutable access to the `waker`
-            // field then update the field with the new join worker.
-            //
-            // This requires two atomic operations, unsetting the bit and
-            // then resetting it. If the task transitions to complete
-            // concurrently to either one of those operations, then setting
-            // the join waker fails and we proceed to reading the task
-            // output.
+            // Otherwise swap the stored waker with the provided waker by
+            // following the rule 5 in task/mod.rs.
             header
                 .state
                 .unset_waker()
                 .and_then(|snapshot| set_join_waker(header, trailer, waker.clone(), snapshot))
         } else {
+            // If JOIN_WAKER is unset, then JoinHandle has mutable access to the
+            // waker field per rule 2 in task/mod.rs; therefore, skip step (i)
+            // of rule 5 and try to store the provided waker in the waker field.
             set_join_waker(header, trailer, waker.clone(), snapshot)
         };
 
@@ -417,7 +412,7 @@ fn set_join_waker(
     snapshot: Snapshot,
 ) -> Result<Snapshot, Snapshot> {
     assert!(snapshot.is_join_interested());
-    assert!(!snapshot.has_join_waker());
+    assert!(!snapshot.is_join_waker_set());
 
     // Safety: Only the `JoinHandle` may set the `waker` field. When
     // `JOIN_INTEREST` is **not** set, nothing else will touch the field.

tokio/src/runtime/task/id.rs

@@ -0,0 +1,87 @@
+use crate::runtime::context;
+
+use std::fmt;
+
+/// An opaque ID that uniquely identifies a task relative to all other currently
+/// running tasks.
+///
+/// # Notes
+///
+/// - Task IDs are unique relative to other *currently running* tasks. When a
+///   task completes, the same ID may be used for another task.
+/// - Task IDs are *not* sequential, and do not indicate the order in which
+///   tasks are spawned, what runtime a task is spawned on, or any other data.
+/// - The task ID of the currently running task can be obtained from inside the
+///   task via the [`task::try_id()`](crate::task::try_id()) and
+///   [`task::id()`](crate::task::id()) functions and from outside the task via
+///   the [`JoinHandle::id()`](crate::task::JoinHandle::id()) function.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [unstable]: crate#unstable-features
+#[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))]
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
+pub struct Id(u64);
+
+/// Returns the [`Id`] of the currently running task.
+///
+/// # Panics
+///
+/// This function panics if called from outside a task. Please note that calls
+/// to `block_on` do not have task IDs, so the method will panic if called from
+/// within a call to `block_on`. For a version of this function that doesn't
+/// panic, see [`task::try_id()`](crate::runtime::task::try_id()).
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [task ID]: crate::task::Id
+/// [unstable]: crate#unstable-features
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[track_caller]
+pub fn id() -> Id {
+    context::current_task_id().expect("Can't get a task id when not inside a task")
+}
+
+/// Returns the [`Id`] of the currently running task, or `None` if called outside
+/// of a task.
+///
+/// This function is similar to  [`task::id()`](crate::runtime::task::id()), except
+/// that it returns `None` rather than panicking if called outside of a task
+/// context.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [task ID]: crate::task::Id
+/// [unstable]: crate#unstable-features
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[track_caller]
+pub fn try_id() -> Option<Id> {
+    context::current_task_id()
+}
+
+impl fmt::Display for Id {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl Id {
+    pub(crate) fn next() -> Self {
+        use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64};
+
+        static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1);
+
+        Self(NEXT_ID.fetch_add(1, Relaxed))
+    }
+
+    pub(crate) fn as_u64(&self) -> u64 {
+        self.0
+    }
+}

tokio/src/runtime/task/mod.rs

@@ -168,19 +168,20 @@
 // unstable. This should be removed once `JoinSet` is stabilized.
 #![cfg_attr(not(tokio_unstable), allow(dead_code))]
 
-use crate::runtime::context;
-
 mod core;
 use self::core::Cell;
 use self::core::Header;
 
 mod error;
-#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411
 pub use self::error::JoinError;
 
 mod harness;
 use self::harness::Harness;
 
+mod id;
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+pub use id::{id, try_id, Id};
+
 cfg_rt_multi_thread! {
     mod inject;
     pub(super) use self::inject::Inject;
@@ -191,10 +192,8 @@ mod abort;
 mod join;
 
 #[cfg(feature = "rt")]
-#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411
 pub use self::abort::AbortHandle;
 
-#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411
 pub use self::join::JoinHandle;
 
 mod list;
@@ -215,70 +214,6 @@ use std::marker::PhantomData;
 use std::ptr::NonNull;
 use std::{fmt, mem};
 
-/// An opaque ID that uniquely identifies a task relative to all other currently
-/// running tasks.
-///
-/// # Notes
-///
-/// - Task IDs are unique relative to other *currently running* tasks. When a
-///   task completes, the same ID may be used for another task.
-/// - Task IDs are *not* sequential, and do not indicate the order in which
-///   tasks are spawned, what runtime a task is spawned on, or any other data.
-/// - The task ID of the currently running task can be obtained from inside the
-///   task via the [`task::try_id()`](crate::task::try_id()) and
-///   [`task::id()`](crate::task::id()) functions and from outside the task via
-///   the [`JoinHandle::id()`](crate::task::JoinHandle::id()) function.
-///
-/// **Note**: This is an [unstable API][unstable]. The public API of this type
-/// may break in 1.x releases. See [the documentation on unstable
-/// features][unstable] for details.
-///
-/// [unstable]: crate#unstable-features
-#[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))]
-#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
-#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
-pub struct Id(u64);
-
-/// Returns the [`Id`] of the currently running task.
-///
-/// # Panics
-///
-/// This function panics if called from outside a task. Please note that calls
-/// to `block_on` do not have task IDs, so the method will panic if called from
-/// within a call to `block_on`. For a version of this function that doesn't
-/// panic, see [`task::try_id()`](crate::runtime::task::try_id()).
-///
-/// **Note**: This is an [unstable API][unstable]. The public API of this type
-/// may break in 1.x releases. See [the documentation on unstable
-/// features][unstable] for details.
-///
-/// [task ID]: crate::task::Id
-/// [unstable]: crate#unstable-features
-#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
-#[track_caller]
-pub fn id() -> Id {
-    context::current_task_id().expect("Can't get a task id when not inside a task")
-}
-
-/// Returns the [`Id`] of the currently running task, or `None` if called outside
-/// of a task.
-///
-/// This function is similar to  [`task::id()`](crate::runtime::task::id()), except
-/// that it returns `None` rather than panicking if called outside of a task
-/// context.
-///
-/// **Note**: This is an [unstable API][unstable]. The public API of this type
-/// may break in 1.x releases. See [the documentation on unstable
-/// features][unstable] for details.
-///
-/// [task ID]: crate::task::Id
-/// [unstable]: crate#unstable-features
-#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
-#[track_caller]
-pub fn try_id() -> Option<Id> {
-    context::current_task_id()
-}
-
 /// An owned handle to the task, tracked by ref count.
 #[repr(transparent)]
 pub(crate) struct Task<S: 'static> {
@@ -554,66 +489,3 @@ unsafe impl<S> linked_list::Link for Task<S> {
         self::core::Trailer::addr_of_owned(Header::get_trailer(target))
     }
 }
-
-impl fmt::Display for Id {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl Id {
-    // When 64-bit atomics are available, use a static `AtomicU64` counter to
-    // generate task IDs.
-    //
-    // Note(eliza): we _could_ just use `crate::loom::AtomicU64`, which switches
-    // between an atomic and mutex-based implementation here, rather than having
-    // two separate functions for targets with and without 64-bit atomics.
-    // However, because we can't use the mutex-based implementation in a static
-    // initializer directly, the 32-bit impl also has to use a `OnceCell`, and I
-    // thought it was nicer to avoid the `OnceCell` overhead on 64-bit
-    // platforms...
-    cfg_has_atomic_u64! {
-        pub(crate) fn next() -> Self {
-            use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
-            static NEXT_ID: AtomicU64 = AtomicU64::new(1);
-            Self(NEXT_ID.fetch_add(1, Relaxed))
-        }
-    }
-
-    cfg_not_has_atomic_u64! {
-        cfg_has_const_mutex_new! {
-            pub(crate) fn next() -> Self {
-                use crate::loom::sync::Mutex;
-                static NEXT_ID: Mutex<u64> = Mutex::const_new(1);
-
-                let mut lock = NEXT_ID.lock();
-                let id = *lock;
-                *lock += 1;
-                Self(id)
-            }
-        }
-
-        cfg_not_has_const_mutex_new! {
-            pub(crate) fn next() -> Self {
-                use crate::util::once_cell::OnceCell;
-                use crate::loom::sync::Mutex;
-
-                fn init_next_id() -> Mutex<u64> {
-                    Mutex::new(1)
-                }
-
-                static NEXT_ID: OnceCell<Mutex<u64>> = OnceCell::new();
-
-                let next_id = NEXT_ID.get(init_next_id);
-                let mut lock = next_id.lock();
-                let id = *lock;
-                *lock += 1;
-                Self(id)
-            }
-        }
-    }
-
-    pub(crate) fn as_u64(&self) -> u64 {
-        self.0
-    }
-}

tokio/src/runtime/task/state.rs

@@ -378,7 +378,7 @@ impl State {
     pub(super) fn set_join_waker(&self) -> UpdateResult {
         self.fetch_update(|curr| {
             assert!(curr.is_join_interested());
-            assert!(!curr.has_join_waker());
+            assert!(!curr.is_join_waker_set());
 
             if curr.is_complete() {
                 return None;
@@ -398,7 +398,7 @@ impl State {
     pub(super) fn unset_waker(&self) -> UpdateResult {
         self.fetch_update(|curr| {
             assert!(curr.is_join_interested());
-            assert!(curr.has_join_waker());
+            assert!(curr.is_join_waker_set());
 
             if curr.is_complete() {
                 return None;
@@ -546,7 +546,7 @@ impl Snapshot {
         self.0 &= !JOIN_INTEREST
     }
 
-    pub(super) fn has_join_waker(self) -> bool {
+    pub(super) fn is_join_waker_set(self) -> bool {
         self.0 & JOIN_WAKER == JOIN_WAKER
     }
 
@@ -588,7 +588,7 @@ impl fmt::Debug for Snapshot {
             .field("is_notified", &self.is_notified())
             .field("is_cancelled", &self.is_cancelled())
             .field("is_join_interested", &self.is_join_interested())
-            .field("has_join_waker", &self.has_join_waker())
+            .field("is_join_waker_set", &self.is_join_waker_set())
             .field("ref_count", &self.ref_count())
             .finish()
     }

tokio/src/runtime/tests/loom_blocking.rs

@@ -73,6 +73,27 @@ fn spawn_mandatory_blocking_should_run_even_when_shutting_down_from_other_thread
     });
 }
 
+#[test]
+fn spawn_blocking_when_paused() {
+    use std::time::Duration;
+    loom::model(|| {
+        let rt = crate::runtime::Builder::new_current_thread()
+            .enable_time()
+            .start_paused(true)
+            .build()
+            .unwrap();
+        let handle = rt.handle();
+        let _enter = handle.enter();
+        let a = crate::task::spawn_blocking(|| {});
+        let b = crate::task::spawn_blocking(|| {});
+        rt.block_on(crate::time::timeout(Duration::from_millis(1), async move {
+            a.await.expect("blocking task should finish");
+            b.await.expect("blocking task should finish");
+        }))
+        .expect("timeout should not trigger");
+    });
+}
+
 fn mk_runtime(num_threads: usize) -> Runtime {
     runtime::Builder::new_multi_thread()
         .worker_threads(num_threads)

tokio/src/runtime/tests/loom_queue.rs

@@ -1,6 +1,6 @@
-use crate::runtime::blocking::NoopSchedule;
 use crate::runtime::scheduler::multi_thread::queue;
 use crate::runtime::task::Inject;
+use crate::runtime::tests::NoopSchedule;
 use crate::runtime::MetricsBatch;
 
 use loom::thread;

tokio/src/runtime/tests/loom_yield.rs

@@ -0,0 +1,37 @@
+use crate::runtime::park;
+use crate::runtime::tests::loom_oneshot as oneshot;
+use crate::runtime::{self, Runtime};
+
+#[test]
+fn yield_calls_park_before_scheduling_again() {
+    // Don't need to check all permutations
+    let mut loom = loom::model::Builder::default();
+    loom.max_permutations = Some(1);
+    loom.check(|| {
+        let rt = mk_runtime(2);
+        let (tx, rx) = oneshot::channel::<()>();
+
+        rt.spawn(async {
+            let tid = loom::thread::current().id();
+            let park_count = park::current_thread_park_count();
+
+            crate::task::yield_now().await;
+
+            if tid == loom::thread::current().id() {
+                let new_park_count = park::current_thread_park_count();
+                assert_eq!(park_count + 1, new_park_count);
+            }
+
+            tx.send(());
+        });
+
+        rx.recv();
+    });
+}
+
+fn mk_runtime(num_threads: usize) -> Runtime {
+    runtime::Builder::new_multi_thread()
+        .worker_threads(num_threads)
+        .build()
+        .unwrap()
+}

tokio/src/runtime/tests/mod.rs

@@ -2,11 +2,29 @@
 // other code when running loom tests.
 #![cfg_attr(loom, warn(dead_code, unreachable_pub))]
 
+use self::noop_scheduler::NoopSchedule;
 use self::unowned_wrapper::unowned;
 
+mod noop_scheduler {
+    use crate::runtime::task::{self, Task};
+
+    /// `task::Schedule` implementation that does nothing, for testing.
+    pub(crate) struct NoopSchedule;
+
+    impl task::Schedule for NoopSchedule {
+        fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> {
+            None
+        }
+
+        fn schedule(&self, _task: task::Notified<Self>) {
+            unreachable!();
+        }
+    }
+}
+
 mod unowned_wrapper {
-    use crate::runtime::blocking::NoopSchedule;
     use crate::runtime::task::{Id, JoinHandle, Notified};
+    use crate::runtime::tests::NoopSchedule;
 
     #[cfg(all(tokio_unstable, feature = "tracing"))]
     pub(crate) fn unowned<T>(task: T) -> (Notified<NoopSchedule>, JoinHandle<T::Output>)
@@ -41,6 +59,7 @@ cfg_loom! {
     mod loom_queue;
     mod loom_shutdown_join;
     mod loom_join_set;
+    mod loom_yield;
 }
 
 cfg_not_loom! {

tokio/src/runtime/tests/task.rs

@@ -1,5 +1,5 @@
-use crate::runtime::blocking::NoopSchedule;
 use crate::runtime::task::{self, unowned, Id, JoinHandle, OwnedTasks, Schedule, Task};
+use crate::runtime::tests::NoopSchedule;
 use crate::util::TryLock;
 
 use std::collections::VecDeque;

tokio/src/runtime/thread_id.rs

@@ -0,0 +1,31 @@
+use std::num::NonZeroU64;
+
+#[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)]
+pub(crate) struct ThreadId(NonZeroU64);
+
+impl ThreadId {
+    pub(crate) fn next() -> Self {
+        use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64};
+
+        static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(0);
+
+        let mut last = NEXT_ID.load(Relaxed);
+        loop {
+            let id = match last.checked_add(1) {
+                Some(id) => id,
+                None => exhausted(),
+            };
+
+            match NEXT_ID.compare_exchange_weak(last, id, Relaxed, Relaxed) {
+                Ok(_) => return ThreadId(NonZeroU64::new(id).unwrap()),
+                Err(id) => last = id,
+            }
+        }
+    }
+}
+
+#[cold]
+#[allow(dead_code)]
+fn exhausted() -> ! {
+    panic!("failed to generate unique thread ID: bitspace exhausted")
+}

tokio/src/runtime/time/mod.rs

@@ -222,7 +222,7 @@ impl Driver {
             let handle = rt_handle.time();
             let clock = &handle.time_source.clock;
 
-            if clock.is_paused() {
+            if clock.can_auto_advance() {
                 self.park.park_timeout(rt_handle, Duration::from_secs(0));
 
                 // If the time driver was woken, then the park completed

tokio/src/signal/registry.rs

@@ -5,7 +5,6 @@ use crate::sync::watch;
 use crate::util::once_cell::OnceCell;
 
 use std::ops;
-use std::pin::Pin;
 use std::sync::atomic::{AtomicBool, Ordering};
 
 pub(crate) type EventId = usize;
@@ -162,14 +161,14 @@ where
     }
 }
 
-pub(crate) fn globals() -> Pin<&'static Globals>
+pub(crate) fn globals() -> &'static Globals
 where
     OsExtraData: 'static + Send + Sync + Init,
     OsStorage: 'static + Send + Sync + Init,
 {
     static GLOBALS: OnceCell<Globals> = OnceCell::new();
 
-    Pin::new(GLOBALS.get(globals_init))
+    GLOBALS.get(globals_init)
 }
 
 #[cfg(all(test, not(loom)))]

tokio/src/signal/unix.rs

@@ -14,7 +14,6 @@ use crate::sync::watch;
 
 use mio::net::UnixStream;
 use std::io::{self, Error, ErrorKind, Write};
-use std::pin::Pin;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Once;
 use std::task::{Context, Poll};
@@ -240,7 +239,7 @@ impl Default for SignalInfo {
 /// 2. Wake up the driver by writing a byte to a pipe
 ///
 /// Those two operations should both be async-signal safe.
-fn action(globals: Pin<&'static Globals>, signal: libc::c_int) {
+fn action(globals: &'static Globals, signal: libc::c_int) {
     globals.record_event(signal as EventId);
 
     // Send a wakeup, ignore any errors (anything reasonably possible is

tokio/src/sync/broadcast.rs

@@ -18,6 +18,9 @@
 //! returned [`Receiver`] will receive values sent **after** the call to
 //! `subscribe`.
 //!
+//! This channel is also suitable for the single-producer multi-consumer
+//! use-case, where a single sender broadcasts values to many receivers.
+//!
 //! ## Lagging
 //!
 //! As sent messages must be retained until **all** [`Receiver`] handles receive

tokio/src/sync/mod.rs

@@ -94,6 +94,10 @@
 //! producers to a single consumer. This channel is often used to send work to a
 //! task or to receive the result of many computations.
 //!
+//! This is also the channel you should use if you want to send many messages
+//! from a single producer to a single consumer. There is no dedicated spsc
+//! channel.
+//!
 //! **Example:** using an mpsc to incrementally stream the results of a series
 //! of computations.
 //!
@@ -244,6 +248,10 @@
 //! This channel tends to be used less often than `oneshot` and `mpsc` but still
 //! has its use cases.
 //!
+//! This is also the channel you should use if you want to broadcast values from
+//! a single producer to many consumers. There is no dedicated spmc broadcast
+//! channel.
+//!
 //! Basic usage
 //!
 //! ```

tokio/src/sync/mpsc/block.rs

@@ -1,6 +1,7 @@
 use crate::loom::cell::UnsafeCell;
 use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize};
 
+use std::alloc::Layout;
 use std::mem::MaybeUninit;
 use std::ops;
 use std::ptr::{self, NonNull};
@@ -10,6 +11,17 @@ use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Release};
 ///
 /// Each block in the list can hold up to `BLOCK_CAP` messages.
 pub(crate) struct Block<T> {
+    /// The header fields.
+    header: BlockHeader<T>,
+
+    /// Array containing values pushed into the block. Values are stored in a
+    /// continuous array in order to improve cache line behavior when reading.
+    /// The values must be manually dropped.
+    values: Values<T>,
+}
+
+/// Extra fields for a `Block<T>`.
+struct BlockHeader<T> {
     /// The start index of this block.
     ///
     /// Slots in this block have indices in `start_index .. start_index + BLOCK_CAP`.
@@ -24,11 +36,6 @@ pub(crate) struct Block<T> {
     /// The observed `tail_position` value *after* the block has been passed by
     /// `block_tail`.
     observed_tail_position: UnsafeCell<usize>,
-
-    /// Array containing values pushed into the block. Values are stored in a
-    /// continuous array in order to improve cache line behavior when reading.
-    /// The values must be manually dropped.
-    values: Values<T>,
 }
 
 pub(crate) enum Read<T> {
@@ -36,6 +43,7 @@ pub(crate) enum Read<T> {
     Closed,
 }
 
+#[repr(transparent)]
 struct Values<T>([UnsafeCell<MaybeUninit<T>>; BLOCK_CAP]);
 
 use super::BLOCK_CAP;
@@ -71,28 +79,56 @@ pub(crate) fn offset(slot_index: usize) -> usize {
     SLOT_MASK & slot_index
 }
 
+generate_addr_of_methods! {
+    impl<T> Block<T> {
+        unsafe fn addr_of_header(self: NonNull<Self>) -> NonNull<BlockHeader<T>> {
+            &self.header
+        }
+
+        unsafe fn addr_of_values(self: NonNull<Self>) -> NonNull<Values<T>> {
+            &self.values
+        }
+    }
+}
+
 impl<T> Block<T> {
-    pub(crate) fn new(start_index: usize) -> Block<T> {
-        Block {
-            // The absolute index in the channel of the first slot in the block.
-            start_index,
+    pub(crate) fn new(start_index: usize) -> Box<Block<T>> {
+        unsafe {
+            // Allocate the block on the heap.
+            // SAFETY: The size of the Block<T> is non-zero, since it is at least the size of the header.
+            let block = std::alloc::alloc(Layout::new::<Block<T>>()) as *mut Block<T>;
+            let block = match NonNull::new(block) {
+                Some(block) => block,
+                None => std::alloc::handle_alloc_error(Layout::new::<Block<T>>()),
+            };
 
-            // Pointer to the next block in the linked list.
-            next: AtomicPtr::new(ptr::null_mut()),
+            // Write the header to the block.
+            Block::addr_of_header(block).as_ptr().write(BlockHeader {
+                // The absolute index in the channel of the first slot in the block.
+                start_index,
 
-            ready_slots: AtomicUsize::new(0),
+                // Pointer to the next block in the linked list.
+                next: AtomicPtr::new(ptr::null_mut()),
 
-            observed_tail_position: UnsafeCell::new(0),
+                ready_slots: AtomicUsize::new(0),
 
-            // Value storage
-            values: unsafe { Values::uninitialized() },
+                observed_tail_position: UnsafeCell::new(0),
+            });
+
+            // Initialize the values array.
+            Values::initialize(Block::addr_of_values(block));
+
+            // Convert the pointer to a `Box`.
+            // Safety: The raw pointer was allocated using the global allocator, and with
+            // the layout for a `Block<T>`, so it's valid to convert it to box.
+            Box::from_raw(block.as_ptr())
         }
     }
 
     /// Returns `true` if the block matches the given index.
     pub(crate) fn is_at_index(&self, index: usize) -> bool {
         debug_assert!(offset(index) == 0);
-        self.start_index == index
+        self.header.start_index == index
     }
 
     /// Returns the number of blocks between `self` and the block at the
@@ -101,7 +137,7 @@ impl<T> Block<T> {
     /// `start_index` must represent a block *after* `self`.
     pub(crate) fn distance(&self, other_index: usize) -> usize {
         debug_assert!(offset(other_index) == 0);
-        other_index.wrapping_sub(self.start_index) / BLOCK_CAP
+        other_index.wrapping_sub(self.header.start_index) / BLOCK_CAP
     }
 
     /// Reads the value at the given offset.
@@ -116,7 +152,7 @@ impl<T> Block<T> {
     pub(crate) unsafe fn read(&self, slot_index: usize) -> Option<Read<T>> {
         let offset = offset(slot_index);
 
-        let ready_bits = self.ready_slots.load(Acquire);
+        let ready_bits = self.header.ready_slots.load(Acquire);
 
         if !is_ready(ready_bits, offset) {
             if is_tx_closed(ready_bits) {
@@ -156,7 +192,7 @@ impl<T> Block<T> {
 
     /// Signal to the receiver that the sender half of the list is closed.
     pub(crate) unsafe fn tx_close(&self) {
-        self.ready_slots.fetch_or(TX_CLOSED, Release);
+        self.header.ready_slots.fetch_or(TX_CLOSED, Release);
     }
 
     /// Resets the block to a blank state. This enables reusing blocks in the
@@ -169,9 +205,9 @@ impl<T> Block<T> {
     /// * All slots are empty.
     /// * The caller holds a unique pointer to the block.
     pub(crate) unsafe fn reclaim(&mut self) {
-        self.start_index = 0;
-        self.next = AtomicPtr::new(ptr::null_mut());
-        self.ready_slots = AtomicUsize::new(0);
+        self.header.start_index = 0;
+        self.header.next = AtomicPtr::new(ptr::null_mut());
+        self.header.ready_slots = AtomicUsize::new(0);
     }
 
     /// Releases the block to the rx half for freeing.
@@ -187,19 +223,20 @@ impl<T> Block<T> {
     pub(crate) unsafe fn tx_release(&self, tail_position: usize) {
         // Track the observed tail_position. Any sender targeting a greater
         // tail_position is guaranteed to not access this block.
-        self.observed_tail_position
+        self.header
+            .observed_tail_position
             .with_mut(|ptr| *ptr = tail_position);
 
         // Set the released bit, signalling to the receiver that it is safe to
         // free the block's memory as soon as all slots **prior** to
         // `observed_tail_position` have been filled.
-        self.ready_slots.fetch_or(RELEASED, Release);
+        self.header.ready_slots.fetch_or(RELEASED, Release);
     }
 
     /// Mark a slot as ready
     fn set_ready(&self, slot: usize) {
         let mask = 1 << slot;
-        self.ready_slots.fetch_or(mask, Release);
+        self.header.ready_slots.fetch_or(mask, Release);
     }
 
     /// Returns `true` when all slots have their `ready` bits set.
@@ -214,25 +251,31 @@ impl<T> Block<T> {
     /// single atomic cell. However, this could have negative impact on cache
     /// behavior as there would be many more mutations to a single slot.
     pub(crate) fn is_final(&self) -> bool {
-        self.ready_slots.load(Acquire) & READY_MASK == READY_MASK
+        self.header.ready_slots.load(Acquire) & READY_MASK == READY_MASK
     }
 
     /// Returns the `observed_tail_position` value, if set
     pub(crate) fn observed_tail_position(&self) -> Option<usize> {
-        if 0 == RELEASED & self.ready_slots.load(Acquire) {
+        if 0 == RELEASED & self.header.ready_slots.load(Acquire) {
             None
         } else {
-            Some(self.observed_tail_position.with(|ptr| unsafe { *ptr }))
+            Some(
+                self.header
+                    .observed_tail_position
+                    .with(|ptr| unsafe { *ptr }),
+            )
         }
     }
 
     /// Loads the next block
     pub(crate) fn load_next(&self, ordering: Ordering) -> Option<NonNull<Block<T>>> {
-        let ret = NonNull::new(self.next.load(ordering));
+        let ret = NonNull::new(self.header.next.load(ordering));
 
         debug_assert!(unsafe {
-            ret.map(|block| block.as_ref().start_index == self.start_index.wrapping_add(BLOCK_CAP))
-                .unwrap_or(true)
+            ret.map(|block| {
+                block.as_ref().header.start_index == self.header.start_index.wrapping_add(BLOCK_CAP)
+            })
+            .unwrap_or(true)
         });
 
         ret
@@ -260,9 +303,10 @@ impl<T> Block<T> {
         success: Ordering,
         failure: Ordering,
     ) -> Result<(), NonNull<Block<T>>> {
-        block.as_mut().start_index = self.start_index.wrapping_add(BLOCK_CAP);
+        block.as_mut().header.start_index = self.header.start_index.wrapping_add(BLOCK_CAP);
 
         let next_ptr = self
+            .header
             .next
             .compare_exchange(ptr::null_mut(), block.as_ptr(), success, failure)
             .unwrap_or_else(|x| x);
@@ -291,7 +335,7 @@ impl<T> Block<T> {
         // Create the new block. It is assumed that the block will become the
         // next one after `&self`. If this turns out to not be the case,
         // `start_index` is updated accordingly.
-        let new_block = Box::new(Block::new(self.start_index + BLOCK_CAP));
+        let new_block = Block::new(self.header.start_index + BLOCK_CAP);
 
         let mut new_block = unsafe { NonNull::new_unchecked(Box::into_raw(new_block)) };
 
@@ -308,7 +352,8 @@ impl<T> Block<T> {
         // `Release` ensures that the newly allocated block is available to
         // other threads acquiring the next pointer.
         let next = NonNull::new(
-            self.next
+            self.header
+                .next
                 .compare_exchange(ptr::null_mut(), new_block.as_ptr(), AcqRel, Acquire)
                 .unwrap_or_else(|x| x),
         );
@@ -360,19 +405,20 @@ fn is_tx_closed(bits: usize) -> bool {
 }
 
 impl<T> Values<T> {
-    unsafe fn uninitialized() -> Values<T> {
-        let mut vals = MaybeUninit::uninit();
-
+    /// Initialize a `Values` struct from a pointer.
+    ///
+    /// # Safety
+    ///
+    /// The raw pointer must be valid for writing a `Values<T>`.
+    unsafe fn initialize(_value: NonNull<Values<T>>) {
         // When fuzzing, `UnsafeCell` needs to be initialized.
         if_loom! {
-            let p = vals.as_mut_ptr() as *mut UnsafeCell<MaybeUninit<T>>;
+            let p = _value.as_ptr() as *mut UnsafeCell<MaybeUninit<T>>;
             for i in 0..BLOCK_CAP {
                 p.add(i)
                     .write(UnsafeCell::new(MaybeUninit::uninit()));
             }
         }
-
-        Values(vals.assume_init())
     }
 }
 
@@ -383,3 +429,20 @@ impl<T> ops::Index<usize> for Values<T> {
         self.0.index(index)
     }
 }
+
+#[cfg(all(test, not(loom)))]
+#[test]
+fn assert_no_stack_overflow() {
+    // https://github.com/tokio-rs/tokio/issues/5293
+
+    struct Foo {
+        _a: [u8; 2_000_000],
+    }
+
+    assert_eq!(
+        Layout::new::<MaybeUninit<Block<Foo>>>(),
+        Layout::new::<Block<Foo>>()
+    );
+
+    let _block = Block::<Foo>::new(0);
+}

tokio/src/sync/mpsc/list.rs

@@ -44,7 +44,7 @@ pub(crate) enum TryPopResult<T> {
 
 pub(crate) fn channel<T>() -> (Tx<T>, Rx<T>) {
     // Create the initial block shared between the tx and rx halves.
-    let initial_block = Box::new(Block::new(0));
+    let initial_block = Block::new(0);
     let initial_block_ptr = Box::into_raw(initial_block);
 
     let tx = Tx {

tokio/src/sync/mpsc/mod.rs

@@ -21,6 +21,9 @@
 //! when additional capacity is available. In other words, the channel provides
 //! backpressure.
 //!
+//! This channel is also suitable for the single-producer single-consumer
+//! use-case. (Unless you only need to send one message, in which case you
+//! should use the [oneshot] channel.)
 //!
 //! # Disconnection
 //!
@@ -62,7 +65,7 @@
 //! in mind, but they can also be generalized to other kinds of channels. In
 //! general, any channel method that isn't marked async can be called anywhere,
 //! including outside of the runtime. For example, sending a message on a
-//! oneshot channel from outside the runtime is perfectly fine.
+//! [oneshot] channel from outside the runtime is perfectly fine.
 //!
 //! # Multiple runtimes
 //!
@@ -82,6 +85,7 @@
 //! [blocking-recv]: crate::sync::mpsc::Receiver::blocking_recv()
 //! [`UnboundedSender`]: crate::sync::mpsc::UnboundedSender
 //! [`UnboundedReceiver`]: crate::sync::mpsc::UnboundedReceiver
+//! [oneshot]: crate::sync::oneshot
 //! [`Handle::block_on`]: crate::runtime::Handle::block_on()
 //! [std-unbounded]: std::sync::mpsc::channel
 //! [crossbeam-unbounded]: https://docs.rs/crossbeam/*/crossbeam/channel/fn.unbounded.html

tokio/src/sync/watch.rs

@@ -9,10 +9,10 @@
 //! # Usage
 //!
 //! [`channel`] returns a [`Sender`] / [`Receiver`] pair. These are the producer
-//! and sender halves of the channel. The channel is created with an initial
+//! and consumer halves of the channel. The channel is created with an initial
 //! value. The **latest** value stored in the channel is accessed with
 //! [`Receiver::borrow()`]. Awaiting [`Receiver::changed()`] waits for a new
-//! value to sent by the [`Sender`] half.
+//! value to be sent by the [`Sender`] half.
 //!
 //! # Examples
 //!
@@ -90,10 +90,11 @@ pub struct Sender<T> {
 /// Returns a reference to the inner value.
 ///
 /// Outstanding borrows hold a read lock on the inner value. This means that
-/// long lived borrows could cause the produce half to block. It is recommended
-/// to keep the borrow as short lived as possible. Additionally, if you are
+/// long-lived borrows could cause the producer half to block. It is recommended
+/// to keep the borrow as short-lived as possible. Additionally, if you are
 /// running in an environment that allows `!Send` futures, you must ensure that
-/// the returned `Ref` type is never held alive across an `.await` point.
+/// the returned `Ref` type is never held alive across an `.await` point,
+/// otherwise, it can lead to a deadlock.
 ///
 /// The priority policy of the lock is dependent on the underlying lock
 /// implementation, and this type does not guarantee that any particular policy
@@ -350,11 +351,12 @@ impl<T> Receiver<T> {
     /// [`changed`] may return immediately even if you have already seen the
     /// value with a call to `borrow`.
     ///
-    /// Outstanding borrows hold a read lock. This means that long lived borrows
-    /// could cause the send half to block. It is recommended to keep the borrow
-    /// as short lived as possible. Additionally, if you are running in an
-    /// environment that allows `!Send` futures, you must ensure that the
-    /// returned `Ref` type is never held alive across an `.await` point.
+    /// Outstanding borrows hold a read lock on the inner value. This means that
+    /// long-lived borrows could cause the producer half to block. It is recommended
+    /// to keep the borrow as short-lived as possible. Additionally, if you are
+    /// running in an environment that allows `!Send` futures, you must ensure that
+    /// the returned `Ref` type is never held alive across an `.await` point,
+    /// otherwise, it can lead to a deadlock.
     ///
     /// The priority policy of the lock is dependent on the underlying lock
     /// implementation, and this type does not guarantee that any particular policy
@@ -401,11 +403,12 @@ impl<T> Receiver<T> {
     /// will not return immediately until the [`Sender`] has modified the shared
     /// value again.
     ///
-    /// Outstanding borrows hold a read lock. This means that long lived borrows
-    /// could cause the send half to block. It is recommended to keep the borrow
-    /// as short lived as possible. Additionally, if you are running in an
-    /// environment that allows `!Send` futures, you must ensure that the
-    /// returned `Ref` type is never held alive across an `.await` point.
+    /// Outstanding borrows hold a read lock on the inner value. This means that
+    /// long-lived borrows could cause the producer half to block. It is recommended
+    /// to keep the borrow as short-lived as possible. Additionally, if you are
+    /// running in an environment that allows `!Send` futures, you must ensure that
+    /// the returned `Ref` type is never held alive across an `.await` point,
+    /// otherwise, it can lead to a deadlock.
     ///
     /// The priority policy of the lock is dependent on the underlying lock
     /// implementation, and this type does not guarantee that any particular policy
@@ -794,9 +797,12 @@ impl<T> Sender<T> {
 
     /// Returns a reference to the most recently sent value
     ///
-    /// Outstanding borrows hold a read lock. This means that long lived borrows
-    /// could cause the send half to block. It is recommended to keep the borrow
-    /// as short lived as possible.
+    /// Outstanding borrows hold a read lock on the inner value. This means that
+    /// long-lived borrows could cause the producer half to block. It is recommended
+    /// to keep the borrow as short-lived as possible. Additionally, if you are
+    /// running in an environment that allows `!Send` futures, you must ensure that
+    /// the returned `Ref` type is never held alive across an `.await` point,
+    /// otherwise, it can lead to a deadlock.
     ///
     /// # Examples
     ///

tokio/src/task/local.rs

@@ -1,8 +1,8 @@
 //! Runs `!Send` futures on the current thread.
 use crate::loom::cell::UnsafeCell;
 use crate::loom::sync::{Arc, Mutex};
-use crate::loom::thread::{self, ThreadId};
 use crate::runtime::task::{self, JoinHandle, LocalOwnedTasks, Task};
+use crate::runtime::{context, ThreadId};
 use crate::sync::AtomicWaker;
 use crate::util::RcCell;
 
@@ -277,12 +277,10 @@ pin_project! {
 }
 
 tokio_thread_local!(static CURRENT: LocalData = const { LocalData {
-    thread_id: Cell::new(None),
     ctx: RcCell::new(),
 } });
 
 struct LocalData {
-    thread_id: Cell<Option<ThreadId>>,
     ctx: RcCell<Context>,
 }
 
@@ -379,12 +377,14 @@ impl fmt::Debug for LocalEnterGuard {
 impl LocalSet {
     /// Returns a new local task set.
     pub fn new() -> LocalSet {
+        let owner = context::thread_id().expect("cannot create LocalSet during thread shutdown");
+
         LocalSet {
             tick: Cell::new(0),
             context: Rc::new(Context {
                 shared: Arc::new(Shared {
                     local_state: LocalState {
-                        owner: thread_id().expect("cannot create LocalSet during thread shutdown"),
+                        owner,
                         owned: LocalOwnedTasks::new(),
                         local_queue: UnsafeCell::new(VecDeque::with_capacity(INITIAL_CAPACITY)),
                     },
@@ -949,7 +949,7 @@ impl Shared {
 
                 // We are on the thread that owns the `LocalSet`, so we can
                 // wake to the local queue.
-                _ if localdata.get_id() == Some(self.local_state.owner) => {
+                _ if context::thread_id().ok() == Some(self.local_state.owner) => {
                     unsafe {
                         // Safety: we just checked that the thread ID matches
                         // the localset's owner, so this is safe.
@@ -1093,7 +1093,9 @@ impl LocalState {
             // if we couldn't get the thread ID because we're dropping the local
             // data, skip the assertion --- the `Drop` impl is not going to be
             // called from another thread, because `LocalSet` is `!Send`
-            thread_id().map(|id| id == self.owner).unwrap_or(true),
+            context::thread_id()
+                .map(|id| id == self.owner)
+                .unwrap_or(true),
             "`LocalSet`'s local run queue must not be accessed by another thread!"
         );
     }
@@ -1103,26 +1105,6 @@ impl LocalState {
 // ensure they are on the same thread that owns the `LocalSet`.
 unsafe impl Send for LocalState {}
 
-impl LocalData {
-    fn get_id(&self) -> Option<ThreadId> {
-        self.thread_id.get()
-    }
-
-    fn get_or_insert_id(&self) -> ThreadId {
-        self.thread_id.get().unwrap_or_else(|| {
-            let id = thread::current().id();
-            self.thread_id.set(Some(id));
-            id
-        })
-    }
-}
-
-fn thread_id() -> Option<ThreadId> {
-    CURRENT
-        .try_with(|localdata| localdata.get_or_insert_id())
-        .ok()
-}
-
 #[cfg(all(test, not(loom)))]
 mod tests {
     use super::*;

tokio/src/task/yield_now.rs

@@ -1,3 +1,5 @@
+use crate::runtime::context;
+
 use std::future::Future;
 use std::pin::Pin;
 use std::task::{Context, Poll};
@@ -49,7 +51,17 @@ pub async fn yield_now() {
             }
 
             self.yielded = true;
-            cx.waker().wake_by_ref();
+
+            let defer = context::with_defer(|rt| {
+                rt.defer(cx.waker().clone());
+            });
+
+            if defer.is_none() {
+                //  Not currently in a runtime, just notify ourselves
+                //  immediately.
+                cx.waker().wake_by_ref();
+            }
+
             Poll::Pending
         }
     }

tokio/src/time/clock.rs

@@ -65,6 +65,9 @@ cfg_test_util! {
 
         /// Instant at which the clock was last unfrozen.
         unfrozen: Option<std::time::Instant>,
+
+        /// Number of `inhibit_auto_advance` calls still in effect.
+        auto_advance_inhibit_count: usize,
     }
 
     /// Pauses time.
@@ -187,6 +190,7 @@ cfg_test_util! {
                     enable_pausing,
                     base: now,
                     unfrozen: Some(now),
+                    auto_advance_inhibit_count: 0,
                 })),
             };
 
@@ -212,9 +216,20 @@ cfg_test_util! {
             inner.unfrozen = None;
         }
 
-        pub(crate) fn is_paused(&self) -> bool {
+        /// Temporarily stop auto-advancing the clock (see `tokio::time::pause`).
+        pub(crate) fn inhibit_auto_advance(&self) {
+            let mut inner = self.inner.lock();
+            inner.auto_advance_inhibit_count += 1;
+        }
+
+        pub(crate) fn allow_auto_advance(&self) {
+            let mut inner = self.inner.lock();
+            inner.auto_advance_inhibit_count -= 1;
+        }
+
+        pub(crate) fn can_auto_advance(&self) -> bool {
             let inner = self.inner.lock();
-            inner.unfrozen.is_none()
+            inner.unfrozen.is_none() && inner.auto_advance_inhibit_count == 0
         }
 
         #[track_caller]

tokio/src/time/sleep.rs

@@ -357,7 +357,7 @@ impl Sleep {
     fn reset_inner(self: Pin<&mut Self>, deadline: Instant) {
         let mut me = self.project();
         me.entry.as_mut().reset(deadline);
-        (*me.inner).deadline = deadline;
+        (me.inner).deadline = deadline;
 
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         {

tokio/src/util/linked_list.rs

@@ -126,7 +126,7 @@ impl<L: Link> LinkedList<L, L::Target> {
     pub(crate) fn push_front(&mut self, val: L::Handle) {
         // The value should not be dropped, it is being inserted into the list
         let val = ManuallyDrop::new(val);
-        let ptr = L::as_raw(&*val);
+        let ptr = L::as_raw(&val);
         assert_ne!(self.head, Some(ptr));
         unsafe {
             L::pointers(ptr).as_mut().set_next(self.head);

tokio/src/util/once_cell.rs

@@ -25,7 +25,7 @@ impl<T> OnceCell<T> {
     /// If the `init` closure panics, then the `OnceCell` is poisoned and all
     /// future calls to `get` will panic.
     #[inline]
-    pub(crate) fn get(&self, init: fn() -> T) -> &T {
+    pub(crate) fn get(&self, init: impl FnOnce() -> T) -> &T {
         if !self.once.is_completed() {
             self.do_init(init);
         }
@@ -41,7 +41,7 @@ impl<T> OnceCell<T> {
     }
 
     #[cold]
-    fn do_init(&self, init: fn() -> T) {
+    fn do_init(&self, init: impl FnOnce() -> T) {
         let value_ptr = self.value.get() as *mut T;
 
         self.once.call_once(|| {

tokio/src/util/rand.rs

@@ -46,7 +46,7 @@ cfg_rt! {
     }
 }
 
-/// A seed for random numnber generation.
+/// A seed for random number generation.
 ///
 /// In order to make certain functions within a runtime deterministic, a seed
 /// can be specified at the time of creation.

tokio/tests/_require_full.rs

@@ -1,2 +1,8 @@
-#![cfg(not(any(feature = "full", tokio_wasm)))]
+#[cfg(not(any(feature = "full", tokio_wasm)))]
 compile_error!("run main Tokio tests with `--features full`");
+
+// CI sets `--cfg tokio_no_parking_lot` when trying to run tests with
+// `parking_lot` disabled. This check prevents "silent failure" if `parking_lot`
+// accidentally gets enabled.
+#[cfg(all(tokio_no_parking_lot, feature = "parking_lot"))]
+compile_error!("parking_lot feature enabled when it should not be");

tokio/tests/buffered.rs

@@ -18,10 +18,10 @@ async fn echo_server() {
     let msg = "foo bar baz";
 
     let t = thread::spawn(move || {
-        let mut s = assert_ok!(TcpStream::connect(&addr));
+        let mut s = assert_ok!(TcpStream::connect(addr));
 
         let t2 = thread::spawn(move || {
-            let mut s = assert_ok!(TcpStream::connect(&addr));
+            let mut s = assert_ok!(TcpStream::connect(addr));
             let mut b = vec![0; msg.len() * N];
             assert_ok!(s.read_exact(&mut b));
             b

tokio/tests/io_driver.rs

@@ -80,7 +80,7 @@ fn test_drop_on_notify() {
     drop(task);
 
     // Establish a connection to the acceptor
-    let _s = TcpStream::connect(&addr).unwrap();
+    let _s = TcpStream::connect(addr).unwrap();
 
     // Force the reactor to turn
     rt.block_on(async {});

tokio/tests/macros_join.rs

@@ -1,5 +1,5 @@
 #![cfg(feature = "macros")]
-#![allow(clippy::blacklisted_name)]
+#![allow(clippy::disallowed_names)]
 use std::sync::Arc;
 
 #[cfg(tokio_wasm_not_wasi)]

tokio/tests/macros_select.rs

@@ -1,5 +1,5 @@
 #![cfg(feature = "macros")]
-#![allow(clippy::blacklisted_name)]
+#![allow(clippy::disallowed_names)]
 
 #[cfg(tokio_wasm_not_wasi)]
 use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;

tokio/tests/macros_try_join.rs

@@ -1,5 +1,5 @@
 #![cfg(feature = "macros")]
-#![allow(clippy::blacklisted_name)]
+#![allow(clippy::disallowed_names)]
 
 use std::sync::Arc;
 

tokio/tests/rt_common.rs

@@ -9,6 +9,9 @@ macro_rules! rt_test {
         mod current_thread_scheduler {
             $($t)*
 
+            #[cfg(not(target_os="wasi"))]
+            const NUM_WORKERS: usize = 1;
+
             fn rt() -> Arc<Runtime> {
                 tokio::runtime::Builder::new_current_thread()
                     .enable_all()
@@ -22,6 +25,8 @@ macro_rules! rt_test {
         mod threaded_scheduler_4_threads {
             $($t)*
 
+            const NUM_WORKERS: usize = 4;
+
             fn rt() -> Arc<Runtime> {
                 tokio::runtime::Builder::new_multi_thread()
                     .worker_threads(4)
@@ -36,6 +41,8 @@ macro_rules! rt_test {
         mod threaded_scheduler_1_thread {
             $($t)*
 
+            const NUM_WORKERS: usize = 1;
+
             fn rt() -> Arc<Runtime> {
                 tokio::runtime::Builder::new_multi_thread()
                     .worker_threads(1)
@@ -652,7 +659,12 @@ rt_test! {
         for _ in 0..100 {
             rt.spawn(async {
                 loop {
-                    tokio::task::yield_now().await;
+                    // Don't use Tokio's `yield_now()` to avoid special defer
+                    // logic.
+                    futures::future::poll_fn::<(), _>(|cx| {
+                        cx.waker().wake_by_ref();
+                        std::task::Poll::Pending
+                    }).await;
                 }
             });
         }
@@ -680,6 +692,71 @@ rt_test! {
         });
     }
 
+    /// Tests that yielded tasks are not scheduled until **after** resource
+    /// drivers are polled.
+    ///
+    /// Note: we may have to delete this test as it is not necessarily reliable.
+    /// The OS does not guarantee when I/O events are delivered, so there may be
+    /// more yields than anticipated.
+    #[test]
+    #[cfg(not(target_os="wasi"))]
+    fn yield_defers_until_park() {
+        use std::sync::atomic::{AtomicBool, Ordering::SeqCst};
+        use std::sync::Barrier;
+
+        let rt = rt();
+
+        let flag = Arc::new(AtomicBool::new(false));
+        let barrier = Arc::new(Barrier::new(NUM_WORKERS));
+
+        rt.block_on(async {
+            // Make sure other workers cannot steal tasks
+            #[allow(clippy::reversed_empty_ranges)]
+            for _ in 0..(NUM_WORKERS-1) {
+                let flag = flag.clone();
+                let barrier = barrier.clone();
+
+                tokio::spawn(async move {
+                    barrier.wait();
+
+                    while !flag.load(SeqCst) {
+                        std::thread::sleep(std::time::Duration::from_millis(1));
+                    }
+                });
+            }
+
+            barrier.wait();
+
+            tokio::spawn(async move {
+                // Create a TCP litener
+                let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+                let addr = listener.local_addr().unwrap();
+
+                tokio::join!(
+                    async {
+                        // Done blocking intentionally
+                        let _socket = std::net::TcpStream::connect(addr).unwrap();
+
+                        // Yield until connected
+                        let mut cnt = 0;
+                        while !flag.load(SeqCst){
+                            tokio::task::yield_now().await;
+                            cnt += 1;
+
+                            if cnt >= 10 {
+                                panic!("yielded too many times; TODO: delete this test?");
+                            }
+                        }
+                    },
+                    async {
+                        let _ = listener.accept().await.unwrap();
+                        flag.store(true, SeqCst);
+                    }
+                );
+            }).await.unwrap();
+        });
+    }
+
     #[cfg(not(target_os="wasi"))] // Wasi does not support threads
     #[test]
     fn client_server_block_on() {

tokio/tests/rt_metrics.rs

@@ -31,6 +31,19 @@ fn num_idle_blocking_threads() {
     rt.block_on(async {
         time::sleep(Duration::from_millis(5)).await;
     });
+
+    // We need to wait until the blocking thread has become idle. Usually 5ms is
+    // enough for this to happen, but not always. When it isn't enough, sleep
+    // for another second. We don't always wait for a whole second since we want
+    // the test suite to finish quickly.
+    //
+    // Note that the timeout for idle threads to be killed is 10 seconds.
+    if 0 == rt.metrics().num_idle_blocking_threads() {
+        rt.block_on(async {
+            time::sleep(Duration::from_secs(1)).await;
+        });
+    }
+
     assert_eq!(1, rt.metrics().num_idle_blocking_threads());
 }
 
@@ -128,7 +141,7 @@ fn worker_noop_count() {
         time::sleep(Duration::from_millis(1)).await;
     });
     drop(rt);
-    assert!(2 <= metrics.worker_noop_count(0));
+    assert!(0 < metrics.worker_noop_count(0));
 
     let rt = threaded();
     let metrics = rt.metrics();
@@ -136,8 +149,8 @@ fn worker_noop_count() {
         time::sleep(Duration::from_millis(1)).await;
     });
     drop(rt);
-    assert!(1 <= metrics.worker_noop_count(0));
-    assert!(1 <= metrics.worker_noop_count(1));
+    assert!(0 < metrics.worker_noop_count(0));
+    assert!(0 < metrics.worker_noop_count(1));
 }
 
 #[test]

tokio/tests/rt_threaded.rs

@@ -415,6 +415,32 @@ fn coop_and_block_in_place() {
     });
 }
 
+#[test]
+fn yield_after_block_in_place() {
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .build()
+        .unwrap();
+
+    rt.block_on(async {
+        tokio::spawn(async move {
+            // Block in place then enter a new runtime
+            tokio::task::block_in_place(|| {
+                let rt = tokio::runtime::Builder::new_current_thread()
+                    .build()
+                    .unwrap();
+
+                rt.block_on(async {});
+            });
+
+            // Yield, then complete
+            tokio::task::yield_now().await;
+        })
+        .await
+        .unwrap()
+    });
+}
+
 // Testing this does not panic
 #[test]
 fn max_blocking_threads() {

tokio/tests/support/leaked_buffers.rs

@@ -18,9 +18,9 @@ impl LeakedBuffers {
         }
     }
     pub unsafe fn create<'a>(&mut self, size: usize) -> &'a mut [u8] {
-        let mut new_mem = vec![0u8; size].into_boxed_slice();
-        let slice = std::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), new_mem.len());
+        let new_mem = vec![0u8; size].into_boxed_slice();
         self.leaked_vecs.push(new_mem);
-        slice
+        let new_mem = self.leaked_vecs.last_mut().unwrap();
+        std::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), new_mem.len())
     }
 }

tokio/tests/support/panic.rs

@@ -1,9 +1,8 @@
-use parking_lot::{const_mutex, Mutex};
 use std::panic;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 pub fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<String> {
-    static PANIC_MUTEX: Mutex<()> = const_mutex(());
+    static PANIC_MUTEX: Mutex<()> = Mutex::new(());
 
     {
         let _guard = PANIC_MUTEX.lock();
@@ -16,6 +15,7 @@ pub fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<Stri
                 let panic_location = panic_info.location().unwrap();
                 panic_file
                     .lock()
+                    .unwrap()
                     .clone_from(&Some(panic_location.file().to_string()));
             }));
         }
@@ -26,7 +26,7 @@ pub fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<Stri
         panic::set_hook(prev_hook);
 
         if result.is_err() {
-            panic_file.lock().clone()
+            panic_file.lock().unwrap().clone()
         } else {
             None
         }

tokio/tests/sync_once_cell.rs

@@ -4,178 +4,7 @@
 use std::mem;
 use std::ops::Drop;
 use std::sync::atomic::{AtomicU32, Ordering};
-use std::time::Duration;
-use tokio::runtime;
-use tokio::sync::{OnceCell, SetError};
-use tokio::time;
-
-async fn func1() -> u32 {
-    5
-}
-
-async fn func2() -> u32 {
-    time::sleep(Duration::from_millis(1)).await;
-    10
-}
-
-async fn func_err() -> Result<u32, ()> {
-    Err(())
-}
-
-async fn func_ok() -> Result<u32, ()> {
-    Ok(10)
-}
-
-async fn func_panic() -> u32 {
-    time::sleep(Duration::from_millis(1)).await;
-    panic!();
-}
-
-async fn sleep_and_set() -> u32 {
-    // Simulate sleep by pausing time and waiting for another thread to
-    // resume clock when calling `set`, then finding the cell being initialized
-    // by this call
-    time::sleep(Duration::from_millis(2)).await;
-    5
-}
-
-async fn advance_time_and_set(cell: &'static OnceCell<u32>, v: u32) -> Result<(), SetError<u32>> {
-    time::advance(Duration::from_millis(1)).await;
-    cell.set(v)
-}
-
-#[test]
-fn get_or_init() {
-    let rt = runtime::Builder::new_current_thread()
-        .enable_time()
-        .start_paused(true)
-        .build()
-        .unwrap();
-
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-
-    rt.block_on(async {
-        let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
-        let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await });
-
-        time::advance(Duration::from_millis(1)).await;
-        time::resume();
-
-        let result1 = handle1.await.unwrap();
-        let result2 = handle2.await.unwrap();
-
-        assert_eq!(*result1, 5);
-        assert_eq!(*result2, 5);
-    });
-}
-
-#[test]
-fn get_or_init_panic() {
-    let rt = runtime::Builder::new_current_thread()
-        .enable_time()
-        .build()
-        .unwrap();
-
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-
-    rt.block_on(async {
-        time::pause();
-
-        let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
-        let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await });
-
-        time::advance(Duration::from_millis(1)).await;
-
-        let result1 = handle1.await.unwrap();
-        let result2 = handle2.await.unwrap();
-
-        assert_eq!(*result1, 5);
-        assert_eq!(*result2, 5);
-    });
-}
-
-#[test]
-fn set_and_get() {
-    let rt = runtime::Builder::new_current_thread()
-        .enable_time()
-        .build()
-        .unwrap();
-
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-
-    rt.block_on(async {
-        let _ = rt.spawn(async { ONCE.set(5) }).await;
-        let value = ONCE.get().unwrap();
-        assert_eq!(*value, 5);
-    });
-}
-
-#[test]
-fn get_uninit() {
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-    let uninit = ONCE.get();
-    assert!(uninit.is_none());
-}
-
-#[test]
-fn set_twice() {
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-
-    let first = ONCE.set(5);
-    assert_eq!(first, Ok(()));
-    let second = ONCE.set(6);
-    assert!(second.err().unwrap().is_already_init_err());
-}
-
-#[test]
-fn set_while_initializing() {
-    let rt = runtime::Builder::new_current_thread()
-        .enable_time()
-        .build()
-        .unwrap();
-
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-
-    rt.block_on(async {
-        time::pause();
-
-        let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await });
-        let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await });
-
-        time::advance(Duration::from_millis(2)).await;
-
-        let result1 = handle1.await.unwrap();
-        let result2 = handle2.await.unwrap();
-
-        assert_eq!(*result1, 5);
-        assert!(result2.err().unwrap().is_initializing_err());
-    });
-}
-
-#[test]
-fn get_or_try_init() {
-    let rt = runtime::Builder::new_current_thread()
-        .enable_time()
-        .start_paused(true)
-        .build()
-        .unwrap();
-
-    static ONCE: OnceCell<u32> = OnceCell::const_new();
-
-    rt.block_on(async {
-        let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await });
-        let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await });
-
-        time::advance(Duration::from_millis(1)).await;
-        time::resume();
-
-        let result1 = handle1.await.unwrap();
-        assert!(result1.is_err());
-
-        let result2 = handle2.await.unwrap();
-        assert_eq!(*result2.unwrap(), 10);
-    });
-}
+use tokio::sync::OnceCell;
 
 #[test]
 fn drop_cell() {
@@ -272,3 +101,185 @@ fn from() {
     let cell = OnceCell::from(2);
     assert_eq!(*cell.get().unwrap(), 2);
 }
+
+#[cfg(feature = "parking_lot")]
+mod parking_lot {
+    use super::*;
+
+    use tokio::runtime;
+    use tokio::sync::SetError;
+    use tokio::time;
+
+    use std::time::Duration;
+
+    async fn func1() -> u32 {
+        5
+    }
+
+    async fn func2() -> u32 {
+        time::sleep(Duration::from_millis(1)).await;
+        10
+    }
+
+    async fn func_err() -> Result<u32, ()> {
+        Err(())
+    }
+
+    async fn func_ok() -> Result<u32, ()> {
+        Ok(10)
+    }
+
+    async fn func_panic() -> u32 {
+        time::sleep(Duration::from_millis(1)).await;
+        panic!();
+    }
+
+    async fn sleep_and_set() -> u32 {
+        // Simulate sleep by pausing time and waiting for another thread to
+        // resume clock when calling `set`, then finding the cell being initialized
+        // by this call
+        time::sleep(Duration::from_millis(2)).await;
+        5
+    }
+
+    async fn advance_time_and_set(
+        cell: &'static OnceCell<u32>,
+        v: u32,
+    ) -> Result<(), SetError<u32>> {
+        time::advance(Duration::from_millis(1)).await;
+        cell.set(v)
+    }
+
+    #[test]
+    fn get_or_init() {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_time()
+            .start_paused(true)
+            .build()
+            .unwrap();
+
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+        rt.block_on(async {
+            let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
+            let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await });
+
+            time::advance(Duration::from_millis(1)).await;
+            time::resume();
+
+            let result1 = handle1.await.unwrap();
+            let result2 = handle2.await.unwrap();
+
+            assert_eq!(*result1, 5);
+            assert_eq!(*result2, 5);
+        });
+    }
+
+    #[test]
+    fn get_or_init_panic() {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .unwrap();
+
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+        rt.block_on(async {
+            time::pause();
+
+            let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
+            let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await });
+
+            time::advance(Duration::from_millis(1)).await;
+
+            let result1 = handle1.await.unwrap();
+            let result2 = handle2.await.unwrap();
+
+            assert_eq!(*result1, 5);
+            assert_eq!(*result2, 5);
+        });
+    }
+
+    #[test]
+    fn set_and_get() {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .unwrap();
+
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+        rt.block_on(async {
+            let _ = rt.spawn(async { ONCE.set(5) }).await;
+            let value = ONCE.get().unwrap();
+            assert_eq!(*value, 5);
+        });
+    }
+
+    #[test]
+    fn get_uninit() {
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+        let uninit = ONCE.get();
+        assert!(uninit.is_none());
+    }
+
+    #[test]
+    fn set_twice() {
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+        let first = ONCE.set(5);
+        assert_eq!(first, Ok(()));
+        let second = ONCE.set(6);
+        assert!(second.err().unwrap().is_already_init_err());
+    }
+
+    #[test]
+    fn set_while_initializing() {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .unwrap();
+
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+        rt.block_on(async {
+            time::pause();
+
+            let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await });
+            let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await });
+
+            time::advance(Duration::from_millis(2)).await;
+
+            let result1 = handle1.await.unwrap();
+            let result2 = handle2.await.unwrap();
+
+            assert_eq!(*result1, 5);
+            assert!(result2.err().unwrap().is_initializing_err());
+        });
+    }
+
+    #[test]
+    fn get_or_try_init() {
+        let rt = runtime::Builder::new_current_thread()
+            .enable_time()
+            .start_paused(true)
+            .build()
+            .unwrap();
+
+        static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+        rt.block_on(async {
+            let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await });
+            let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await });
+
+            time::advance(Duration::from_millis(1)).await;
+            time::resume();
+
+            let result1 = handle1.await.unwrap();
+            assert!(result1.is_err());
+
+            let result2 = handle2.await.unwrap();
+            assert_eq!(*result2.unwrap(), 10);
+        });
+    }
+}

tokio/tests/task_blocking.rs

@@ -1,7 +1,7 @@
 #![warn(rust_2018_idioms)]
 #![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
 
-use tokio::{runtime, task};
+use tokio::{runtime, task, time};
 use tokio_test::assert_ok;
 
 use std::thread;
@@ -226,3 +226,84 @@ fn coop_disabled_in_block_in_place_in_block_on() {
 
     done_rx.recv().unwrap().unwrap();
 }
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn blocking_when_paused() {
+    // Do not auto-advance time when we have started a blocking task that has
+    // not yet finished.
+    time::timeout(
+        Duration::from_secs(3),
+        task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))),
+    )
+    .await
+    .expect("timeout should not trigger")
+    .expect("blocking task should finish");
+
+    // Really: Do not auto-advance time, even if the timeout is short and the
+    // blocking task runs for longer than that. It doesn't matter: Tokio time
+    // is paused; system time is not.
+    time::timeout(
+        Duration::from_millis(1),
+        task::spawn_blocking(|| thread::sleep(Duration::from_millis(50))),
+    )
+    .await
+    .expect("timeout should not trigger")
+    .expect("blocking task should finish");
+}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn blocking_task_wakes_paused_runtime() {
+    let t0 = std::time::Instant::now();
+    time::timeout(
+        Duration::from_secs(15),
+        task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))),
+    )
+    .await
+    .expect("timeout should not trigger")
+    .expect("blocking task should finish");
+    assert!(
+        t0.elapsed() < Duration::from_secs(10),
+        "completing a spawn_blocking should wake the scheduler if it's parked while time is paused"
+    );
+}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn unawaited_blocking_task_wakes_paused_runtime() {
+    let t0 = std::time::Instant::now();
+
+    // When this task finishes, time should auto-advance, even though the
+    // JoinHandle has not been awaited yet.
+    let a = task::spawn_blocking(|| {
+        thread::sleep(Duration::from_millis(1));
+    });
+
+    crate::time::sleep(Duration::from_secs(15)).await;
+    a.await.expect("blocking task should finish");
+    assert!(
+        t0.elapsed() < Duration::from_secs(10),
+        "completing a spawn_blocking should wake the scheduler if it's parked while time is paused"
+    );
+}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn panicking_blocking_task_wakes_paused_runtime() {
+    let t0 = std::time::Instant::now();
+    let result = time::timeout(
+        Duration::from_secs(15),
+        task::spawn_blocking(|| {
+            thread::sleep(Duration::from_millis(1));
+            panic!("blocking task panicked");
+        }),
+    )
+    .await
+    .expect("timeout should not trigger");
+    assert!(result.is_err(), "blocking task should have panicked");
+    assert!(
+        t0.elapsed() < Duration::from_secs(10),
+        "completing a spawn_blocking should wake the scheduler if it's parked while time is paused"
+    );
+}

tokio/tests/task_join_set.rs

@@ -5,8 +5,6 @@ use tokio::sync::oneshot;
 use tokio::task::JoinSet;
 use tokio::time::Duration;
 
-use futures::future::FutureExt;
-
 fn rt() -> tokio::runtime::Runtime {
     tokio::runtime::Builder::new_current_thread()
         .build()
@@ -156,49 +154,6 @@ fn runtime_gone() {
         .is_cancelled());
 }
 
-// This ensures that `join_next` works correctly when the coop budget is
-// exhausted.
-#[tokio::test(flavor = "current_thread")]
-async fn join_set_coop() {
-    // Large enough to trigger coop.
-    const TASK_NUM: u32 = 1000;
-
-    static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0);
-
-    let mut set = JoinSet::new();
-
-    for _ in 0..TASK_NUM {
-        set.spawn(async {
-            SEM.add_permits(1);
-        });
-    }
-
-    // Wait for all tasks to complete.
-    //
-    // Since this is a `current_thread` runtime, there's no race condition
-    // between the last permit being added and the task completing.
-    let _ = SEM.acquire_many(TASK_NUM).await.unwrap();
-
-    let mut count = 0;
-    let mut coop_count = 0;
-    loop {
-        match set.join_next().now_or_never() {
-            Some(Some(Ok(()))) => {}
-            Some(Some(Err(err))) => panic!("failed: {}", err),
-            None => {
-                coop_count += 1;
-                tokio::task::yield_now().await;
-                continue;
-            }
-            Some(None) => break,
-        }
-
-        count += 1;
-    }
-    assert!(coop_count >= 1);
-    assert_eq!(count, TASK_NUM);
-}
-
 #[tokio::test(start_paused = true)]
 async fn abort_all() {
     let mut set: JoinSet<()> = JoinSet::new();
@@ -228,3 +183,53 @@ async fn abort_all() {
     assert_eq!(count, 10);
     assert_eq!(set.len(), 0);
 }
+
+#[cfg(feature = "parking_lot")]
+mod parking_lot {
+    use super::*;
+
+    use futures::future::FutureExt;
+
+    // This ensures that `join_next` works correctly when the coop budget is
+    // exhausted.
+    #[tokio::test(flavor = "current_thread")]
+    async fn join_set_coop() {
+        // Large enough to trigger coop.
+        const TASK_NUM: u32 = 1000;
+
+        static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0);
+
+        let mut set = JoinSet::new();
+
+        for _ in 0..TASK_NUM {
+            set.spawn(async {
+                SEM.add_permits(1);
+            });
+        }
+
+        // Wait for all tasks to complete.
+        //
+        // Since this is a `current_thread` runtime, there's no race condition
+        // between the last permit being added and the task completing.
+        let _ = SEM.acquire_many(TASK_NUM).await.unwrap();
+
+        let mut count = 0;
+        let mut coop_count = 0;
+        loop {
+            match set.join_next().now_or_never() {
+                Some(Some(Ok(()))) => {}
+                Some(Some(Err(err))) => panic!("failed: {}", err),
+                None => {
+                    coop_count += 1;
+                    tokio::task::yield_now().await;
+                    continue;
+                }
+                Some(None) => break,
+            }
+
+            count += 1;
+        }
+        assert!(coop_count >= 1);
+        assert_eq!(count, TASK_NUM);
+    }
+}

tokio/tests/tcp_peek.rs

@@ -15,7 +15,7 @@ async fn peek() {
     let addr = listener.local_addr().unwrap();
     let t = thread::spawn(move || assert_ok!(listener.accept()).0);
 
-    let left = net::TcpStream::connect(&addr).unwrap();
+    let left = net::TcpStream::connect(addr).unwrap();
     let mut right = t.join().unwrap();
     let _ = right.write(&[1, 2, 3, 4]).unwrap();