debugging

This patch simplifies the implementation of `task::Id` by moving
conditional compilation into the `AtomicU64` definition. To handle
platforms that do not include `const fn Mutex::new()`, `StaticAtomicU64`
is defined as always having a `const fn new()`. `StaticAtomicU64` is
implemented with `OnceCell` when needed.

.cirrus.yml

@@ -1,5 +1,5 @@
 freebsd_instance:
-  image: freebsd-12-3-release-amd64
+  image: freebsd-12-4-release-amd64
 env:
   RUST_STABLE: stable
   RUST_NIGHTLY: nightly-2022-10-25

.github/workflows/ci.yml

@@ -9,10 +9,11 @@ name: CI
 env:
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: 1
+  ACTIONS_STEP_DEBUG: true
   # Change to specific Rust release to pin
   rust_stable: stable
   rust_nightly: nightly-2022-11-03
-  rust_clippy: 1.60.0
+  rust_clippy: 1.65.0
   # When updating this, also update:
   # - README.md
   # - tokio/README.md
@@ -42,7 +43,9 @@ jobs:
       - test-unstable
       - miri
       - asan
-      - cross
+      - cross-check
+      - cross-test
+      - no-atomic-u64
       - features
       - minrust
       - minimal-versions
@@ -234,13 +237,12 @@ jobs:
           # Ignore `trybuild` errors as they are irrelevant and flaky on nightly
           TRYBUILD: overwrite
 
-  cross:
-    name: cross
+  cross-check:
+    name: cross-check
     runs-on: ubuntu-latest
     strategy:
       matrix:
         target:
-          - i686-unknown-linux-gnu
           - powerpc-unknown-linux-gnu
           - powerpc64-unknown-linux-gnu
           - mips-unknown-linux-gnu
@@ -259,13 +261,70 @@ jobs:
           use-cross: true
           command: check
           args: --workspace --all-features --target ${{ matrix.target }}
+        env:
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+
+  cross-test:
+    name: cross-test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: i686-unknown-linux-gnu
+          - target: arm-unknown-linux-gnueabihf
+          - target: armv7-unknown-linux-gnueabihf
+          - target: aarch64-unknown-linux-gnu
+
+          # Run a platform without AtomicU64 and no const Mutex::new
+          - target: arm-unknown-linux-gnueabihf
+            rustflags: --cfg tokio_no_const_mutex_new
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_stable }}
+          target: ${{ matrix.target }}
+          override: true
+      # First run with all features (including parking_lot)
       - uses: actions-rs/cargo@v1
         with:
           use-cross: true
-          command: check
-          args: --workspace --all-features --target ${{ matrix.target }}
+          command: test
+          args: -p tokio --all-features --target ${{ matrix.target }} --tests
         env:
-          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 ${{ matrix.rustflags }}
+      # Now run without parking_lot
+      - name: Remove `parking_lot` from `full` feature
+        run: sed -i '0,/parking_lot/{/parking_lot/d;}' tokio/Cargo.toml
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: test
+          # The `tokio_no_parking_lot` cfg is here to ensure the `sed` above does not silently break.
+          args: -p tokio --features full,test-util --target ${{ matrix.target }} --tests
+        env:
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_ipv6 --cfg tokio_no_parking_lot ${{ matrix.rustflags }}
+
+  # See https://github.com/tokio-rs/tokio/issues/5187
+  no-atomic-u64:
+    name: Test i686-unknown-linux-gnu without AtomicU64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_nightly }}
+          components: rust-src
+          override: true
+      # Install linker and libraries for i686-unknown-linux-gnu
+      - uses: taiki-e/setup-cross-toolchain-action@v1
+        with:
+          target: i686-unknown-linux-gnu
+      - run: cargo test -Zbuild-std --target target-specs/i686-unknown-linux-gnu.json -p tokio --all-features
+        env:
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings --cfg tokio_no_atomic_u64
 
   features:
     name: features

.github/workflows/loom.yml

@@ -45,4 +45,5 @@ jobs:
         env:
           RUSTFLAGS: --cfg loom --cfg tokio_unstable -Dwarnings
           LOOM_MAX_PREEMPTIONS: 2
+          LOOM_MAX_BRANCHES: 10000
           SCOPE: ${{ matrix.scope }}

Cross.toml

@@ -1,4 +1,5 @@
 [build.env]
 passthrough = [
     "RUSTFLAGS",
+    "RUST_BACKTRACE",
 ]

README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.22.0", features = ["full"] }
+tokio = { version = "1.23.0", features = ["full"] }
 ```
 Then, on your main.rs:
 

examples/Cargo.toml

@@ -24,8 +24,8 @@ httpdate = "1.0"
 once_cell = "1.5.2"
 rand = "0.8.3"
 
-[target.'cfg(windows)'.dev-dependencies.winapi]
-version = "0.3.8"
+[target.'cfg(windows)'.dev-dependencies.windows-sys]
+version = "0.42.0"
 
 [[example]]
 name = "chat"

examples/hello_world.rs

@@ -1,9 +1,7 @@
-//! Hello world server.
-//!
 //! A simple client that opens a TCP stream, writes "hello world\n", and closes
 //! the connection.
 //!
-//! You can test this out by running:
+//! To start a server that this client can talk to on port 6142, you can use this command:
 //!
 //!     ncat -l 6142
 //!

examples/named-pipe-multi-client.rs

@@ -6,7 +6,7 @@ async fn windows_main() -> io::Result<()> {
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
     use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions};
     use tokio::time;
-    use winapi::shared::winerror;
+    use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
 
     const PIPE_NAME: &str = r"\\.\pipe\named-pipe-multi-client";
     const N: usize = 10;
@@ -59,7 +59,7 @@ async fn windows_main() -> io::Result<()> {
             let mut client = loop {
                 match ClientOptions::new().open(PIPE_NAME) {
                     Ok(client) => break client,
-                    Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+                    Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
                     Err(e) => return Err(e),
                 }
 

target-specs/i686-unknown-linux-gnu.json

@@ -0,0 +1,40 @@
+{
+  "arch": "x86",
+  "cpu": "pentium4",
+  "crt-static-respected": true,
+  "data-layout": "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128",
+  "dynamic-linking": true,
+  "env": "gnu",
+  "has-rpath": true,
+  "has-thread-local": true,
+  "llvm-target": "i686-unknown-linux-gnu",
+  "max-atomic-width": 32,
+  "os": "linux",
+  "position-independent-executables": true,
+  "pre-link-args": {
+    "gcc": [
+      "-m32"
+    ]
+  },
+  "relro-level": "full",
+  "stack-probes": {
+    "kind": "inline-or-call",
+    "min-llvm-version-for-inline": [
+      16,
+      0,
+      0
+    ]
+  },
+  "supported-sanitizers": [
+    "address"
+  ],
+  "supported-split-debuginfo": [
+    "packed",
+    "unpacked",
+    "off"
+  ],
+  "target-family": [
+    "unix"
+  ],
+  "target-pointer-width": "32"
+}

tests-integration/Cargo.toml

@@ -58,3 +58,4 @@ tokio = { path = "../tokio" }
 tokio-test = { path = "../tokio-test", optional = true }
 doc-comment = "0.3.1"
 futures = { version = "0.3.0", features = ["async-await"] }
+bytes = "1.0.0"

tests-integration/tests/process_stdio.rs

@@ -190,3 +190,54 @@ async fn pipe_from_one_command_to_another() {
     assert!(second_status.expect("second status").success());
     assert!(third_status.expect("third status").success());
 }
+
+#[tokio::test]
+async fn vectored_writes() {
+    use bytes::{Buf, Bytes};
+    use std::{io::IoSlice, pin::Pin};
+    use tokio::io::AsyncWrite;
+
+    let mut cat = cat().spawn().unwrap();
+    let mut stdin = cat.stdin.take().unwrap();
+    let are_writes_vectored = stdin.is_write_vectored();
+    let mut stdout = cat.stdout.take().unwrap();
+
+    let write = async {
+        let mut input = Bytes::from_static(b"hello\n").chain(Bytes::from_static(b"world!\n"));
+        let mut writes_completed = 0;
+
+        futures::future::poll_fn(|cx| loop {
+            let mut slices = [IoSlice::new(&[]); 2];
+            let vectored = input.chunks_vectored(&mut slices);
+            if vectored == 0 {
+                return std::task::Poll::Ready(std::io::Result::Ok(()));
+            }
+            let n = futures::ready!(Pin::new(&mut stdin).poll_write_vectored(cx, &slices))?;
+            writes_completed += 1;
+            input.advance(n);
+        })
+        .await?;
+
+        drop(stdin);
+
+        std::io::Result::Ok(writes_completed)
+    };
+
+    let read = async {
+        let mut buffer = Vec::with_capacity(6 + 7);
+        stdout.read_to_end(&mut buffer).await?;
+        std::io::Result::Ok(buffer)
+    };
+
+    let (write, read, status) = future::join3(write, read, cat.wait()).await;
+
+    assert!(status.unwrap().success());
+
+    let writes_completed = write.unwrap();
+    // on unix our small payload should always fit in whatever default sized pipe with a single
+    // syscall. if multiple are used, then the forwarding does not work, or we are on a platform
+    // for which the `std` does not support vectored writes.
+    assert_eq!(writes_completed == 1, are_writes_vectored);
+
+    assert_eq!(&read.unwrap(), b"hello\nworld!\n");
+}

tokio-macros/CHANGELOG.md

@@ -1,3 +1,21 @@
+# 1.8.2 (November 30th, 2022)
+
+- fix a regression introduced in 1.8.1 ([#5244])
+
+[#5244]: https://github.com/tokio-rs/tokio/pull/5244
+
+# 1.8.1 (November 29th, 2022)
+
+(yanked)
+
+- macros: Pin Futures in `#[tokio::test]` to stack ([#5205])
+- macros: Reduce usage of last statement spans in proc-macros ([#5092])
+- macros: Improve the documentation for `#[tokio::test]` ([#4761])
+
+[#5205]: https://github.com/tokio-rs/tokio/pull/5205
+[#5092]: https://github.com/tokio-rs/tokio/pull/5092
+[#4761]: https://github.com/tokio-rs/tokio/pull/4761
+
 # 1.8.0 (June 4th, 2022)
 
 - macros: always emit return statement ([#4636])

tokio-macros/Cargo.toml

@@ -3,8 +3,8 @@ name = "tokio-macros"
 # When releasing to crates.io:
 # - Remove path dependencies
 # - Update CHANGELOG.md.
-# - Create "tokio-macros-1.0.x" git tag.
-version = "1.8.0"
+# - Create "tokio-macros-1.x.y" git tag.
+version = "1.8.2"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]

tokio-macros/src/entry.rs

@@ -383,6 +383,7 @@ fn parse_knobs(mut input: syn::ItemFn, is_test: bool, config: FinalConfig) -> To
 
     let body = &input.block;
     let brace_token = input.block.brace_token;
+    let body_ident = quote! { body };
     let block_expr = quote_spanned! {last_stmt_end_span=>
         #[allow(clippy::expect_used, clippy::diverging_sub_expression)]
         {
@@ -390,12 +391,41 @@ fn parse_knobs(mut input: syn::ItemFn, is_test: bool, config: FinalConfig) -> To
                 .enable_all()
                 .build()
                 .expect("Failed building the Runtime")
-                .block_on(body);
+                .block_on(#body_ident);
         }
     };
+
+    // For test functions pin the body to the stack and use `Pin<&mut dyn
+    // Future>` to reduce the amount of `Runtime::block_on` (and related
+    // functions) copies we generate during compilation due to the generic
+    // parameter `F` (the future to block on). This could have an impact on
+    // performance, but because it's only for testing it's unlikely to be very
+    // large.
+    //
+    // We don't do this for the main function as it should only be used once so
+    // there will be no benefit.
+    let body = if is_test {
+        let output_type = match &input.sig.output {
+            // For functions with no return value syn doesn't print anything,
+            // but that doesn't work as `Output` for our boxed `Future`, so
+            // default to `()` (the same type as the function output).
+            syn::ReturnType::Default => quote! { () },
+            syn::ReturnType::Type(_, ret_type) => quote! { #ret_type },
+        };
+        quote! {
+            let body = async #body;
+            #crate_ident::pin!(body);
+            let body: ::std::pin::Pin<&mut dyn ::std::future::Future<Output = #output_type>> = body;
+        }
+    } else {
+        quote! {
+            let body = async #body;
+        }
+    };
+
     input.block = syn::parse2(quote! {
         {
-            let body = async #body;
+            #body
             #block_expr
         }
     })
@@ -450,7 +480,7 @@ pub(crate) fn test(args: TokenStream, item: TokenStream, rt_multi_thread: bool)
     };
     let config = if let Some(attr) = input.attrs.iter().find(|attr| attr.path.is_ident("test")) {
         let msg = "second test attribute is supplied";
-        Err(syn::Error::new_spanned(&attr, msg))
+        Err(syn::Error::new_spanned(attr, msg))
     } else {
         AttributeArgs::parse_terminated
             .parse(args)

tokio-macros/src/select.rs

@@ -100,10 +100,10 @@ fn clean_pattern(pat: &mut syn::Pat) {
         }
         syn::Pat::Reference(reference) => {
             reference.mutability = None;
-            clean_pattern(&mut *reference.pat);
+            clean_pattern(&mut reference.pat);
         }
         syn::Pat::Type(type_pat) => {
-            clean_pattern(&mut *type_pat.pat);
+            clean_pattern(&mut type_pat.pat);
         }
         _ => {}
     }

tokio-stream/src/stream_ext/then.rs

@@ -72,7 +72,7 @@ where
     }
 
     fn size_hint(&self) -> (usize, Option<usize>) {
-        let future_len = if self.future.is_some() { 1 } else { 0 };
+        let future_len = usize::from(self.future.is_some());
         let (lower, upper) = self.stream.size_hint();
 
         let lower = lower.saturating_add(future_len);

tokio-util/src/codec/framed.rs

@@ -368,10 +368,7 @@ pub struct FramedParts<T, U> {
 
 impl<T, U> FramedParts<T, U> {
     /// Create a new, default, `FramedParts`
-    pub fn new<I>(io: T, codec: U) -> FramedParts<T, U>
-    where
-        U: Encoder<I>,
-    {
+    pub fn new(io: T, codec: U) -> FramedParts<T, U> {
         FramedParts {
             io,
             codec,

tokio-util/src/io/sync_bridge.rs

@@ -1,5 +1,7 @@
-use std::io::{Read, Write};
-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use std::io::{BufRead, Read, Write};
+use tokio::io::{
+    AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt,
+};
 
 /// Use a [`tokio::io::AsyncRead`] synchronously as a [`std::io::Read`] or
 /// a [`tokio::io::AsyncWrite`] as a [`std::io::Write`].
@@ -9,6 +11,28 @@ pub struct SyncIoBridge<T> {
     rt: tokio::runtime::Handle,
 }
 
+impl<T: AsyncBufRead + Unpin> BufRead for SyncIoBridge<T> {
+    fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
+        let src = &mut self.src;
+        self.rt.block_on(AsyncBufReadExt::fill_buf(src))
+    }
+
+    fn consume(&mut self, amt: usize) {
+        let src = &mut self.src;
+        AsyncBufReadExt::consume(src, amt)
+    }
+
+    fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> std::io::Result<usize> {
+        let src = &mut self.src;
+        self.rt
+            .block_on(AsyncBufReadExt::read_until(src, byte, buf))
+    }
+    fn read_line(&mut self, buf: &mut String) -> std::io::Result<usize> {
+        let src = &mut self.src;
+        self.rt.block_on(AsyncBufReadExt::read_line(src, buf))
+    }
+}
+
 impl<T: AsyncRead + Unpin> Read for SyncIoBridge<T> {
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
         let src = &mut self.src;

tokio-util/src/sync/cancellation_token.rs

@@ -66,6 +66,23 @@ pin_project! {
     }
 }
 
+pin_project! {
+    /// A Future that is resolved once the corresponding [`CancellationToken`]
+    /// is cancelled.
+    ///
+    /// This is the counterpart to [`WaitForCancellationFuture`] that takes
+    /// [`CancellationToken`] by value instead of using a reference.
+    #[must_use = "futures do nothing unless polled"]
+    pub struct WaitForCancellationFutureOwned {
+        // Since `future` is the first field, it is dropped before the
+        // cancellation_token field. This ensures that the reference inside the
+        // `Notified` remains valid.
+        #[pin]
+        future: tokio::sync::futures::Notified<'static>,
+        cancellation_token: CancellationToken,
+    }
+}
+
 // ===== impl CancellationToken =====
 
 impl core::fmt::Debug for CancellationToken {
@@ -183,6 +200,21 @@ impl CancellationToken {
         }
     }
 
+    /// Returns a `Future` that gets fulfilled when cancellation is requested.
+    ///
+    /// The future will complete immediately if the token is already cancelled
+    /// when this method is called.
+    ///
+    /// The function takes self by value and returns a future that owns the
+    /// token.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe.
+    pub fn cancelled_owned(self) -> WaitForCancellationFutureOwned {
+        WaitForCancellationFutureOwned::new(self)
+    }
+
     /// Creates a `DropGuard` for this token.
     ///
     /// Returned guard will cancel this token (and all its children) on drop
@@ -222,3 +254,68 @@ impl<'a> Future for WaitForCancellationFuture<'a> {
         }
     }
 }
+
+// ===== impl WaitForCancellationFutureOwned =====
+
+impl core::fmt::Debug for WaitForCancellationFutureOwned {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.debug_struct("WaitForCancellationFutureOwned").finish()
+    }
+}
+
+impl WaitForCancellationFutureOwned {
+    fn new(cancellation_token: CancellationToken) -> Self {
+        WaitForCancellationFutureOwned {
+            // cancellation_token holds a heap allocation and is guaranteed to have a
+            // stable deref, thus it would be ok to move the cancellation_token while
+            // the future holds a reference to it.
+            //
+            // # Safety
+            //
+            // cancellation_token is dropped after future due to the field ordering.
+            future: unsafe { Self::new_future(&cancellation_token) },
+            cancellation_token,
+        }
+    }
+
+    /// # Safety
+    /// The returned future must be destroyed before the cancellation token is
+    /// destroyed.
+    unsafe fn new_future(
+        cancellation_token: &CancellationToken,
+    ) -> tokio::sync::futures::Notified<'static> {
+        let inner_ptr = Arc::as_ptr(&cancellation_token.inner);
+        // SAFETY: The `Arc::as_ptr` method guarantees that `inner_ptr` remains
+        // valid until the strong count of the Arc drops to zero, and the caller
+        // guarantees that they will drop the future before that happens.
+        (*inner_ptr).notified()
+    }
+}
+
+impl Future for WaitForCancellationFutureOwned {
+    type Output = ();
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
+        let mut this = self.project();
+
+        loop {
+            if this.cancellation_token.is_cancelled() {
+                return Poll::Ready(());
+            }
+
+            // No wakeups can be lost here because there is always a call to
+            // `is_cancelled` between the creation of the future and the call to
+            // `poll`, and the code that sets the cancelled flag does so before
+            // waking the `Notified`.
+            if this.future.as_mut().poll(cx).is_pending() {
+                return Poll::Pending;
+            }
+
+            // # Safety
+            //
+            // cancellation_token is dropped after future due to the field ordering.
+            this.future
+                .set(unsafe { Self::new_future(this.cancellation_token) });
+        }
+    }
+}

tokio-util/src/sync/mod.rs

@@ -1,7 +1,9 @@
 //! Synchronization primitives
 
 mod cancellation_token;
-pub use cancellation_token::{guard::DropGuard, CancellationToken, WaitForCancellationFuture};
+pub use cancellation_token::{
+    guard::DropGuard, CancellationToken, WaitForCancellationFuture, WaitForCancellationFutureOwned,
+};
 
 mod mpsc;
 pub use mpsc::{PollSendError, PollSender};

tokio-util/src/sync/poll_semaphore.rs

@@ -166,6 +166,6 @@ impl fmt::Debug for PollSemaphore {
 
 impl AsRef<Semaphore> for PollSemaphore {
     fn as_ref(&self) -> &Semaphore {
-        &*self.semaphore
+        &self.semaphore
     }
 }

tokio-util/src/sync/tests/loom_cancellation_token.rs

@@ -24,6 +24,27 @@ fn cancel_token() {
     });
 }
 
+#[test]
+fn cancel_token_owned() {
+    loom::model(|| {
+        let token = CancellationToken::new();
+        let token1 = token.clone();
+
+        let th1 = thread::spawn(move || {
+            block_on(async {
+                token1.cancelled_owned().await;
+            });
+        });
+
+        let th2 = thread::spawn(move || {
+            token.cancel();
+        });
+
+        assert_ok!(th1.join());
+        assert_ok!(th2.join());
+    });
+}
+
 #[test]
 fn cancel_with_child() {
     loom::model(|| {

tokio-util/tests/spawn_pinned.rs

@@ -82,8 +82,8 @@ async fn task_panic_propagates() {
     assert!(result.is_err());
     let error = result.unwrap_err();
     assert!(error.is_panic());
-    let panic_str: &str = *error.into_panic().downcast().unwrap();
-    assert_eq!(panic_str, "Test panic");
+    let panic_str = error.into_panic().downcast::<&'static str>().unwrap();
+    assert_eq!(*panic_str, "Test panic");
 
     // Trying again with a "safe" task still works
     let join_handle = pool.spawn_pinned(|| async { "test" });
@@ -108,8 +108,8 @@ async fn callback_panic_does_not_kill_worker() {
     assert!(result.is_err());
     let error = result.unwrap_err();
     assert!(error.is_panic());
-    let panic_str: &str = *error.into_panic().downcast().unwrap();
-    assert_eq!(panic_str, "Test panic");
+    let panic_str = error.into_panic().downcast::<&'static str>().unwrap();
+    assert_eq!(*panic_str, "Test panic");
 
     // Trying again with a "safe" callback works
     let join_handle = pool.spawn_pinned(|| async { "test" });

tokio-util/tests/sync_cancellation_token.rs

@@ -39,6 +39,56 @@ fn cancel_token() {
     );
 }
 
+#[test]
+fn cancel_token_owned() {
+    let (waker, wake_counter) = new_count_waker();
+    let token = CancellationToken::new();
+    assert!(!token.is_cancelled());
+
+    let wait_fut = token.clone().cancelled_owned();
+    pin!(wait_fut);
+
+    assert_eq!(
+        Poll::Pending,
+        wait_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 0);
+
+    let wait_fut_2 = token.clone().cancelled_owned();
+    pin!(wait_fut_2);
+
+    token.cancel();
+    assert_eq!(wake_counter, 1);
+    assert!(token.is_cancelled());
+
+    assert_eq!(
+        Poll::Ready(()),
+        wait_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        wait_fut_2.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+}
+
+#[test]
+fn cancel_token_owned_drop_test() {
+    let (waker, wake_counter) = new_count_waker();
+    let token = CancellationToken::new();
+
+    let future = token.cancelled_owned();
+    pin!(future);
+
+    assert_eq!(
+        Poll::Pending,
+        future.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 0);
+
+    // let future be dropped while pinned and under pending state to
+    // find potential memory related bugs.
+}
+
 #[test]
 fn cancel_child_token_through_parent() {
     let (waker, wake_counter) = new_count_waker();

tokio-util/tests/time_delay_queue.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::blacklisted_name)]
+#![allow(clippy::disallowed_names)]
 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
 

tokio/CHANGELOG.md

@@ -1,3 +1,23 @@
+# 1.23.0 (December 5, 2022)
+
+### Fixed
+
+ - net: fix Windows named pipe connect ([#5208])
+ - io: support vectored writes for `ChildStdin` ([#5216])
+ - io: fix `async fn ready()` false positive for OS-specific events ([#5231])
+
+ ### Changed
+ - runtime: `yield_now` defers task until after driver poll ([#5223])
+ - runtime: reduce amount of codegen needed per spawned task ([#5213])
+ - windows: replace `winapi` dependency with `windows-sys` ([#5204])
+
+ [#5208]: https://github.com/tokio-rs/tokio/pull/5208
+ [#5216]: https://github.com/tokio-rs/tokio/pull/5216
+ [#5213]: https://github.com/tokio-rs/tokio/pull/5213
+ [#5204]: https://github.com/tokio-rs/tokio/pull/5204
+ [#5223]: https://github.com/tokio-rs/tokio/pull/5223
+ [#5231]: https://github.com/tokio-rs/tokio/pull/5231
+
 # 1.22.0 (November 17, 2022)
 
 ### Added

tokio/Cargo.toml

@@ -5,8 +5,8 @@ name = "tokio"
 # - Update doc url
 #   - README.md
 # - Update CHANGELOG.md.
-# - Create "v1.0.x" git tag.
-version = "1.22.0"
+# - Create "v1.x.y" git tag.
+version = "1.23.0"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -52,14 +52,11 @@ net = [
   "mio/os-ext",
   "mio/net",
   "socket2",
-  "winapi/fileapi",
-  "winapi/handleapi",
-  "winapi/namedpipeapi",
-  "winapi/winbase",
-  "winapi/winnt",
-  "winapi/minwindef",
-  "winapi/accctrl",
-  "winapi/aclapi",
+  "windows-sys/Win32_Foundation",
+  "windows-sys/Win32_Security",
+  "windows-sys/Win32_Storage_FileSystem",
+  "windows-sys/Win32_System_Pipes",
+  "windows-sys/Win32_System_SystemServices",
 ]
 process = [
   "bytes",
@@ -68,12 +65,9 @@ process = [
   "mio/os-ext",
   "mio/net",
   "signal-hook-registry",
-  "winapi/handleapi",
-  "winapi/minwindef",
-  "winapi/processthreadsapi",
-  "winapi/threadpoollegacyapiset",
-  "winapi/winbase",
-  "winapi/winnt",
+  "windows-sys/Win32_Foundation",
+  "windows-sys/Win32_System_Threading",
+  "windows-sys/Win32_System_WindowsProgramming",
 ]
 # Includes basic task execution capabilities
 rt = []
@@ -87,9 +81,8 @@ signal = [
   "mio/net",
   "mio/os-ext",
   "signal-hook-registry",
-  "winapi/consoleapi",
-  "winapi/wincon",
-  "winapi/minwindef",
+  "windows-sys/Win32_Foundation",
+  "windows-sys/Win32_System_Console",
 ]
 sync = []
 test-util = ["rt", "sync", "time"]
@@ -131,12 +124,17 @@ signal-hook-registry = { version = "1.1.1", optional = true }
 libc = { version = "0.2.42" }
 nix = { version = "0.24", default-features = false, features = ["fs", "socket"] }
 
-[target.'cfg(windows)'.dependencies.winapi]
-version = "0.3.8"
-default-features = false
-features = ["std"]
+[target.'cfg(windows)'.dependencies.windows-sys]
+version = "0.42.0"
 optional = true
 
+[target.'cfg(docsrs)'.dependencies.windows-sys]
+version = "0.42.0"
+features = [
+    "Win32_Foundation",
+    "Win32_Security_Authorization",
+]
+
 [target.'cfg(windows)'.dev-dependencies.ntapi]
 version = "0.3.6"
 
@@ -159,7 +157,7 @@ rand = "0.8.0"
 wasm-bindgen-test = "0.3.0"
 
 [target.'cfg(target_os = "freebsd")'.dev-dependencies]
-mio-aio = { version = "0.6.0", features = ["tokio"] }
+mio-aio = { version = "0.7.0", features = ["tokio"] }
 
 [target.'cfg(loom)'.dev-dependencies]
 loom = { version = "0.5.2", features = ["futures", "checkpoint"] }

tokio/README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.22.0", features = ["full"] }
+tokio = { version = "1.23.0", features = ["full"] }
 ```
 Then, on your main.rs:
 

tokio/build.rs

@@ -24,10 +24,25 @@ const CONST_MUTEX_NEW_PROBE: &str = r#"
 }
 "#;
 
+const TARGET_HAS_ATOMIC_PROBE: &str = r#"
+{
+    #[cfg(target_has_atomic = "ptr")]
+    let _ = (); 
+}
+"#;
+
+const TARGET_ATOMIC_U64_PROBE: &str = r#"
+{
+    use std::sync::atomic::AtomicU64 as _;
+}
+"#;
+
 fn main() {
     let mut enable_const_thread_local = false;
     let mut enable_addr_of = false;
+    let mut enable_target_has_atomic = false;
     let mut enable_const_mutex_new = false;
+    let mut target_needs_atomic_u64_fallback = false;
 
     match AutoCfg::new() {
         Ok(ac) => {
@@ -66,6 +81,27 @@ fn main() {
                 }
             }
 
+            // The `target_has_atomic` cfg was stabilized in 1.60.
+            if ac.probe_rustc_version(1, 61) {
+                enable_target_has_atomic = true;
+            } else if ac.probe_rustc_version(1, 60) {
+                // This compiler claims to be 1.60, but there are some nightly
+                // compilers that claim to be 1.60 without supporting the
+                // feature. Explicitly probe to check if code using them
+                // compiles.
+                //
+                // The oldest nightly that supports the feature is 2022-02-11.
+                if ac.probe_expression(TARGET_HAS_ATOMIC_PROBE) {
+                    enable_target_has_atomic = true;
+                }
+            }
+
+            // If we can't tell using `target_has_atomic`, tell if the target
+            // has `AtomicU64` by trying to use it.
+            if !enable_target_has_atomic && !ac.probe_expression(TARGET_ATOMIC_U64_PROBE) {
+                target_needs_atomic_u64_fallback = true;
+            }
+
             // The `Mutex::new` method was made const in 1.63.
             if ac.probe_rustc_version(1, 64) {
                 enable_const_mutex_new = true;
@@ -109,6 +145,14 @@ fn main() {
         autocfg::emit("tokio_no_addr_of")
     }
 
+    if !enable_target_has_atomic {
+        // To disable this feature on compilers that support it, you can
+        // explicitly pass this flag with the following environment variable:
+        //
+        // RUSTFLAGS="--cfg tokio_no_target_has_atomic"
+        autocfg::emit("tokio_no_target_has_atomic")
+    }
+
     if !enable_const_mutex_new {
         // To disable this feature on compilers that support it, you can
         // explicitly pass this flag with the following environment variable:
@@ -117,6 +161,14 @@ fn main() {
         autocfg::emit("tokio_no_const_mutex_new")
     }
 
+    if target_needs_atomic_u64_fallback {
+        // To disable this feature on compilers that support it, you can
+        // explicitly pass this flag with the following environment variable:
+        //
+        // RUSTFLAGS="--cfg tokio_no_atomic_u64"
+        autocfg::emit("tokio_no_atomic_u64")
+    }
+
     let target = ::std::env::var("TARGET").unwrap_or_default();
 
     // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54.

tokio/src/doc/mod.rs

@@ -21,4 +21,3 @@
 pub enum NotDefinedHere {}
 
 pub mod os;
-pub mod winapi;

tokio/src/doc/winapi.rs

@@ -1,66 +0,0 @@
-//! See [winapi].
-//!
-//! [winapi]: https://docs.rs/winapi
-
-/// See [winapi::shared](https://docs.rs/winapi/*/winapi/shared/index.html).
-pub mod shared {
-    /// See [winapi::shared::winerror](https://docs.rs/winapi/*/winapi/shared/winerror/index.html).
-    #[allow(non_camel_case_types)]
-    pub mod winerror {
-        /// See [winapi::shared::winerror::ERROR_ACCESS_DENIED][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_ACCESS_DENIED.html
-        pub type ERROR_ACCESS_DENIED = crate::doc::NotDefinedHere;
-
-        /// See [winapi::shared::winerror::ERROR_PIPE_BUSY][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_PIPE_BUSY.html
-        pub type ERROR_PIPE_BUSY = crate::doc::NotDefinedHere;
-
-        /// See [winapi::shared::winerror::ERROR_MORE_DATA][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_MORE_DATA.html
-        pub type ERROR_MORE_DATA = crate::doc::NotDefinedHere;
-    }
-}
-
-/// See [winapi::um](https://docs.rs/winapi/*/winapi/um/index.html).
-pub mod um {
-    /// See [winapi::um::winbase](https://docs.rs/winapi/*/winapi/um/winbase/index.html).
-    #[allow(non_camel_case_types)]
-    pub mod winbase {
-        /// See [winapi::um::winbase::PIPE_TYPE_MESSAGE][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_MESSAGE.html
-        pub type PIPE_TYPE_MESSAGE = crate::doc::NotDefinedHere;
-
-        /// See [winapi::um::winbase::PIPE_TYPE_BYTE][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_BYTE.html
-        pub type PIPE_TYPE_BYTE = crate::doc::NotDefinedHere;
-
-        /// See [winapi::um::winbase::PIPE_CLIENT_END][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_CLIENT_END.html
-        pub type PIPE_CLIENT_END = crate::doc::NotDefinedHere;
-
-        /// See [winapi::um::winbase::PIPE_SERVER_END][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_SERVER_END.html
-        pub type PIPE_SERVER_END = crate::doc::NotDefinedHere;
-
-        /// See [winapi::um::winbase::SECURITY_IDENTIFICATION][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.SECURITY_IDENTIFICATION.html
-        pub type SECURITY_IDENTIFICATION = crate::doc::NotDefinedHere;
-    }
-
-    /// See [winapi::um::minwinbase](https://docs.rs/winapi/*/winapi/um/minwinbase/index.html).
-    #[allow(non_camel_case_types)]
-    pub mod minwinbase {
-        /// See [winapi::um::minwinbase::SECURITY_ATTRIBUTES][winapi]
-        ///
-        /// [winapi]: https://docs.rs/winapi/*/winapi/um/minwinbase/constant.SECURITY_ATTRIBUTES.html
-        pub type SECURITY_ATTRIBUTES = crate::doc::NotDefinedHere;
-    }
-}

tokio/src/fs/open_options.rs

@@ -542,7 +542,7 @@ feature! {
         /// # Examples
         ///
         /// ```no_run
-        /// use winapi::um::winbase::FILE_FLAG_DELETE_ON_CLOSE;
+        /// use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_DELETE_ON_CLOSE;
         /// use tokio::fs::OpenOptions;
         ///
         /// # #[tokio::main]
@@ -581,7 +581,7 @@ feature! {
         /// # Examples
         ///
         /// ```no_run
-        /// use winapi::um::winnt::FILE_ATTRIBUTE_HIDDEN;
+        /// use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN;
         /// use tokio::fs::OpenOptions;
         ///
         /// # #[tokio::main]
@@ -624,7 +624,7 @@ feature! {
         /// # Examples
         ///
         /// ```no_run
-        /// use winapi::um::winbase::SECURITY_IDENTIFICATION;
+        /// use windows_sys::Win32::Storage::FileSystem::SECURITY_IDENTIFICATION;
         /// use tokio::fs::OpenOptions;
         ///
         /// # #[tokio::main]

tokio/src/io/poll_evented.rs

@@ -208,7 +208,7 @@ feature! {
             }
         }
 
-        #[cfg(feature = "net")]
+        #[cfg(any(feature = "net", feature = "process"))]
         pub(crate) fn poll_write_vectored<'a>(
             &'a self,
             cx: &mut Context<'_>,

tokio/src/io/util/read.rs

@@ -48,7 +48,7 @@ where
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
         let me = self.project();
-        let mut buf = ReadBuf::new(*me.buf);
+        let mut buf = ReadBuf::new(me.buf);
         ready!(Pin::new(me.reader).poll_read(cx, &mut buf))?;
         Poll::Ready(Ok(buf.filled().len()))
     }

tokio/src/lib.rs

@@ -174,12 +174,15 @@
 //! swapping the currently running task on each thread. However, this kind of
 //! swapping can only happen at `.await` points, so code that spends a long time
 //! without reaching an `.await` will prevent other tasks from running. To
-//! combat this, Tokio provides two kinds of threads: Core threads and blocking
-//! threads. The core threads are where all asynchronous code runs, and Tokio
-//! will by default spawn one for each CPU core. The blocking threads are
-//! spawned on demand, can be used to run blocking code that would otherwise
-//! block other tasks from running and are kept alive when not used for a certain
-//! amount of time which can be configured with [`thread_keep_alive`].
+//! combat this, Tokio provides two kinds of threads: Core threads and blocking threads.
+//!
+//! The core threads are where all asynchronous code runs, and Tokio will by default
+//! spawn one for each CPU core. You can use the environment variable `TOKIO_WORKER_THREADS`
+//! to override the default value.
+//!
+//! The blocking threads are spawned on demand, can be used to run blocking code
+//! that would otherwise block other tasks from running and are kept alive when
+//! not used for a certain amount of time which can be configured with [`thread_keep_alive`].
 //! Since it is not possible for Tokio to swap out blocking tasks, like it
 //! can do with asynchronous code, the upper limit on the number of blocking
 //! threads is very large. These limits can be configured on the [`Builder`].
@@ -328,20 +331,15 @@
 //! - `signal`: Enables all `tokio::signal` types.
 //! - `fs`: Enables `tokio::fs` types.
 //! - `test-util`: Enables testing based infrastructure for the Tokio runtime.
+//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's
+//!                  synchronization primitives internally. Also, this
+//!                  dependency is necessary to construct some of our primitives
+//!                  in a const context. MSRV may increase according to the
+//!                  _parking_lot_ release in use.
 //!
 //! _Note: `AsyncRead` and `AsyncWrite` traits do not require any features and are
 //! always available._
 //!
-//! ### Internal features
-//!
-//! These features do not expose any new API, but influence internal
-//! implementation aspects of Tokio, and can pull in additional
-//! dependencies.
-//!
-//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's
-//! synchronization primitives internally. MSRV may increase according to the
-//! _parking_lot_ release in use.
-//!
 //! ### Unstable features
 //!
 //! Some feature flags are only available when specifying the `tokio_unstable` flag:
@@ -583,14 +581,6 @@ pub(crate) use self::doc::os;
 #[allow(unused)]
 pub(crate) use std::os;
 
-#[cfg(docsrs)]
-#[allow(unused)]
-pub(crate) use self::doc::winapi;
-
-#[cfg(all(not(docsrs), windows, feature = "net"))]
-#[allow(unused)]
-pub(crate) use winapi;
-
 cfg_macros! {
     /// Implementation detail of the `select!` macro. This macro is **not**
     /// intended to be used as part of the public API and is permitted to

tokio/src/loom/mocked.rs

@@ -25,6 +25,13 @@ pub(crate) mod sync {
         }
     }
     pub(crate) use loom::sync::*;
+
+    pub(crate) mod atomic {
+        pub(crate) use loom::sync::atomic::*;
+
+        // TODO: implement a loom version
+        pub(crate) type StaticAtomicU64 = std::sync::atomic::AtomicU64;
+    }
 }
 
 pub(crate) mod rand {

tokio/src/loom/std/atomic_u64.rs

@@ -7,80 +7,13 @@
 // `#[cfg(target_has_atomic = "64")]`.
 // Refs: https://github.com/rust-lang/rust/tree/master/src/librustc_target
 cfg_has_atomic_u64! {
-    pub(crate) use std::sync::atomic::AtomicU64;
+    #[path = "atomic_u64_native.rs"]
+    mod imp;
 }
 
 cfg_not_has_atomic_u64! {
-    use crate::loom::sync::Mutex;
-    use std::sync::atomic::Ordering;
-
-    #[derive(Debug)]
-    pub(crate) struct AtomicU64 {
-        inner: Mutex<u64>,
-    }
-
-    impl AtomicU64 {
-        pub(crate) fn new(val: u64) -> Self {
-            Self {
-                inner: Mutex::new(val),
-            }
-        }
-
-        pub(crate) fn load(&self, _: Ordering) -> u64 {
-            *self.inner.lock()
-        }
-
-        pub(crate) fn store(&self, val: u64, _: Ordering) {
-            *self.inner.lock() = val;
-        }
-
-        pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 {
-            let mut lock = self.inner.lock();
-            let prev = *lock;
-            *lock = prev + val;
-            prev
-        }
-
-        pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
-            let mut lock = self.inner.lock();
-            let prev = *lock;
-            *lock = prev | val;
-            prev
-        }
-
-        pub(crate) fn compare_exchange(
-            &self,
-            current: u64,
-            new: u64,
-            _success: Ordering,
-            _failure: Ordering,
-        ) -> Result<u64, u64> {
-            let mut lock = self.inner.lock();
-
-            if *lock == current {
-                *lock = new;
-                Ok(current)
-            } else {
-                Err(*lock)
-            }
-        }
-
-        pub(crate) fn compare_exchange_weak(
-            &self,
-            current: u64,
-            new: u64,
-            success: Ordering,
-            failure: Ordering,
-        ) -> Result<u64, u64> {
-            self.compare_exchange(current, new, success, failure)
-        }
-    }
-
-    impl Default for AtomicU64 {
-        fn default() -> AtomicU64 {
-            Self {
-                inner: Mutex::new(0),
-            }
-        }
-    }
+    #[path = "atomic_u64_as_mutex.rs"]
+    mod imp;
 }
+
+pub(crate) use imp::{AtomicU64, StaticAtomicU64};

tokio/src/loom/std/atomic_u64_as_mutex.rs

@@ -0,0 +1,76 @@
+use crate::loom::sync::Mutex;
+use std::sync::atomic::Ordering;
+
+cfg_has_const_mutex_new! {
+    #[path = "atomic_u64_static_const_new.rs"]
+    mod static_macro;
+}
+
+cfg_not_has_const_mutex_new! {
+    #[path = "atomic_u64_static_once_cell.rs"]
+    mod static_macro;
+}
+
+pub(crate) use static_macro::StaticAtomicU64;
+
+#[derive(Debug)]
+pub(crate) struct AtomicU64 {
+    inner: Mutex<u64>,
+}
+
+impl AtomicU64 {
+    pub(crate) fn load(&self, _: Ordering) -> u64 {
+        *self.inner.lock()
+    }
+
+    pub(crate) fn store(&self, val: u64, _: Ordering) {
+        *self.inner.lock() = val;
+    }
+
+    pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 {
+        let mut lock = self.inner.lock();
+        let prev = *lock;
+        *lock = prev + val;
+        prev
+    }
+
+    pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
+        let mut lock = self.inner.lock();
+        let prev = *lock;
+        *lock = prev | val;
+        prev
+    }
+
+    pub(crate) fn compare_exchange(
+        &self,
+        current: u64,
+        new: u64,
+        _success: Ordering,
+        _failure: Ordering,
+    ) -> Result<u64, u64> {
+        let mut lock = self.inner.lock();
+
+        if *lock == current {
+            *lock = new;
+            Ok(current)
+        } else {
+            Err(*lock)
+        }
+    }
+
+    pub(crate) fn compare_exchange_weak(
+        &self,
+        current: u64,
+        new: u64,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<u64, u64> {
+        self.compare_exchange(current, new, success, failure)
+    }
+}
+
+impl Default for AtomicU64 {
+    fn default() -> AtomicU64 {
+        AtomicU64::new(u64::default())
+    }
+}

tokio/src/loom/std/atomic_u64_native.rs

@@ -0,0 +1,4 @@
+pub(crate) use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Alias `AtomicU64` to `StaticAtomicU64`
+pub(crate) type StaticAtomicU64 = AtomicU64;

tokio/src/loom/std/atomic_u64_static_const_new.rs

@@ -0,0 +1,12 @@
+use super::AtomicU64;
+use crate::loom::sync::Mutex;
+
+pub(crate) type StaticAtomicU64 = AtomicU64;
+
+impl AtomicU64 {
+    pub(crate) const fn new(val: u64) -> Self {
+        Self {
+            inner: Mutex::const_new(val),
+        }
+    }
+}

tokio/src/loom/std/atomic_u64_static_once_cell.rs

@@ -0,0 +1,36 @@
+use super::AtomicU64;
+use crate::loom::sync::{atomic::Ordering, Mutex};
+use crate::util::once_cell::OnceCell;
+
+pub(crate) struct StaticAtomicU64 {
+    init: u64,
+    cell: OnceCell<Mutex<u64>>,
+}
+
+impl AtomicU64 {
+    pub(crate) fn new(val: u64) -> Self {
+        Self {
+            inner: Mutex::new(val),
+        }
+    }
+}
+
+impl StaticAtomicU64 {
+    pub(crate) const fn new(val: u64) -> StaticAtomicU64 {
+        StaticAtomicU64 {
+            init: val,
+            cell: OnceCell::new(),
+        }
+    }
+
+    pub(crate) fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+        let mut lock = self.inner().lock();
+        let prev = *lock;
+        *lock = prev + val;
+        prev
+    }
+
+    fn inner(&self) -> &Mutex<u64> {
+        self.cell.get(|| Mutex::new(self.init))
+    }
+}

tokio/src/loom/std/mod.rs

@@ -71,7 +71,7 @@ pub(crate) mod sync {
     pub(crate) mod atomic {
         pub(crate) use crate::loom::std::atomic_u16::AtomicU16;
         pub(crate) use crate::loom::std::atomic_u32::AtomicU32;
-        pub(crate) use crate::loom::std::atomic_u64::AtomicU64;
+        pub(crate) use crate::loom::std::atomic_u64::{AtomicU64, StaticAtomicU64};
         pub(crate) use crate::loom::std::atomic_usize::AtomicUsize;
 
         pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering};
@@ -81,7 +81,27 @@ pub(crate) mod sync {
 pub(crate) mod sys {
     #[cfg(feature = "rt-multi-thread")]
     pub(crate) fn num_cpus() -> usize {
-        usize::max(1, num_cpus::get())
+        const ENV_WORKER_THREADS: &str = "TOKIO_WORKER_THREADS";
+
+        match std::env::var(ENV_WORKER_THREADS) {
+            Ok(s) => {
+                let n = s.parse().unwrap_or_else(|e| {
+                    panic!(
+                        "\"{}\" must be usize, error: {}, value: {}",
+                        ENV_WORKER_THREADS, e, s
+                    )
+                });
+                assert!(n > 0, "\"{}\" cannot be set to 0", ENV_WORKER_THREADS);
+                n
+            }
+            Err(std::env::VarError::NotPresent) => usize::max(1, num_cpus::get()),
+            Err(std::env::VarError::NotUnicode(e)) => {
+                panic!(
+                    "\"{}\" must be valid unicode, error: {:?}",
+                    ENV_WORKER_THREADS, e
+                )
+            }
+        }
     }
 
     #[cfg(not(feature = "rt-multi-thread"))]

tokio/src/macros/cfg.rs

@@ -461,13 +461,14 @@ macro_rules! cfg_not_coop {
 macro_rules! cfg_has_atomic_u64 {
     ($($item:item)*) => {
         $(
-            #[cfg(not(any(
-                    target_arch = "arm",
-                    target_arch = "mips",
-                    target_arch = "powerpc",
-                    target_arch = "riscv32",
-                    tokio_wasm
-                    )))]
+            #[cfg_attr(
+                not(tokio_no_target_has_atomic),
+                cfg(all(target_has_atomic = "64", not(tokio_no_atomic_u64))
+            ))]
+            #[cfg_attr(
+                tokio_no_target_has_atomic,
+                cfg(not(tokio_no_atomic_u64))
+            )]
             $item
         )*
     }
@@ -476,13 +477,14 @@ macro_rules! cfg_has_atomic_u64 {
 macro_rules! cfg_not_has_atomic_u64 {
     ($($item:item)*) => {
         $(
-            #[cfg(any(
-                    target_arch = "arm",
-                    target_arch = "mips",
-                    target_arch = "powerpc",
-                    target_arch = "riscv32",
-                    tokio_wasm
-                    ))]
+            #[cfg_attr(
+                not(tokio_no_target_has_atomic),
+                cfg(any(not(target_has_atomic = "64"), tokio_no_atomic_u64)
+            ))]
+            #[cfg_attr(
+                tokio_no_target_has_atomic,
+                cfg(tokio_no_atomic_u64)
+            )]
             $item
         )*
     }

tokio/src/net/tcp/split.rs

@@ -145,6 +145,12 @@ impl ReadHalf<'_> {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety
@@ -273,6 +279,12 @@ impl WriteHalf<'_> {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety

tokio/src/net/tcp/split_owned.rs

@@ -200,6 +200,12 @@ impl OwnedReadHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety
@@ -355,6 +361,12 @@ impl OwnedWriteHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// This function is equivalent to [`TcpStream::ready`].
     ///
     /// # Cancel safety
@@ -478,12 +490,12 @@ impl AsyncWrite for OwnedWriteHalf {
 
 impl AsRef<TcpStream> for OwnedReadHalf {
     fn as_ref(&self) -> &TcpStream {
-        &*self.inner
+        &self.inner
     }
 }
 
 impl AsRef<TcpStream> for OwnedWriteHalf {
     fn as_ref(&self) -> &TcpStream {
-        &*self.inner
+        &self.inner
     }
 }

tokio/src/net/tcp/stream.rs

@@ -377,6 +377,12 @@ impl TcpStream {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method
@@ -964,7 +970,7 @@ impl TcpStream {
     /// Tries to read or write from the socket using a user-provided IO operation.
     ///
     /// If the socket is ready, the provided closure is called. The closure
-    /// should attempt to perform IO operation from the socket by manually
+    /// should attempt to perform IO operation on the socket by manually
     /// calling the appropriate syscall. If the operation fails because the
     /// socket is not actually ready, then the closure should return a
     /// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -983,6 +989,11 @@ impl TcpStream {
     /// defined on the Tokio `TcpStream` type, as this will mess with the
     /// readiness flag and can cause the socket to behave incorrectly.
     ///
+    /// This method is not intended to be used with combined interests.
+    /// The closure should perform only one type of IO operation, so it should not
+    /// require more than one ready state. This method may panic or sleep forever
+    /// if it is called with a combined interest.
+    ///
     /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
     ///
     /// [`readable()`]: TcpStream::readable()

tokio/src/net/udp.rs

@@ -357,7 +357,9 @@ impl UdpSocket {
     ///
     /// The function may complete without the socket being ready. This is a
     /// false-positive and attempting an operation will return with
-    /// `io::ErrorKind::WouldBlock`.
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
     ///
     /// # Cancel safety
     ///
@@ -1271,7 +1273,7 @@ impl UdpSocket {
     /// Tries to read or write from the socket using a user-provided IO operation.
     ///
     /// If the socket is ready, the provided closure is called. The closure
-    /// should attempt to perform IO operation from the socket by manually
+    /// should attempt to perform IO operation on the socket by manually
     /// calling the appropriate syscall. If the operation fails because the
     /// socket is not actually ready, then the closure should return a
     /// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -1290,6 +1292,11 @@ impl UdpSocket {
     /// defined on the Tokio `UdpSocket` type, as this will mess with the
     /// readiness flag and can cause the socket to behave incorrectly.
     ///
+    /// This method is not intended to be used with combined interests.
+    /// The closure should perform only one type of IO operation, so it should not
+    /// require more than one ready state. This method may panic or sleep forever
+    /// if it is called with a combined interest.
+    ///
     /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
     ///
     /// [`readable()`]: UdpSocket::readable()

tokio/src/net/unix/datagram/socket.rs

@@ -104,7 +104,9 @@ impl UnixDatagram {
     ///
     /// The function may complete without the socket being ready. This is a
     /// false-positive and attempting an operation will return with
-    /// `io::ErrorKind::WouldBlock`.
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
     ///
     /// # Cancel safety
     ///
@@ -1214,7 +1216,7 @@ impl UnixDatagram {
     /// Tries to read or write from the socket using a user-provided IO operation.
     ///
     /// If the socket is ready, the provided closure is called. The closure
-    /// should attempt to perform IO operation from the socket by manually
+    /// should attempt to perform IO operation on the socket by manually
     /// calling the appropriate syscall. If the operation fails because the
     /// socket is not actually ready, then the closure should return a
     /// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -1233,6 +1235,11 @@ impl UnixDatagram {
     /// defined on the Tokio `UnixDatagram` type, as this will mess with the
     /// readiness flag and can cause the socket to behave incorrectly.
     ///
+    /// This method is not intended to be used with combined interests.
+    /// The closure should perform only one type of IO operation, so it should not
+    /// require more than one ready state. This method may panic or sleep forever
+    /// if it is called with a combined interest.
+    ///
     /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
     ///
     /// [`readable()`]: UnixDatagram::readable()

tokio/src/net/unix/split.rs

@@ -182,6 +182,12 @@ impl WriteHalf<'_> {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method

tokio/src/net/unix/split_owned.rs

@@ -114,6 +114,12 @@ impl OwnedReadHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method
@@ -265,6 +271,12 @@ impl OwnedWriteHalf {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method
@@ -386,12 +398,12 @@ impl AsyncWrite for OwnedWriteHalf {
 
 impl AsRef<UnixStream> for OwnedReadHalf {
     fn as_ref(&self) -> &UnixStream {
-        &*self.inner
+        &self.inner
     }
 }
 
 impl AsRef<UnixStream> for OwnedWriteHalf {
     fn as_ref(&self) -> &UnixStream {
-        &*self.inner
+        &self.inner
     }
 }

tokio/src/net/unix/stream.rs

@@ -66,6 +66,12 @@ impl UnixStream {
     /// can be used to concurrently read / write to the same socket on a single
     /// task without splitting the socket.
     ///
+    /// The function may complete without the socket being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Cancel safety
     ///
     /// This method is cancel safe. Once a readiness event occurs, the method
@@ -661,7 +667,7 @@ impl UnixStream {
     /// Tries to read or write from the socket using a user-provided IO operation.
     ///
     /// If the socket is ready, the provided closure is called. The closure
-    /// should attempt to perform IO operation from the socket by manually
+    /// should attempt to perform IO operation on the socket by manually
     /// calling the appropriate syscall. If the operation fails because the
     /// socket is not actually ready, then the closure should return a
     /// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -680,6 +686,11 @@ impl UnixStream {
     /// defined on the Tokio `UnixStream` type, as this will mess with the
     /// readiness flag and can cause the socket to behave incorrectly.
     ///
+    /// This method is not intended to be used with combined interests.
+    /// The closure should perform only one type of IO operation, so it should not
+    /// require more than one ready state. This method may panic or sleep forever
+    /// if it is called with a combined interest.
+    ///
     /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
     ///
     /// [`readable()`]: UnixStream::readable()

tokio/src/net/windows/named_pipe.rs

@@ -20,21 +20,18 @@ cfg_io_util! {
 #[cfg(not(docsrs))]
 mod doc {
     pub(super) use crate::os::windows::ffi::OsStrExt;
-    pub(super) use crate::winapi::shared::minwindef::{DWORD, FALSE};
-    pub(super) use crate::winapi::um::fileapi;
-    pub(super) use crate::winapi::um::handleapi;
-    pub(super) use crate::winapi::um::namedpipeapi;
-    pub(super) use crate::winapi::um::winbase;
-    pub(super) use crate::winapi::um::winnt;
-
+    pub(super) mod windows_sys {
+        pub(crate) use windows_sys::{
+            Win32::Foundation::*, Win32::Storage::FileSystem::*, Win32::System::Pipes::*,
+            Win32::System::SystemServices::*,
+        };
+    }
     pub(super) use mio::windows as mio_windows;
 }
 
 // NB: none of these shows up in public API, so don't document them.
 #[cfg(docsrs)]
 mod doc {
-    pub type DWORD = crate::doc::NotDefinedHere;
-
     pub(super) mod mio_windows {
         pub type NamedPipe = crate::doc::NotDefinedHere;
     }
@@ -101,7 +98,6 @@ use self::doc::*;
 /// # Ok(()) }
 /// ```
 ///
-/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY
 /// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
 #[derive(Debug)]
 pub struct NamedPipeServer {
@@ -192,17 +188,15 @@ impl NamedPipeServer {
     /// # Ok(()) }
     /// ```
     pub async fn connect(&self) -> io::Result<()> {
-        loop {
-            match self.io.connect() {
-                Ok(()) => break,
-                Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
-                    self.io.registration().readiness(Interest::WRITABLE).await?;
-                }
-                Err(e) => return Err(e),
+        match self.io.connect() {
+            Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+                self.io
+                    .registration()
+                    .async_io(Interest::WRITABLE, || self.io.connect())
+                    .await
             }
+            x => x,
         }
-
-        Ok(())
     }
 
     /// Disconnects the server end of a named pipe instance from a client
@@ -211,7 +205,7 @@ impl NamedPipeServer {
     /// ```
     /// use tokio::io::AsyncWriteExt;
     /// use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions};
-    /// use winapi::shared::winerror;
+    /// use windows_sys::Win32::Foundation::ERROR_PIPE_NOT_CONNECTED;
     ///
     /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-disconnect";
     ///
@@ -231,7 +225,7 @@ impl NamedPipeServer {
     /// // Write fails with an OS-specific error after client has been
     /// // disconnected.
     /// let e = client.write(b"ping").await.unwrap_err();
-    /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_NOT_CONNECTED as i32));
+    /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_NOT_CONNECTED as i32));
     /// # Ok(()) }
     /// ```
     pub fn disconnect(&self) -> io::Result<()> {
@@ -244,6 +238,12 @@ impl NamedPipeServer {
     /// can be used to concurrently read / write to the same pipe on a single
     /// task without splitting the pipe.
     ///
+    /// The function may complete without the pipe being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Examples
     ///
     /// Concurrently read and write to the pipe on the same task without
@@ -540,7 +540,7 @@ impl NamedPipeServer {
         /// Tries to read data from the stream into the provided buffer, advancing the
         /// buffer's internal cursor, returning how many bytes were read.
         ///
-        /// Receives any pending data from the socket but does not wait for new data
+        /// Receives any pending data from the pipe but does not wait for new data
         /// to arrive. On success, returns the number of bytes read. Because
         /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by
         /// the async task and can exist entirely on the stack.
@@ -571,7 +571,7 @@ impl NamedPipeServer {
         ///     let server = named_pipe::ServerOptions::new().create(PIPE_NAME)?;
         ///
         ///     loop {
-        ///         // Wait for the socket to be readable
+        ///         // Wait for the pipe to be readable
         ///         server.readable().await?;
         ///
         ///         let mut buf = Vec::with_capacity(4096);
@@ -812,27 +812,32 @@ impl NamedPipeServer {
             .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf))
     }
 
-    /// Tries to read or write from the socket using a user-provided IO operation.
+    /// Tries to read or write from the pipe using a user-provided IO operation.
     ///
-    /// If the socket is ready, the provided closure is called. The closure
-    /// should attempt to perform IO operation from the socket by manually
+    /// If the pipe is ready, the provided closure is called. The closure
+    /// should attempt to perform IO operation from the pipe by manually
     /// calling the appropriate syscall. If the operation fails because the
-    /// socket is not actually ready, then the closure should return a
+    /// pipe is not actually ready, then the closure should return a
     /// `WouldBlock` error and the readiness flag is cleared. The return value
     /// of the closure is then returned by `try_io`.
     ///
-    /// If the socket is not ready, then the closure is not called
+    /// If the pipe is not ready, then the closure is not called
     /// and a `WouldBlock` error is returned.
     ///
     /// The closure should only return a `WouldBlock` error if it has performed
-    /// an IO operation on the socket that failed due to the socket not being
+    /// an IO operation on the pipe that failed due to the pipe not being
     /// ready. Returning a `WouldBlock` error in any other situation will
-    /// incorrectly clear the readiness flag, which can cause the socket to
+    /// incorrectly clear the readiness flag, which can cause the pipe to
     /// behave incorrectly.
     ///
     /// The closure should not perform the IO operation using any of the
     /// methods defined on the Tokio `NamedPipeServer` type, as this will mess with
-    /// the readiness flag and can cause the socket to behave incorrectly.
+    /// the readiness flag and can cause the pipe to behave incorrectly.
+    ///
+    /// This method is not intended to be used with combined interests.
+    /// The closure should perform only one type of IO operation, so it should not
+    /// require more than one ready state. This method may panic or sleep forever
+    /// if it is called with a combined interest.
     ///
     /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
     ///
@@ -907,7 +912,7 @@ impl AsRawHandle for NamedPipeServer {
 /// use std::time::Duration;
 /// use tokio::net::windows::named_pipe::ClientOptions;
 /// use tokio::time;
-/// use winapi::shared::winerror;
+/// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
 ///
 /// const PIPE_NAME: &str = r"\\.\pipe\named-pipe-idiomatic-client";
 ///
@@ -915,7 +920,7 @@ impl AsRawHandle for NamedPipeServer {
 /// let client = loop {
 ///     match ClientOptions::new().open(PIPE_NAME) {
 ///         Ok(client) => break client,
-///         Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+///         Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
 ///         Err(e) => return Err(e),
 ///     }
 ///
@@ -926,7 +931,7 @@ impl AsRawHandle for NamedPipeServer {
 /// # Ok(()) }
 /// ```
 ///
-/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY
+/// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html
 /// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
 #[derive(Debug)]
 pub struct NamedPipeClient {
@@ -990,6 +995,12 @@ impl NamedPipeClient {
     /// can be used to concurrently read / write to the same pipe on a single
     /// task without splitting the pipe.
     ///
+    /// The function may complete without the pipe being ready. This is a
+    /// false-positive and attempting an operation will return with
+    /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+    /// [`Ready`] set, so you should always check the returned value and possibly
+    /// wait again if the requested states are not set.
+    ///
     /// # Examples
     ///
     /// Concurrently read and write to the pipe on the same task without
@@ -1282,7 +1293,7 @@ impl NamedPipeClient {
         /// Tries to read data from the stream into the provided buffer, advancing the
         /// buffer's internal cursor, returning how many bytes were read.
         ///
-        /// Receives any pending data from the socket but does not wait for new data
+        /// Receives any pending data from the pipe but does not wait for new data
         /// to arrive. On success, returns the number of bytes read. Because
         /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by
         /// the async task and can exist entirely on the stack.
@@ -1313,7 +1324,7 @@ impl NamedPipeClient {
         ///     let client = named_pipe::ClientOptions::new().open(PIPE_NAME)?;
         ///
         ///     loop {
-        ///         // Wait for the socket to be readable
+        ///         // Wait for the pipe to be readable
         ///         client.readable().await?;
         ///
         ///         let mut buf = Vec::with_capacity(4096);
@@ -1551,27 +1562,32 @@ impl NamedPipeClient {
             .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf))
     }
 
-    /// Tries to read or write from the socket using a user-provided IO operation.
+    /// Tries to read or write from the pipe using a user-provided IO operation.
     ///
-    /// If the socket is ready, the provided closure is called. The closure
-    /// should attempt to perform IO operation from the socket by manually
+    /// If the pipe is ready, the provided closure is called. The closure
+    /// should attempt to perform IO operation from the pipe by manually
     /// calling the appropriate syscall. If the operation fails because the
-    /// socket is not actually ready, then the closure should return a
+    /// pipe is not actually ready, then the closure should return a
     /// `WouldBlock` error and the readiness flag is cleared. The return value
     /// of the closure is then returned by `try_io`.
     ///
-    /// If the socket is not ready, then the closure is not called
+    /// If the pipe is not ready, then the closure is not called
     /// and a `WouldBlock` error is returned.
     ///
     /// The closure should only return a `WouldBlock` error if it has performed
-    /// an IO operation on the socket that failed due to the socket not being
+    /// an IO operation on the pipe that failed due to the pipe not being
     /// ready. Returning a `WouldBlock` error in any other situation will
-    /// incorrectly clear the readiness flag, which can cause the socket to
+    /// incorrectly clear the readiness flag, which can cause the pipe to
     /// behave incorrectly.
     ///
     /// The closure should not perform the IO operation using any of the methods
     /// defined on the Tokio `NamedPipeClient` type, as this will mess with the
-    /// readiness flag and can cause the socket to behave incorrectly.
+    /// readiness flag and can cause the pipe to behave incorrectly.
+    ///
+    /// This method is not intended to be used with combined interests.
+    /// The closure should perform only one type of IO operation, so it should not
+    /// require more than one ready state. This method may panic or sleep forever
+    /// if it is called with a combined interest.
     ///
     /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
     ///
@@ -1649,12 +1665,12 @@ macro_rules! bool_flag {
 /// See [`ServerOptions::create`].
 #[derive(Debug, Clone)]
 pub struct ServerOptions {
-    open_mode: DWORD,
-    pipe_mode: DWORD,
-    max_instances: DWORD,
-    out_buffer_size: DWORD,
-    in_buffer_size: DWORD,
-    default_timeout: DWORD,
+    open_mode: u32,
+    pipe_mode: u32,
+    max_instances: u32,
+    out_buffer_size: u32,
+    in_buffer_size: u32,
+    default_timeout: u32,
 }
 
 impl ServerOptions {
@@ -1671,9 +1687,9 @@ impl ServerOptions {
     /// ```
     pub fn new() -> ServerOptions {
         ServerOptions {
-            open_mode: winbase::PIPE_ACCESS_DUPLEX | winbase::FILE_FLAG_OVERLAPPED,
-            pipe_mode: winbase::PIPE_TYPE_BYTE | winbase::PIPE_REJECT_REMOTE_CLIENTS,
-            max_instances: winbase::PIPE_UNLIMITED_INSTANCES,
+            open_mode: windows_sys::PIPE_ACCESS_DUPLEX | windows_sys::FILE_FLAG_OVERLAPPED,
+            pipe_mode: windows_sys::PIPE_TYPE_BYTE | windows_sys::PIPE_REJECT_REMOTE_CLIENTS,
+            max_instances: windows_sys::PIPE_UNLIMITED_INSTANCES,
             out_buffer_size: 65536,
             in_buffer_size: 65536,
             default_timeout: 0,
@@ -1690,8 +1706,8 @@ impl ServerOptions {
     /// [`dwPipeMode`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn pipe_mode(&mut self, pipe_mode: PipeMode) -> &mut Self {
         self.pipe_mode = match pipe_mode {
-            PipeMode::Byte => winbase::PIPE_TYPE_BYTE,
-            PipeMode::Message => winbase::PIPE_TYPE_MESSAGE,
+            PipeMode::Byte => windows_sys::PIPE_TYPE_BYTE,
+            PipeMode::Message => windows_sys::PIPE_TYPE_MESSAGE,
         };
 
         self
@@ -1789,7 +1805,7 @@ impl ServerOptions {
     /// # Ok(()) }
     /// ```
     pub fn access_inbound(&mut self, allowed: bool) -> &mut Self {
-        bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_INBOUND);
+        bool_flag!(self.open_mode, allowed, windows_sys::PIPE_ACCESS_INBOUND);
         self
     }
 
@@ -1887,7 +1903,7 @@ impl ServerOptions {
     /// # Ok(()) }
     /// ```
     pub fn access_outbound(&mut self, allowed: bool) -> &mut Self {
-        bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_OUTBOUND);
+        bool_flag!(self.open_mode, allowed, windows_sys::PIPE_ACCESS_OUTBOUND);
         self
     }
 
@@ -1958,7 +1974,7 @@ impl ServerOptions {
         bool_flag!(
             self.open_mode,
             first,
-            winbase::FILE_FLAG_FIRST_PIPE_INSTANCE
+            windows_sys::FILE_FLAG_FIRST_PIPE_INSTANCE
         );
         self
     }
@@ -1973,9 +1989,10 @@ impl ServerOptions {
     /// use std::{io, os::windows::prelude::AsRawHandle, ptr};
     //
     /// use tokio::net::windows::named_pipe::ServerOptions;
-    /// use winapi::{
-    ///     shared::winerror::ERROR_SUCCESS,
-    ///     um::{accctrl::SE_KERNEL_OBJECT, aclapi::SetSecurityInfo, winnt::DACL_SECURITY_INFORMATION},
+    /// use windows_sys::{
+    ///     Win32::Foundation::ERROR_SUCCESS,
+    ///     Win32::Security::DACL_SECURITY_INFORMATION,
+    ///     Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT},
     /// };
     ///
     /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe";
@@ -1989,7 +2006,7 @@ impl ServerOptions {
     ///     assert_eq!(
     ///         ERROR_SUCCESS,
     ///         SetSecurityInfo(
-    ///             pipe.as_raw_handle(),
+    ///             pipe.as_raw_handle() as _,
     ///             SE_KERNEL_OBJECT,
     ///             DACL_SECURITY_INFORMATION,
     ///             ptr::null_mut(),
@@ -2007,9 +2024,10 @@ impl ServerOptions {
     /// use std::{io, os::windows::prelude::AsRawHandle, ptr};
     //
     /// use tokio::net::windows::named_pipe::ServerOptions;
-    /// use winapi::{
-    ///     shared::winerror::ERROR_ACCESS_DENIED,
-    ///     um::{accctrl::SE_KERNEL_OBJECT, aclapi::SetSecurityInfo, winnt::DACL_SECURITY_INFORMATION},
+    /// use windows_sys::{
+    ///     Win32::Foundation::ERROR_ACCESS_DENIED,
+    ///     Win32::Security::DACL_SECURITY_INFORMATION,
+    ///     Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT},
     /// };
     ///
     /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe_fail";
@@ -2023,7 +2041,7 @@ impl ServerOptions {
     ///     assert_eq!(
     ///         ERROR_ACCESS_DENIED,
     ///         SetSecurityInfo(
-    ///             pipe.as_raw_handle(),
+    ///             pipe.as_raw_handle() as _,
     ///             SE_KERNEL_OBJECT,
     ///             DACL_SECURITY_INFORMATION,
     ///             ptr::null_mut(),
@@ -2039,7 +2057,7 @@ impl ServerOptions {
     ///
     /// [`WRITE_DAC`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn write_dac(&mut self, requested: bool) -> &mut Self {
-        bool_flag!(self.open_mode, requested, winnt::WRITE_DAC);
+        bool_flag!(self.open_mode, requested, windows_sys::WRITE_DAC);
         self
     }
 
@@ -2049,7 +2067,7 @@ impl ServerOptions {
     ///
     /// [`WRITE_OWNER`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn write_owner(&mut self, requested: bool) -> &mut Self {
-        bool_flag!(self.open_mode, requested, winnt::WRITE_OWNER);
+        bool_flag!(self.open_mode, requested, windows_sys::WRITE_OWNER);
         self
     }
 
@@ -2059,7 +2077,11 @@ impl ServerOptions {
     ///
     /// [`ACCESS_SYSTEM_SECURITY`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn access_system_security(&mut self, requested: bool) -> &mut Self {
-        bool_flag!(self.open_mode, requested, winnt::ACCESS_SYSTEM_SECURITY);
+        bool_flag!(
+            self.open_mode,
+            requested,
+            windows_sys::ACCESS_SYSTEM_SECURITY
+        );
         self
     }
 
@@ -2070,7 +2092,11 @@ impl ServerOptions {
     ///
     /// [`PIPE_REJECT_REMOTE_CLIENTS`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea#pipe_reject_remote_clients
     pub fn reject_remote_clients(&mut self, reject: bool) -> &mut Self {
-        bool_flag!(self.pipe_mode, reject, winbase::PIPE_REJECT_REMOTE_CLIENTS);
+        bool_flag!(
+            self.pipe_mode,
+            reject,
+            windows_sys::PIPE_REJECT_REMOTE_CLIENTS
+        );
         self
     }
 
@@ -2092,7 +2118,7 @@ impl ServerOptions {
     /// ```
     /// use std::io;
     /// use tokio::net::windows::named_pipe::{ServerOptions, ClientOptions};
-    /// use winapi::shared::winerror;
+    /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
     ///
     /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-max-instances";
     ///
@@ -2108,11 +2134,11 @@ impl ServerOptions {
     ///
     /// // Too many servers!
     /// let e = server.create(PIPE_NAME).unwrap_err();
-    /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32));
+    /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32));
     ///
     /// // Still too many servers even if we specify a higher value!
     /// let e = server.max_instances(100).create(PIPE_NAME).unwrap_err();
-    /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32));
+    /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32));
     /// # Ok(()) }
     /// ```
     ///
@@ -2131,7 +2157,7 @@ impl ServerOptions {
     #[track_caller]
     pub fn max_instances(&mut self, instances: usize) -> &mut Self {
         assert!(instances < 255, "cannot specify more than 254 instances");
-        self.max_instances = instances as DWORD;
+        self.max_instances = instances as u32;
         self
     }
 
@@ -2141,7 +2167,7 @@ impl ServerOptions {
     ///
     /// [`nOutBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn out_buffer_size(&mut self, buffer: u32) -> &mut Self {
-        self.out_buffer_size = buffer as DWORD;
+        self.out_buffer_size = buffer;
         self
     }
 
@@ -2151,7 +2177,7 @@ impl ServerOptions {
     ///
     /// [`nInBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
     pub fn in_buffer_size(&mut self, buffer: u32) -> &mut Self {
-        self.in_buffer_size = buffer as DWORD;
+        self.in_buffer_size = buffer;
         self
     }
 
@@ -2208,7 +2234,7 @@ impl ServerOptions {
     ///
     /// [`create`]: ServerOptions::create
     /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
-    /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES
+    /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html
     pub unsafe fn create_with_security_attributes_raw(
         &self,
         addr: impl AsRef<OsStr>,
@@ -2216,7 +2242,7 @@ impl ServerOptions {
     ) -> io::Result<NamedPipeServer> {
         let addr = encode_addr(addr);
 
-        let h = namedpipeapi::CreateNamedPipeW(
+        let h = windows_sys::CreateNamedPipeW(
             addr.as_ptr(),
             self.open_mode,
             self.pipe_mode,
@@ -2227,11 +2253,11 @@ impl ServerOptions {
             attrs as *mut _,
         );
 
-        if h == handleapi::INVALID_HANDLE_VALUE {
+        if h == windows_sys::INVALID_HANDLE_VALUE {
             return Err(io::Error::last_os_error());
         }
 
-        NamedPipeServer::from_raw_handle(h)
+        NamedPipeServer::from_raw_handle(h as _)
     }
 }
 
@@ -2241,8 +2267,8 @@ impl ServerOptions {
 /// See [`ClientOptions::open`].
 #[derive(Debug, Clone)]
 pub struct ClientOptions {
-    desired_access: DWORD,
-    security_qos_flags: DWORD,
+    desired_access: u32,
+    security_qos_flags: u32,
 }
 
 impl ClientOptions {
@@ -2261,8 +2287,9 @@ impl ClientOptions {
     /// ```
     pub fn new() -> Self {
         Self {
-            desired_access: winnt::GENERIC_READ | winnt::GENERIC_WRITE,
-            security_qos_flags: winbase::SECURITY_IDENTIFICATION | winbase::SECURITY_SQOS_PRESENT,
+            desired_access: windows_sys::GENERIC_READ | windows_sys::GENERIC_WRITE,
+            security_qos_flags: windows_sys::SECURITY_IDENTIFICATION
+                | windows_sys::SECURITY_SQOS_PRESENT,
         }
     }
 
@@ -2273,7 +2300,7 @@ impl ClientOptions {
     /// [`GENERIC_READ`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights
     /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
     pub fn read(&mut self, allowed: bool) -> &mut Self {
-        bool_flag!(self.desired_access, allowed, winnt::GENERIC_READ);
+        bool_flag!(self.desired_access, allowed, windows_sys::GENERIC_READ);
         self
     }
 
@@ -2284,7 +2311,7 @@ impl ClientOptions {
     /// [`GENERIC_WRITE`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights
     /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
     pub fn write(&mut self, allowed: bool) -> &mut Self {
-        bool_flag!(self.desired_access, allowed, winnt::GENERIC_WRITE);
+        bool_flag!(self.desired_access, allowed, windows_sys::GENERIC_WRITE);
         self
     }
 
@@ -2307,11 +2334,11 @@ impl ClientOptions {
     /// automatically when using this method.
     ///
     /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea
-    /// [`SECURITY_IDENTIFICATION`]: crate::winapi::um::winbase::SECURITY_IDENTIFICATION
+    /// [`SECURITY_IDENTIFICATION`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Storage/FileSystem/constant.SECURITY_IDENTIFICATION.html
     /// [Impersonation Levels]: https://docs.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level
     pub fn security_qos_flags(&mut self, flags: u32) -> &mut Self {
         // See: https://github.com/rust-lang/rust/pull/58216
-        self.security_qos_flags = flags | winbase::SECURITY_SQOS_PRESENT;
+        self.security_qos_flags = flags | windows_sys::SECURITY_SQOS_PRESENT;
         self
     }
 
@@ -2336,8 +2363,7 @@ impl ClientOptions {
     ///   but the server is not currently waiting for a connection. Please see the
     ///   examples for how to check for this error.
     ///
-    /// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY
-    /// [`winapi`]: crate::winapi
+    /// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html
     /// [enabled I/O]: crate::runtime::Builder::enable_io
     /// [Tokio Runtime]: crate::runtime::Runtime
     ///
@@ -2348,7 +2374,7 @@ impl ClientOptions {
     /// use std::time::Duration;
     /// use tokio::net::windows::named_pipe::ClientOptions;
     /// use tokio::time;
-    /// use winapi::shared::winerror;
+    /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
     ///
     /// const PIPE_NAME: &str = r"\\.\pipe\mynamedpipe";
     ///
@@ -2356,7 +2382,7 @@ impl ClientOptions {
     /// let client = loop {
     ///     match ClientOptions::new().open(PIPE_NAME) {
     ///         Ok(client) => break client,
-    ///         Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+    ///         Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
     ///         Err(e) => return Err(e),
     ///     }
     ///
@@ -2386,7 +2412,7 @@ impl ClientOptions {
     ///
     /// [`open`]: ClientOptions::open
     /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
-    /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES
+    /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html
     pub unsafe fn open_with_security_attributes_raw(
         &self,
         addr: impl AsRef<OsStr>,
@@ -2395,28 +2421,28 @@ impl ClientOptions {
         let addr = encode_addr(addr);
 
         // NB: We could use a platform specialized `OpenOptions` here, but since
-        // we have access to winapi it ultimately doesn't hurt to use
+        // we have access to windows_sys it ultimately doesn't hurt to use
         // `CreateFile` explicitly since it allows the use of our already
         // well-structured wide `addr` to pass into CreateFileW.
-        let h = fileapi::CreateFileW(
+        let h = windows_sys::CreateFileW(
             addr.as_ptr(),
             self.desired_access,
             0,
             attrs as *mut _,
-            fileapi::OPEN_EXISTING,
+            windows_sys::OPEN_EXISTING,
             self.get_flags(),
-            ptr::null_mut(),
+            0,
         );
 
-        if h == handleapi::INVALID_HANDLE_VALUE {
+        if h == windows_sys::INVALID_HANDLE_VALUE {
             return Err(io::Error::last_os_error());
         }
 
-        NamedPipeClient::from_raw_handle(h)
+        NamedPipeClient::from_raw_handle(h as _)
     }
 
     fn get_flags(&self) -> u32 {
-        self.security_qos_flags | winbase::FILE_FLAG_OVERLAPPED
+        self.security_qos_flags | windows_sys::FILE_FLAG_OVERLAPPED
     }
 }
 
@@ -2429,16 +2455,19 @@ pub enum PipeMode {
     /// Data is written to the pipe as a stream of bytes. The pipe does not
     /// distinguish bytes written during different write operations.
     ///
-    /// Corresponds to [`PIPE_TYPE_BYTE`][crate::winapi::um::winbase::PIPE_TYPE_BYTE].
+    /// Corresponds to [`PIPE_TYPE_BYTE`].
+    ///
+    /// [`PIPE_TYPE_BYTE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_BYTE.html
     Byte,
     /// Data is written to the pipe as a stream of messages. The pipe treats the
     /// bytes written during each write operation as a message unit. Any reading
     /// on a named pipe returns [`ERROR_MORE_DATA`] when a message is not read
     /// completely.
     ///
-    /// Corresponds to [`PIPE_TYPE_MESSAGE`][crate::winapi::um::winbase::PIPE_TYPE_MESSAGE].
+    /// Corresponds to [`PIPE_TYPE_MESSAGE`].
     ///
-    /// [`ERROR_MORE_DATA`]: crate::winapi::shared::winerror::ERROR_MORE_DATA
+    /// [`ERROR_MORE_DATA`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_MORE_DATA.html
+    /// [`PIPE_TYPE_MESSAGE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_MESSAGE.html
     Message,
 }
 
@@ -2448,11 +2477,15 @@ pub enum PipeMode {
 pub enum PipeEnd {
     /// The named pipe refers to the client end of a named pipe instance.
     ///
-    /// Corresponds to [`PIPE_CLIENT_END`][crate::winapi::um::winbase::PIPE_CLIENT_END].
+    /// Corresponds to [`PIPE_CLIENT_END`].
+    ///
+    /// [`PIPE_CLIENT_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_CLIENT_END.html
     Client,
     /// The named pipe refers to the server end of a named pipe instance.
     ///
-    /// Corresponds to [`PIPE_SERVER_END`][crate::winapi::um::winbase::PIPE_SERVER_END].
+    /// Corresponds to [`PIPE_SERVER_END`].
+    ///
+    /// [`PIPE_SERVER_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_SERVER_END.html
     Server,
 }
 
@@ -2490,26 +2523,26 @@ unsafe fn named_pipe_info(handle: RawHandle) -> io::Result<PipeInfo> {
     let mut in_buffer_size = 0;
     let mut max_instances = 0;
 
-    let result = namedpipeapi::GetNamedPipeInfo(
-        handle,
+    let result = windows_sys::GetNamedPipeInfo(
+        handle as _,
         &mut flags,
         &mut out_buffer_size,
         &mut in_buffer_size,
         &mut max_instances,
     );
 
-    if result == FALSE {
+    if result == 0 {
         return Err(io::Error::last_os_error());
     }
 
     let mut end = PipeEnd::Client;
     let mut mode = PipeMode::Byte;
 
-    if flags & winbase::PIPE_SERVER_END != 0 {
+    if flags & windows_sys::PIPE_SERVER_END != 0 {
         end = PipeEnd::Server;
     }
 
-    if flags & winbase::PIPE_TYPE_MESSAGE != 0 {
+    if flags & windows_sys::PIPE_TYPE_MESSAGE != 0 {
         mode = PipeMode::Message;
     }
 

tokio/src/process/mod.rs

@@ -1329,6 +1329,18 @@ impl AsyncWrite for ChildStdin {
     fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
         Pin::new(&mut self.inner).poll_shutdown(cx)
     }
+
+    fn poll_write_vectored(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[io::IoSlice<'_>],
+    ) -> Poll<Result<usize, io::Error>> {
+        Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.is_write_vectored()
+    }
 }
 
 impl AsyncRead for ChildStdout {

tokio/src/process/unix/mod.rs

@@ -182,6 +182,10 @@ impl<'a> io::Write for &'a Pipe {
     fn flush(&mut self) -> io::Result<()> {
         (&self.fd).flush()
     }
+
+    fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result<usize> {
+        (&self.fd).write_vectored(bufs)
+    }
 }
 
 impl AsRawFd for Pipe {
@@ -258,6 +262,18 @@ impl AsyncWrite for ChildStdio {
     fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
         Poll::Ready(Ok(()))
     }
+
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[io::IoSlice<'_>],
+    ) -> Poll<Result<usize, io::Error>> {
+        self.inner.poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        true
+    }
 }
 
 impl AsyncRead for ChildStdio {

tokio/src/process/unix/orphan.rs

@@ -294,7 +294,7 @@ pub(crate) mod test {
     #[cfg_attr(miri, ignore)] // Miri does not support epoll.
     #[test]
     fn does_not_register_signal_if_queue_empty() {
-        let (io_driver, io_handle) = IoDriver::new().unwrap();
+        let (io_driver, io_handle) = IoDriver::new(1024).unwrap();
         let signal_driver = SignalDriver::new(io_driver, &io_handle).unwrap();
         let handle = signal_driver.handle();
 

tokio/src/process/windows.rs

@@ -28,16 +28,18 @@ use std::os::windows::prelude::{AsRawHandle, IntoRawHandle, RawHandle};
 use std::pin::Pin;
 use std::process::Stdio;
 use std::process::{Child as StdChild, Command as StdCommand, ExitStatus};
-use std::ptr;
 use std::sync::Arc;
 use std::task::{Context, Poll};
-use winapi::shared::minwindef::{DWORD, FALSE};
-use winapi::um::handleapi::{DuplicateHandle, INVALID_HANDLE_VALUE};
-use winapi::um::processthreadsapi::GetCurrentProcess;
-use winapi::um::threadpoollegacyapiset::UnregisterWaitEx;
-use winapi::um::winbase::{RegisterWaitForSingleObject, INFINITE};
-use winapi::um::winnt::{
-    BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, PVOID, WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE,
+
+use windows_sys::{
+    Win32::Foundation::{
+        DuplicateHandle, BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, INVALID_HANDLE_VALUE,
+    },
+    Win32::System::Threading::{
+        GetCurrentProcess, RegisterWaitForSingleObject, UnregisterWaitEx, WT_EXECUTEINWAITTHREAD,
+        WT_EXECUTEONLYONCE,
+    },
+    Win32::System::WindowsProgramming::INFINITE,
 };
 
 #[must_use = "futures do nothing unless polled"]
@@ -119,11 +121,11 @@ impl Future for Child {
             }
             let (tx, rx) = oneshot::channel();
             let ptr = Box::into_raw(Box::new(Some(tx)));
-            let mut wait_object = ptr::null_mut();
+            let mut wait_object = 0;
             let rc = unsafe {
                 RegisterWaitForSingleObject(
                     &mut wait_object,
-                    inner.child.as_raw_handle(),
+                    inner.child.as_raw_handle() as _,
                     Some(callback),
                     ptr as *mut _,
                     INFINITE,
@@ -162,7 +164,7 @@ impl Drop for Waiting {
     }
 }
 
-unsafe extern "system" fn callback(ptr: PVOID, _timer_fired: BOOLEAN) {
+unsafe extern "system" fn callback(ptr: *mut std::ffi::c_void, _timer_fired: BOOLEAN) {
     let complete = &mut *(ptr as *mut Option<oneshot::Sender<()>>);
     let _ = complete.take().unwrap().send(());
 }
@@ -257,11 +259,11 @@ fn duplicate_handle<T: AsRawHandle>(io: &T) -> io::Result<StdFile> {
 
         let status = DuplicateHandle(
             cur_proc,
-            io.as_raw_handle(),
+            io.as_raw_handle() as _,
             cur_proc,
             &mut dup_handle,
-            0 as DWORD,
-            FALSE,
+            0,
+            0,
             DUPLICATE_SAME_ACCESS,
         );
 
@@ -269,6 +271,6 @@ fn duplicate_handle<T: AsRawHandle>(io: &T) -> io::Result<StdFile> {
             return Err(io::Error::last_os_error());
         }
 
-        Ok(StdFile::from_raw_handle(dup_handle))
+        Ok(StdFile::from_raw_handle(dup_handle as _))
     }
 }

tokio/src/runtime/blocking/mod.rs

@@ -17,8 +17,6 @@ cfg_trace! {
 mod schedule;
 mod shutdown;
 mod task;
-#[cfg(all(test, not(tokio_wasm)))]
-pub(crate) use schedule::NoopSchedule;
 pub(crate) use task::BlockingTask;
 
 use crate::runtime::Builder;

tokio/src/runtime/blocking/pool.rs

@@ -2,7 +2,7 @@
 
 use crate::loom::sync::{Arc, Condvar, Mutex};
 use crate::loom::thread;
-use crate::runtime::blocking::schedule::NoopSchedule;
+use crate::runtime::blocking::schedule::BlockingSchedule;
 use crate::runtime::blocking::{shutdown, BlockingTask};
 use crate::runtime::builder::ThreadNameFn;
 use crate::runtime::task::{self, JoinHandle};
@@ -120,7 +120,7 @@ struct Shared {
 }
 
 pub(crate) struct Task {
-    task: task::UnownedTask<NoopSchedule>,
+    task: task::UnownedTask<BlockingSchedule>,
     mandatory: Mandatory,
 }
 
@@ -151,7 +151,7 @@ impl From<SpawnError> for io::Error {
 }
 
 impl Task {
-    pub(crate) fn new(task: task::UnownedTask<NoopSchedule>, mandatory: Mandatory) -> Task {
+    pub(crate) fn new(task: task::UnownedTask<BlockingSchedule>, mandatory: Mandatory) -> Task {
         Task { task, mandatory }
     }
 
@@ -379,7 +379,8 @@ impl Spawner {
         #[cfg(not(all(tokio_unstable, feature = "tracing")))]
         let _ = name;
 
-        let (task, handle) = task::unowned(fut, NoopSchedule, id);
+        let (task, handle) = task::unowned(fut, BlockingSchedule::new(rt), id);
+
         let spawned = self.spawn_task(Task::new(task, is_mandatory), rt);
         (handle, spawned)
     }

tokio/src/runtime/blocking/schedule.rs

@@ -1,15 +1,52 @@
+#[cfg(feature = "test-util")]
+use crate::runtime::scheduler;
 use crate::runtime::task::{self, Task};
+use crate::runtime::Handle;
 
-/// `task::Schedule` implementation that does nothing. This is unique to the
-/// blocking scheduler as tasks scheduled are not really futures but blocking
-/// operations.
+/// `task::Schedule` implementation that does nothing (except some bookkeeping
+/// in test-util builds). This is unique to the blocking scheduler as tasks
+/// scheduled are not really futures but blocking operations.
 ///
 /// We avoid storing the task by forgetting it in `bind` and re-materializing it
-/// in `release.
-pub(crate) struct NoopSchedule;
+/// in `release`.
+pub(crate) struct BlockingSchedule {
+    #[cfg(feature = "test-util")]
+    handle: Handle,
+}
 
-impl task::Schedule for NoopSchedule {
+impl BlockingSchedule {
+    #[cfg_attr(not(feature = "test-util"), allow(unused_variables))]
+    pub(crate) fn new(handle: &Handle) -> Self {
+        #[cfg(feature = "test-util")]
+        {
+            match &handle.inner {
+                scheduler::Handle::CurrentThread(handle) => {
+                    handle.driver.clock.inhibit_auto_advance();
+                }
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThread(_) => {}
+            }
+        }
+        BlockingSchedule {
+            #[cfg(feature = "test-util")]
+            handle: handle.clone(),
+        }
+    }
+}
+
+impl task::Schedule for BlockingSchedule {
     fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> {
+        #[cfg(feature = "test-util")]
+        {
+            match &self.handle.inner {
+                scheduler::Handle::CurrentThread(handle) => {
+                    handle.driver.clock.allow_auto_advance();
+                    handle.driver.unpark();
+                }
+                #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+                scheduler::Handle::MultiThread(_) => {}
+            }
+        }
         None
     }
 

tokio/src/runtime/builder.rs

@@ -44,6 +44,7 @@ pub struct Builder {
 
     /// Whether or not to enable the I/O driver
     enable_io: bool,
+    nevents: usize,
 
     /// Whether or not to enable the time driver
     enable_time: bool,
@@ -181,6 +182,7 @@ cfg_unstable! {
 
 pub(crate) type ThreadNameFn = std::sync::Arc<dyn Fn() -> String + Send + Sync + 'static>;
 
+#[derive(Clone, Copy)]
 pub(crate) enum Kind {
     CurrentThread,
     #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
@@ -228,6 +230,7 @@ impl Builder {
 
             // I/O defaults to "off"
             enable_io: false,
+            nevents: 1024,
 
             // Time defaults to "off"
             enable_time: false,
@@ -235,6 +238,7 @@ impl Builder {
             // The clock starts not-paused
             start_paused: false,
 
+            // Read from environment variable first in multi-threaded mode.
             // Default to lazy auto-detection (one thread per CPU core)
             worker_threads: None,
 
@@ -302,6 +306,8 @@ impl Builder {
     /// This can be any number above 0 though it is advised to keep this value
     /// on the smaller side.
     ///
+    /// This will override the value read from environment variable `TOKIO_WORKER_THREADS`.
+    ///
     /// # Default
     ///
     /// The default value is the number of cores available to the system.
@@ -647,6 +653,7 @@ impl Builder {
             enable_io: self.enable_io,
             enable_time: self.enable_time,
             start_paused: self.start_paused,
+            nevents: self.nevents,
         }
     }
 
@@ -818,7 +825,7 @@ impl Builder {
         ///
         /// This configuration option is considered a workaround for the LIFO
         /// slot not being stealable. When the slot becomes stealable, we will
-        /// revisit whther or not this option is necessary. See
+        /// revisit whether or not this option is necessary. See
         /// tokio-rs/tokio#4941.
         ///
         /// # Examples
@@ -938,6 +945,25 @@ cfg_io_driver! {
             self.enable_io = true;
             self
         }
+
+        /// Enables the I/O driver and configures the max number of events to be
+        /// processed per tick.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime;
+        ///
+        /// let rt = runtime::Builder::new_current_thread()
+        ///     .enable_io()
+        ///     .max_io_events_per_tick(1024)
+        ///     .build()
+        ///     .unwrap();
+        /// ```
+        pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self {
+            self.nevents = capacity;
+            self
+        }
     }
 }
 

tokio/src/runtime/context.rs

@@ -7,8 +7,7 @@ use std::cell::Cell;
 use crate::util::rand::{FastRand, RngSeed};
 
 cfg_rt! {
-    use crate::runtime::scheduler;
-    use crate::runtime::task::Id;
+    use crate::runtime::{scheduler, task::Id, Defer};
 
     use std::cell::RefCell;
     use std::marker::PhantomData;
@@ -19,6 +18,7 @@ struct Context {
     /// Handle to the runtime scheduler running on the current thread.
     #[cfg(feature = "rt")]
     handle: RefCell<Option<scheduler::Handle>>,
+
     #[cfg(feature = "rt")]
     current_task_id: Cell<Option<Id>>,
 
@@ -30,6 +30,11 @@ struct Context {
     #[cfg(feature = "rt")]
     runtime: Cell<EnterRuntime>,
 
+    /// Yielded task wakers are stored here and notified after resource drivers
+    /// are polled.
+    #[cfg(feature = "rt")]
+    defer: RefCell<Option<Defer>>,
+
     #[cfg(any(feature = "rt", feature = "macros"))]
     rng: FastRand,
 
@@ -56,6 +61,9 @@ tokio_thread_local! {
             #[cfg(feature = "rt")]
             runtime: Cell::new(EnterRuntime::NotEntered),
 
+            #[cfg(feature = "rt")]
+            defer: RefCell::new(None),
+
             #[cfg(any(feature = "rt", feature = "macros"))]
             rng: FastRand::new(RngSeed::new()),
 
@@ -99,9 +107,17 @@ cfg_rt! {
     /// Guard tracking that a caller has entered a runtime context.
     #[must_use]
     pub(crate) struct EnterRuntimeGuard {
+        /// Tracks that the current thread has entered a blocking function call.
         pub(crate) blocking: BlockingRegionGuard,
+
         #[allow(dead_code)] // Only tracking the guard.
         pub(crate) handle: SetCurrentGuard,
+
+        /// If true, then this is the root runtime guard. It is possible to nest
+        /// runtime guards by using `block_in_place` between the calls. We need
+        /// to track the root guard as this is the guard responsible for freeing
+        /// the deferred task queue.
+        is_root: bool,
     }
 
     /// Guard tracking that a caller has entered a blocking region.
@@ -159,10 +175,23 @@ cfg_rt! {
             if c.runtime.get().is_entered() {
                 None
             } else {
+                // Set the entered flag
                 c.runtime.set(EnterRuntime::Entered { allow_block_in_place });
+
+                // Initialize queue to track yielded tasks
+                let mut defer = c.defer.borrow_mut();
+
+                let is_root = if defer.is_none() {
+                    *defer = Some(Defer::new());
+                    true
+                } else {
+                    false
+                };
+
                 Some(EnterRuntimeGuard {
                     blocking: BlockingRegionGuard::new(),
                     handle: c.set_current(handle),
+                    is_root,
                 })
             }
         })
@@ -201,6 +230,13 @@ cfg_rt! {
         DisallowBlockInPlaceGuard(reset)
     }
 
+    pub(crate) fn with_defer<R>(f: impl FnOnce(&mut Defer) -> R) -> Option<R> {
+        CONTEXT.with(|c| {
+            let mut defer = c.defer.borrow_mut();
+            defer.as_mut().map(f)
+        })
+    }
+
     impl Context {
         fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard {
             let rng_seed = handle.seed_generator().next_seed();
@@ -235,6 +271,10 @@ cfg_rt! {
             CONTEXT.with(|c| {
                 assert!(c.runtime.get().is_entered());
                 c.runtime.set(EnterRuntime::NotEntered);
+
+                if self.is_root {
+                    *c.defer.borrow_mut() = None;
+                }
             });
         }
     }
@@ -286,6 +326,10 @@ cfg_rt! {
                     return Err(());
                 }
 
+                // Wake any yielded tasks before parking in order to avoid
+                // blocking.
+                with_defer(|defer| defer.wake());
+
                 park.park_timeout(when - now);
             }
         }

tokio/src/runtime/defer.rs

@@ -0,0 +1,27 @@
+use std::task::Waker;
+
+pub(crate) struct Defer {
+    deferred: Vec<Waker>,
+}
+
+impl Defer {
+    pub(crate) fn new() -> Defer {
+        Defer {
+            deferred: Default::default(),
+        }
+    }
+
+    pub(crate) fn defer(&mut self, waker: Waker) {
+        self.deferred.push(waker);
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.deferred.is_empty()
+    }
+
+    pub(crate) fn wake(&mut self) {
+        for waker in self.deferred.drain(..) {
+            waker.wake();
+        }
+    }
+}

tokio/src/runtime/driver.rs

@@ -36,11 +36,12 @@ pub(crate) struct Cfg {
     pub(crate) enable_time: bool,
     pub(crate) enable_pause_time: bool,
     pub(crate) start_paused: bool,
+    pub(crate) nevents: usize,
 }
 
 impl Driver {
     pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Handle)> {
-        let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io)?;
+        let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io, cfg.nevents)?;
 
         let clock = create_clock(cfg.enable_pause_time, cfg.start_paused);
 
@@ -135,12 +136,12 @@ cfg_io_driver! {
         Disabled(UnparkThread),
     }
 
-    fn create_io_stack(enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
+    fn create_io_stack(enabled: bool, nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
         #[cfg(loom)]
         assert!(!enabled);
 
         let ret = if enabled {
-            let (io_driver, io_handle) = crate::runtime::io::Driver::new()?;
+            let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents)?;
 
             let (signal_driver, signal_handle) = create_signal_driver(io_driver, &io_handle)?;
             let process_driver = create_process_driver(signal_driver);
@@ -201,7 +202,7 @@ cfg_not_io_driver! {
     #[derive(Debug)]
     pub(crate) struct IoStack(ParkThread);
 
-    fn create_io_stack(_enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
+    fn create_io_stack(_enabled: bool, _nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
         let park_thread = ParkThread::new();
         let unpark_thread = park_thread.unpark();
         Ok((IoStack(park_thread), unpark_thread, Default::default()))

tokio/src/runtime/io/mod.rs

@@ -104,7 +104,7 @@ fn _assert_kinds() {
 impl Driver {
     /// Creates a new event loop, returning any error that happened during the
     /// creation.
-    pub(crate) fn new() -> io::Result<(Driver, Handle)> {
+    pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> {
         let poll = mio::Poll::new()?;
         #[cfg(not(tokio_wasi))]
         let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
@@ -116,7 +116,7 @@ impl Driver {
         let driver = Driver {
             tick: 0,
             signal_ready: false,
-            events: mio::Events::with_capacity(1024),
+            events: mio::Events::with_capacity(nevents),
             poll,
             resources: slab,
         };

tokio/src/runtime/io/scheduled_io.rs

@@ -510,14 +510,24 @@ cfg_io_readiness! {
                         drop(waiters);
                     }
                     State::Done => {
-                        let tick = TICK.unpack(scheduled_io.readiness.load(Acquire)) as u8;
-
                         // Safety: State::Done means it is no longer shared
                         let w = unsafe { &mut *waiter.get() };
 
+                        let curr = scheduled_io.readiness.load(Acquire);
+
+                        // The returned tick might be newer than the event
+                        // which notified our waker. This is ok because the future
+                        // still didn't return `Poll::Ready`.
+                        let tick = TICK.unpack(curr) as u8;
+
+                        // The readiness state could have been cleared in the meantime,
+                        // but we allow the returned ready set to be empty.
+                        let curr_ready = Ready::from_usize(READINESS.unpack(curr));
+                        let ready = curr_ready.intersection(w.interest);
+
                         return Poll::Ready(ReadyEvent {
                             tick,
-                            ready: Ready::from_interest(w.interest),
+                            ready,
                         });
                     }
                 }

tokio/src/runtime/mod.rs

@@ -228,6 +228,9 @@ cfg_rt! {
         pub use crate::util::rand::RngSeed;
     }
 
+    mod defer;
+    pub(crate) use defer::Defer;
+
     mod handle;
     pub use handle::{EnterGuard, Handle, TryCurrentError};
 

tokio/src/runtime/park.rs

@@ -32,6 +32,12 @@ tokio_thread_local! {
     static CURRENT_PARKER: ParkThread = ParkThread::new();
 }
 
+// Bit of a hack, but it is only for loom
+#[cfg(loom)]
+tokio_thread_local! {
+    static CURRENT_THREAD_PARK_COUNT: AtomicUsize = AtomicUsize::new(0);
+}
+
 // ==== impl ParkThread ====
 
 impl ParkThread {
@@ -51,10 +57,15 @@ impl ParkThread {
     }
 
     pub(crate) fn park(&mut self) {
+        #[cfg(loom)]
+        CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
         self.inner.park();
     }
 
     pub(crate) fn park_timeout(&mut self, duration: Duration) {
+        #[cfg(loom)]
+        CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
+
         // Wasm doesn't have threads, so just sleep.
         #[cfg(not(tokio_wasm))]
         self.inner.park_timeout(duration);
@@ -273,6 +284,11 @@ impl CachedParkThread {
                 return Ok(v);
             }
 
+            // Wake any yielded tasks before parking in order to avoid
+            // blocking.
+            #[cfg(feature = "rt")]
+            crate::runtime::context::with_defer(|defer| defer.wake());
+
             self.park();
         }
     }
@@ -330,3 +346,8 @@ unsafe fn wake_by_ref(raw: *const ()) {
     // We don't actually own a reference to the unparker
     mem::forget(unparker);
 }
+
+#[cfg(loom)]
+pub(crate) fn current_thread_park_count() -> usize {
+    CURRENT_THREAD_PARK_COUNT.with(|count| count.load(SeqCst))
+}

tokio/src/runtime/runtime.rs

@@ -138,6 +138,9 @@ impl Runtime {
     /// The returned handle can be used to spawn tasks that run on this runtime, and can
     /// be cloned to allow moving the `Handle` to other threads.
     ///
+    /// Calling [`Handle::block_on`] on a handle to a `current_thread` runtime is error-prone.
+    /// Refer to the documentation of [`Handle::block_on`] for more.
+    ///
     /// # Examples
     ///
     /// ```

tokio/src/runtime/scheduler/current_thread.rs

@@ -3,7 +3,7 @@ use crate::loom::sync::atomic::AtomicBool;
 use crate::loom::sync::{Arc, Mutex};
 use crate::runtime::driver::{self, Driver};
 use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task};
-use crate::runtime::{blocking, scheduler, Config};
+use crate::runtime::{blocking, context, scheduler, Config};
 use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
 use crate::sync::notify::Notify;
 use crate::util::atomic_cell::AtomicCell;
@@ -267,6 +267,14 @@ impl Core {
     }
 }
 
+fn did_defer_tasks() -> bool {
+    context::with_defer(|deferred| !deferred.is_empty()).unwrap()
+}
+
+fn wake_deferred_tasks() {
+    context::with_defer(|deferred| deferred.wake());
+}
+
 // ===== impl Context =====
 
 impl Context {
@@ -299,6 +307,7 @@ impl Context {
 
             let (c, _) = self.enter(core, || {
                 driver.park(&handle.driver);
+                wake_deferred_tasks();
             });
 
             core = c;
@@ -324,6 +333,7 @@ impl Context {
         core.metrics.submit(&handle.shared.worker_metrics);
         let (mut core, _) = self.enter(core, || {
             driver.park_timeout(&handle.driver, Duration::from_millis(0));
+            wake_deferred_tasks();
         });
 
         core.driver = Some(driver);
@@ -557,7 +567,11 @@ impl CoreGuard<'_> {
                     let task = match entry {
                         Some(entry) => entry,
                         None => {
-                            core = context.park(core, handle);
+                            core = if did_defer_tasks() {
+                                context.park_yield(core, handle)
+                            } else {
+                                context.park(core, handle)
+                            };
 
                             // Try polling the `block_on` future next
                             continue 'outer;

tokio/src/runtime/scheduler/multi_thread/queue.rs

@@ -263,7 +263,7 @@ impl<T> Local<T> {
         // safety: The CAS above ensures that no consumer will look at these
         // values again, and we are the only producer.
         let batch_iter = BatchTaskIter {
-            buffer: &*self.inner.buffer,
+            buffer: &self.inner.buffer,
             head: head as UnsignedLong,
             i: 0,
         };

tokio/src/runtime/scheduler/multi_thread/worker.rs

@@ -368,6 +368,22 @@ impl Launch {
 }
 
 fn run(worker: Arc<Worker>) {
+    struct AbortOnPanic;
+
+    impl Drop for AbortOnPanic {
+        fn drop(&mut self) {
+            if std::thread::panicking() {
+                eprintln!("worker thread panicking; aborting process");
+                std::process::abort();
+            }
+        }
+    }
+
+    // Catching panics on worker threads in tests is quite tricky. Instead, when
+    // debug assertions are enabled, we just abort the process.
+    #[cfg(debug_assertions)]
+    let _abort_on_panic = AbortOnPanic;
+
     // Acquire a core. If this fails, then another thread is running this
     // worker and there is nothing further to do.
     let core = match worker.core.take() {
@@ -388,6 +404,11 @@ fn run(worker: Arc<Worker>) {
         // This should always be an error. It only returns a `Result` to support
         // using `?` to short circuit.
         assert!(cx.run(core).is_err());
+
+        // Check if there are any deferred tasks to notify. This can happen when
+        // the worker core is lost due to `block_in_place()` being called from
+        // within the task.
+        wake_deferred_tasks();
     });
 }
 
@@ -412,7 +433,11 @@ impl Context {
                 core = self.run_task(task, core)?;
             } else {
                 // Wait for work
-                core = self.park(core);
+                core = if did_defer_tasks() {
+                    self.park_timeout(core, Some(Duration::from_millis(0)))
+                } else {
+                    self.park(core)
+                };
             }
         }
 
@@ -535,6 +560,8 @@ impl Context {
             park.park(&self.worker.handle.driver);
         }
 
+        wake_deferred_tasks();
+
         // Remove `core` from context
         core = self.core.borrow_mut().take().expect("core missing");
 
@@ -853,6 +880,14 @@ impl Handle {
     }
 }
 
+fn did_defer_tasks() -> bool {
+    context::with_defer(|deferred| !deferred.is_empty()).unwrap()
+}
+
+fn wake_deferred_tasks() {
+    context::with_defer(|deferred| deferred.wake());
+}
+
 cfg_metrics! {
     impl Shared {
         pub(super) fn injection_queue_depth(&self) -> usize {

tokio/src/runtime/signal/mod.rs

@@ -103,7 +103,7 @@ impl Driver {
     }
 
     fn process(&mut self) {
-        // If the signal pipe has not recieved a readiness event, then there is
+        // If the signal pipe has not received a readiness event, then there is
         // nothing else to do.
         if !self.io.consume_signal_ready() {
             return;

tokio/src/runtime/task/abort.rs

@@ -1,4 +1,4 @@
-use crate::runtime::task::{Id, RawTask};
+use crate::runtime::task::{Header, RawTask};
 use std::fmt;
 use std::panic::{RefUnwindSafe, UnwindSafe};
 
@@ -14,13 +14,12 @@ use std::panic::{RefUnwindSafe, UnwindSafe};
 /// [`JoinHandle`]: crate::task::JoinHandle
 #[cfg_attr(docsrs, doc(cfg(feature = "rt")))]
 pub struct AbortHandle {
-    raw: Option<RawTask>,
-    id: Id,
+    raw: RawTask,
 }
 
 impl AbortHandle {
-    pub(super) fn new(raw: Option<RawTask>, id: Id) -> Self {
-        Self { raw, id }
+    pub(super) fn new(raw: RawTask) -> Self {
+        Self { raw }
     }
 
     /// Abort the task associated with the handle.
@@ -35,9 +34,7 @@ impl AbortHandle {
     /// [cancelled]: method@super::error::JoinError::is_cancelled
     /// [`JoinHandle::abort`]: method@super::JoinHandle::abort
     pub fn abort(&self) {
-        if let Some(ref raw) = self.raw {
-            raw.remote_abort();
-        }
+        self.raw.remote_abort();
     }
 
     /// Checks if the task associated with this `AbortHandle` has finished.
@@ -47,12 +44,8 @@ impl AbortHandle {
     /// some time, and this method does not return `true` until it has
     /// completed.
     pub fn is_finished(&self) -> bool {
-        if let Some(raw) = self.raw {
-            let state = raw.header().state.load();
-            state.is_complete()
-        } else {
-            true
-        }
+        let state = self.raw.state().load();
+        state.is_complete()
     }
 
     /// Returns a [task ID] that uniquely identifies this task relative to other
@@ -67,7 +60,8 @@ impl AbortHandle {
     #[cfg(tokio_unstable)]
     #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
     pub fn id(&self) -> super::Id {
-        self.id
+        // Safety: The header pointer is valid.
+        unsafe { Header::get_id(self.raw.header_ptr()) }
     }
 }
 
@@ -79,16 +73,15 @@ impl RefUnwindSafe for AbortHandle {}
 
 impl fmt::Debug for AbortHandle {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt.debug_struct("AbortHandle")
-            .field("id", &self.id)
-            .finish()
+        // Safety: The header pointer is valid.
+        let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) };
+        let id = unsafe { id_ptr.as_ref() };
+        fmt.debug_struct("AbortHandle").field("id", id).finish()
     }
 }
 
 impl Drop for AbortHandle {
     fn drop(&mut self) {
-        if let Some(raw) = self.raw.take() {
-            raw.drop_abort_handle();
-        }
+        self.raw.drop_abort_handle();
     }
 }

tokio/src/runtime/task/core.rs

@@ -25,6 +25,9 @@ use std::task::{Context, Poll, Waker};
 ///
 /// It is critical for `Header` to be the first field as the task structure will
 /// be referenced by both *mut Cell and *mut Header.
+///
+/// Any changes to the layout of this struct _must_ also be reflected in the
+/// const fns in raw.rs.
 #[repr(C)]
 pub(super) struct Cell<T: Future, S> {
     /// Hot task state data
@@ -44,15 +47,19 @@ pub(super) struct CoreStage<T: Future> {
 /// The core of the task.
 ///
 /// Holds the future or output, depending on the stage of execution.
+///
+/// Any changes to the layout of this struct _must_ also be reflected in the
+/// const fns in raw.rs.
+#[repr(C)]
 pub(super) struct Core<T: Future, S> {
     /// Scheduler used to drive this future.
     pub(super) scheduler: S,
 
-    /// Either the future or the output.
-    pub(super) stage: CoreStage<T>,
-
     /// The task's ID, used for populating `JoinError`s.
     pub(super) task_id: Id,
+
+    /// Either the future or the output.
+    pub(super) stage: CoreStage<T>,
 }
 
 /// Crate public as this is also needed by the pool.
@@ -82,7 +89,7 @@ pub(crate) struct Header {
 
     /// The tracing ID for this instrumented task.
     #[cfg(all(tokio_unstable, feature = "tracing"))]
-    pub(super) id: Option<tracing::Id>,
+    pub(super) tracing_id: Option<tracing::Id>,
 }
 
 unsafe impl Send for Header {}
@@ -117,7 +124,7 @@ impl<T: Future, S: Schedule> Cell<T, S> {
     /// structures.
     pub(super) fn new(future: T, scheduler: S, state: State, task_id: Id) -> Box<Cell<T, S>> {
         #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let id = future.id();
+        let tracing_id = future.id();
         let result = Box::new(Cell {
             header: Header {
                 state,
@@ -125,7 +132,7 @@ impl<T: Future, S: Schedule> Cell<T, S> {
                 vtable: raw::vtable::<T, S>(),
                 owner_id: UnsafeCell::new(0),
                 #[cfg(all(tokio_unstable, feature = "tracing"))]
-                id,
+                tracing_id,
             },
             core: Core {
                 scheduler,
@@ -144,8 +151,16 @@ impl<T: Future, S: Schedule> Cell<T, S> {
         {
             let trailer_addr = (&result.trailer) as *const Trailer as usize;
             let trailer_ptr = unsafe { Header::get_trailer(NonNull::from(&result.header)) };
-
             assert_eq!(trailer_addr, trailer_ptr.as_ptr() as usize);
+
+            let scheduler_addr = (&result.core.scheduler) as *const S as usize;
+            let scheduler_ptr =
+                unsafe { Header::get_scheduler::<S>(NonNull::from(&result.header)) };
+            assert_eq!(scheduler_addr, scheduler_ptr.as_ptr() as usize);
+
+            let id_addr = (&result.core.task_id) as *const Id as usize;
+            let id_ptr = unsafe { Header::get_id_ptr(NonNull::from(&result.header)) };
+            assert_eq!(id_addr, id_ptr.as_ptr() as usize);
         }
 
         result
@@ -295,6 +310,51 @@ impl Header {
         let trailer = me.as_ptr().cast::<u8>().add(offset).cast::<Trailer>();
         NonNull::new_unchecked(trailer)
     }
+
+    /// Gets a pointer to the scheduler of the task containing this `Header`.
+    ///
+    /// # Safety
+    ///
+    /// The provided raw pointer must point at the header of a task.
+    ///
+    /// The generic type S must be set to the correct scheduler type for this
+    /// task.
+    pub(super) unsafe fn get_scheduler<S>(me: NonNull<Header>) -> NonNull<S> {
+        let offset = me.as_ref().vtable.scheduler_offset;
+        let scheduler = me.as_ptr().cast::<u8>().add(offset).cast::<S>();
+        NonNull::new_unchecked(scheduler)
+    }
+
+    /// Gets a pointer to the id of the task containing this `Header`.
+    ///
+    /// # Safety
+    ///
+    /// The provided raw pointer must point at the header of a task.
+    pub(super) unsafe fn get_id_ptr(me: NonNull<Header>) -> NonNull<Id> {
+        let offset = me.as_ref().vtable.id_offset;
+        let id = me.as_ptr().cast::<u8>().add(offset).cast::<Id>();
+        NonNull::new_unchecked(id)
+    }
+
+    /// Gets the id of the task containing this `Header`.
+    ///
+    /// # Safety
+    ///
+    /// The provided raw pointer must point at the header of a task.
+    pub(super) unsafe fn get_id(me: NonNull<Header>) -> Id {
+        let ptr = Header::get_id_ptr(me).as_ptr();
+        *ptr
+    }
+
+    /// Gets the tracing id of the task containing this `Header`.
+    ///
+    /// # Safety
+    ///
+    /// The provided raw pointer must point at the header of a task.
+    #[cfg(all(tokio_unstable, feature = "tracing"))]
+    pub(super) unsafe fn get_tracing_id(me: &NonNull<Header>) -> Option<&tracing::Id> {
+        me.as_ref().tracing_id.as_ref()
+    }
 }
 
 impl Trailer {

tokio/src/runtime/task/harness.rs

@@ -2,7 +2,7 @@ use crate::future::Future;
 use crate::runtime::task::core::{Cell, Core, Header, Trailer};
 use crate::runtime::task::state::{Snapshot, State};
 use crate::runtime::task::waker::waker_ref;
-use crate::runtime::task::{JoinError, Notified, Schedule, Task};
+use crate::runtime::task::{JoinError, Notified, RawTask, Schedule, Task};
 
 use std::mem;
 use std::mem::ManuallyDrop;
@@ -47,11 +47,102 @@ where
     }
 }
 
+/// Task operations that can be implemented without being generic over the
+/// scheduler or task. Only one version of these methods should exist in the
+/// final binary.
+impl RawTask {
+    pub(super) fn drop_reference(self) {
+        if self.state().ref_dec() {
+            self.dealloc();
+        }
+    }
+
+    /// This call consumes a ref-count and notifies the task. This will create a
+    /// new Notified and submit it if necessary.
+    ///
+    /// The caller does not need to hold a ref-count besides the one that was
+    /// passed to this call.
+    pub(super) fn wake_by_val(&self) {
+        use super::state::TransitionToNotifiedByVal;
+
+        match self.state().transition_to_notified_by_val() {
+            TransitionToNotifiedByVal::Submit => {
+                // The caller has given us a ref-count, and the transition has
+                // created a new ref-count, so we now hold two. We turn the new
+                // ref-count Notified and pass it to the call to `schedule`.
+                //
+                // The old ref-count is retained for now to ensure that the task
+                // is not dropped during the call to `schedule` if the call
+                // drops the task it was given.
+                self.schedule();
+
+                // Now that we have completed the call to schedule, we can
+                // release our ref-count.
+                self.drop_reference();
+            }
+            TransitionToNotifiedByVal::Dealloc => {
+                self.dealloc();
+            }
+            TransitionToNotifiedByVal::DoNothing => {}
+        }
+    }
+
+    /// This call notifies the task. It will not consume any ref-counts, but the
+    /// caller should hold a ref-count.  This will create a new Notified and
+    /// submit it if necessary.
+    pub(super) fn wake_by_ref(&self) {
+        use super::state::TransitionToNotifiedByRef;
+
+        match self.state().transition_to_notified_by_ref() {
+            TransitionToNotifiedByRef::Submit => {
+                // The transition above incremented the ref-count for a new task
+                // and the caller also holds a ref-count. The caller's ref-count
+                // ensures that the task is not destroyed even if the new task
+                // is dropped before `schedule` returns.
+                self.schedule();
+            }
+            TransitionToNotifiedByRef::DoNothing => {}
+        }
+    }
+
+    /// Remotely aborts the task.
+    ///
+    /// The caller should hold a ref-count, but we do not consume it.
+    ///
+    /// This is similar to `shutdown` except that it asks the runtime to perform
+    /// the shutdown. This is necessary to avoid the shutdown happening in the
+    /// wrong thread for non-Send tasks.
+    pub(super) fn remote_abort(&self) {
+        if self.state().transition_to_notified_and_cancel() {
+            // The transition has created a new ref-count, which we turn into
+            // a Notified and pass to the task.
+            //
+            // Since the caller holds a ref-count, the task cannot be destroyed
+            // before the call to `schedule` returns even if the call drops the
+            // `Notified` internally.
+            self.schedule();
+        }
+    }
+
+    /// Try to set the waker notified when the task is complete. Returns true if
+    /// the task has already completed. If this call returns false, then the
+    /// waker will not be notified.
+    pub(super) fn try_set_join_waker(&self, waker: &Waker) -> bool {
+        can_read_output(self.header(), self.trailer(), waker)
+    }
+}
+
 impl<T, S> Harness<T, S>
 where
     T: Future,
     S: Schedule,
 {
+    pub(super) fn drop_reference(self) {
+        if self.state().ref_dec() {
+            self.dealloc();
+        }
+    }
+
     /// Polls the inner future. A ref-count is consumed.
     ///
     /// All necessary state checks and transitions are performed.
@@ -103,7 +194,7 @@ where
             TransitionToRunning::Success => {
                 let header_ptr = self.header_ptr();
                 let waker_ref = waker_ref::<T, S>(&header_ptr);
-                let cx = Context::from_waker(&*waker_ref);
+                let cx = Context::from_waker(&waker_ref);
                 let res = poll_future(self.core(), cx);
 
                 if res == Poll::Ready(()) {
@@ -185,13 +276,6 @@ where
         }
     }
 
-    /// Try to set the waker notified when the task is complete. Returns true if
-    /// the task has already completed. If this call returns false, then the
-    /// waker will not be notified.
-    pub(super) fn try_set_join_waker(self, waker: &Waker) -> bool {
-        can_read_output(self.header(), self.trailer(), waker)
-    }
-
     pub(super) fn drop_join_handle_slow(self) {
         // Try to unset `JOIN_INTEREST`. This must be done as a first step in
         // case the task concurrently completed.
@@ -214,92 +298,6 @@ where
         self.drop_reference();
     }
 
-    /// Remotely aborts the task.
-    ///
-    /// The caller should hold a ref-count, but we do not consume it.
-    ///
-    /// This is similar to `shutdown` except that it asks the runtime to perform
-    /// the shutdown. This is necessary to avoid the shutdown happening in the
-    /// wrong thread for non-Send tasks.
-    pub(super) fn remote_abort(self) {
-        if self.state().transition_to_notified_and_cancel() {
-            // The transition has created a new ref-count, which we turn into
-            // a Notified and pass to the task.
-            //
-            // Since the caller holds a ref-count, the task cannot be destroyed
-            // before the call to `schedule` returns even if the call drops the
-            // `Notified` internally.
-            self.core()
-                .scheduler
-                .schedule(Notified(self.get_new_task()));
-        }
-    }
-
-    // ===== waker behavior =====
-
-    /// This call consumes a ref-count and notifies the task. This will create a
-    /// new Notified and submit it if necessary.
-    ///
-    /// The caller does not need to hold a ref-count besides the one that was
-    /// passed to this call.
-    pub(super) fn wake_by_val(self) {
-        use super::state::TransitionToNotifiedByVal;
-
-        match self.state().transition_to_notified_by_val() {
-            TransitionToNotifiedByVal::Submit => {
-                // The caller has given us a ref-count, and the transition has
-                // created a new ref-count, so we now hold two. We turn the new
-                // ref-count Notified and pass it to the call to `schedule`.
-                //
-                // The old ref-count is retained for now to ensure that the task
-                // is not dropped during the call to `schedule` if the call
-                // drops the task it was given.
-                self.core()
-                    .scheduler
-                    .schedule(Notified(self.get_new_task()));
-
-                // Now that we have completed the call to schedule, we can
-                // release our ref-count.
-                self.drop_reference();
-            }
-            TransitionToNotifiedByVal::Dealloc => {
-                self.dealloc();
-            }
-            TransitionToNotifiedByVal::DoNothing => {}
-        }
-    }
-
-    /// This call notifies the task. It will not consume any ref-counts, but the
-    /// caller should hold a ref-count.  This will create a new Notified and
-    /// submit it if necessary.
-    pub(super) fn wake_by_ref(&self) {
-        use super::state::TransitionToNotifiedByRef;
-
-        match self.state().transition_to_notified_by_ref() {
-            TransitionToNotifiedByRef::Submit => {
-                // The transition above incremented the ref-count for a new task
-                // and the caller also holds a ref-count. The caller's ref-count
-                // ensures that the task is not destroyed even if the new task
-                // is dropped before `schedule` returns.
-                self.core()
-                    .scheduler
-                    .schedule(Notified(self.get_new_task()));
-            }
-            TransitionToNotifiedByRef::DoNothing => {}
-        }
-    }
-
-    pub(super) fn drop_reference(self) {
-        if self.state().ref_dec() {
-            self.dealloc();
-        }
-    }
-
-    #[cfg(all(tokio_unstable, feature = "tracing"))]
-    pub(super) fn id(&self) -> Option<&tracing::Id> {
-        self.header().id.as_ref()
-    }
-
     // ====== internal ======
 
     /// Completes the task. This method assumes that the state is RUNNING.
@@ -317,9 +315,10 @@ where
                 // this task. It is our responsibility to drop the
                 // output.
                 self.core().drop_future_or_output();
-            } else if snapshot.has_join_waker() {
-                // Notify the join handle. The previous transition obtains the
-                // lock on the waker cell.
+            } else if snapshot.is_join_waker_set() {
+                // Notify the waker. Reading the waker field is safe per rule 4
+                // in task/mod.rs, since the JOIN_WAKER bit is set and the call
+                // to transition_to_complete() above set the COMPLETE bit.
                 self.trailer().wake_join();
             }
         }));
@@ -369,36 +368,30 @@ fn can_read_output(header: &Header, trailer: &Trailer, waker: &Waker) -> bool {
     debug_assert!(snapshot.is_join_interested());
 
     if !snapshot.is_complete() {
-        // The waker must be stored in the task struct.
-        let res = if snapshot.has_join_waker() {
-            // There already is a waker stored in the struct. If it matches
-            // the provided waker, then there is no further work to do.
-            // Otherwise, the waker must be swapped.
-            let will_wake = unsafe {
-                // Safety: when `JOIN_INTEREST` is set, only `JOIN_HANDLE`
-                // may mutate the `waker` field.
-                trailer.will_wake(waker)
-            };
+        // If the task is not complete, try storing the provided waker in the
+        // task's waker field.
 
-            if will_wake {
-                // The task is not complete **and** the waker is up to date,
-                // there is nothing further that needs to be done.
+        let res = if snapshot.is_join_waker_set() {
+            // If JOIN_WAKER is set, then JoinHandle has previously stored a
+            // waker in the waker field per step (iii) of rule 5 in task/mod.rs.
+
+            // Optimization: if the stored waker and the provided waker wake the
+            // same task, then return without touching the waker field. (Reading
+            // the waker field below is safe per rule 3 in task/mod.rs.)
+            if unsafe { trailer.will_wake(waker) } {
                 return false;
             }
 
-            // Unset the `JOIN_WAKER` to gain mutable access to the `waker`
-            // field then update the field with the new join worker.
-            //
-            // This requires two atomic operations, unsetting the bit and
-            // then resetting it. If the task transitions to complete
-            // concurrently to either one of those operations, then setting
-            // the join waker fails and we proceed to reading the task
-            // output.
+            // Otherwise swap the stored waker with the provided waker by
+            // following the rule 5 in task/mod.rs.
             header
                 .state
                 .unset_waker()
                 .and_then(|snapshot| set_join_waker(header, trailer, waker.clone(), snapshot))
         } else {
+            // If JOIN_WAKER is unset, then JoinHandle has mutable access to the
+            // waker field per rule 2 in task/mod.rs; therefore, skip step (i)
+            // of rule 5 and try to store the provided waker in the waker field.
             set_join_waker(header, trailer, waker.clone(), snapshot)
         };
 
@@ -419,7 +412,7 @@ fn set_join_waker(
     snapshot: Snapshot,
 ) -> Result<Snapshot, Snapshot> {
     assert!(snapshot.is_join_interested());
-    assert!(!snapshot.has_join_waker());
+    assert!(!snapshot.is_join_waker_set());
 
     // Safety: Only the `JoinHandle` may set the `waker` field. When
     // `JOIN_INTEREST` is **not** set, nothing else will touch the field.

tokio/src/runtime/task/join.rs

@@ -1,4 +1,4 @@
-use crate::runtime::task::{Id, RawTask};
+use crate::runtime::task::{Header, RawTask};
 
 use std::fmt;
 use std::future::Future;
@@ -154,8 +154,7 @@ cfg_rt! {
     /// [`std::thread::JoinHandle`]: std::thread::JoinHandle
     /// [`JoinError`]: crate::task::JoinError
     pub struct JoinHandle<T> {
-        raw: Option<RawTask>,
-        id: Id,
+        raw: RawTask,
         _p: PhantomData<T>,
     }
 }
@@ -167,10 +166,9 @@ impl<T> UnwindSafe for JoinHandle<T> {}
 impl<T> RefUnwindSafe for JoinHandle<T> {}
 
 impl<T> JoinHandle<T> {
-    pub(super) fn new(raw: RawTask, id: Id) -> JoinHandle<T> {
+    pub(super) fn new(raw: RawTask) -> JoinHandle<T> {
         JoinHandle {
-            raw: Some(raw),
-            id,
+            raw,
             _p: PhantomData,
         }
     }
@@ -209,9 +207,7 @@ impl<T> JoinHandle<T> {
     /// ```
     /// [cancelled]: method@super::error::JoinError::is_cancelled
     pub fn abort(&self) {
-        if let Some(raw) = self.raw {
-            raw.remote_abort();
-        }
+        self.raw.remote_abort();
     }
 
     /// Checks if the task associated with this `JoinHandle` has finished.
@@ -243,31 +239,22 @@ impl<T> JoinHandle<T> {
     /// ```
     /// [`abort`]: method@JoinHandle::abort
     pub fn is_finished(&self) -> bool {
-        if let Some(raw) = self.raw {
-            let state = raw.header().state.load();
-            state.is_complete()
-        } else {
-            true
-        }
+        let state = self.raw.header().state.load();
+        state.is_complete()
     }
 
     /// Set the waker that is notified when the task completes.
     pub(crate) fn set_join_waker(&mut self, waker: &Waker) {
-        if let Some(raw) = self.raw {
-            if raw.try_set_join_waker(waker) {
-                // In this case the task has already completed. We wake the waker immediately.
-                waker.wake_by_ref();
-            }
+        if self.raw.try_set_join_waker(waker) {
+            // In this case the task has already completed. We wake the waker immediately.
+            waker.wake_by_ref();
         }
     }
 
     /// Returns a new `AbortHandle` that can be used to remotely abort this task.
     pub(crate) fn abort_handle(&self) -> super::AbortHandle {
-        let raw = self.raw.map(|raw| {
-            raw.ref_inc();
-            raw
-        });
-        super::AbortHandle::new(raw, self.id)
+        self.raw.ref_inc();
+        super::AbortHandle::new(self.raw)
     }
 
     /// Returns a [task ID] that uniquely identifies this task relative to other
@@ -282,7 +269,8 @@ impl<T> JoinHandle<T> {
     #[cfg(tokio_unstable)]
     #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
     pub fn id(&self) -> super::Id {
-        self.id
+        // Safety: The header pointer is valid.
+        unsafe { Header::get_id(self.raw.header_ptr()) }
     }
 }
 
@@ -297,13 +285,6 @@ impl<T> Future for JoinHandle<T> {
         // Keep track of task budget
         let coop = ready!(crate::runtime::coop::poll_proceed(cx));
 
-        // Raw should always be set. If it is not, this is due to polling after
-        // completion
-        let raw = self
-            .raw
-            .as_ref()
-            .expect("polling after `JoinHandle` already completed");
-
         // Try to read the task output. If the task is not yet complete, the
         // waker is stored and is notified once the task does complete.
         //
@@ -316,7 +297,8 @@ impl<T> Future for JoinHandle<T> {
         //
         // The type of `T` must match the task's output type.
         unsafe {
-            raw.try_read_output(&mut ret as *mut _ as *mut (), cx.waker());
+            self.raw
+                .try_read_output(&mut ret as *mut _ as *mut (), cx.waker());
         }
 
         if ret.is_ready() {
@@ -329,13 +311,11 @@ impl<T> Future for JoinHandle<T> {
 
 impl<T> Drop for JoinHandle<T> {
     fn drop(&mut self) {
-        if let Some(raw) = self.raw.take() {
-            if raw.header().state.drop_join_handle_fast().is_ok() {
-                return;
-            }
-
-            raw.drop_join_handle_slow();
+        if self.raw.state().drop_join_handle_fast().is_ok() {
+            return;
         }
+
+        self.raw.drop_join_handle_slow();
     }
 }
 
@@ -344,8 +324,9 @@ where
     T: fmt::Debug,
 {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt.debug_struct("JoinHandle")
-            .field("id", &self.id)
-            .finish()
+        // Safety: The header pointer is valid.
+        let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) };
+        let id = unsafe { id_ptr.as_ref() };
+        fmt.debug_struct("JoinHandle").field("id", id).finish()
     }
 }

tokio/src/runtime/task/mod.rs

@@ -47,7 +47,8 @@
 //!
 //!  * JOIN_INTEREST - Is set to one if there exists a JoinHandle.
 //!
-//!  * JOIN_WAKER - Is set to one if the JoinHandle has set a waker.
+//!  * JOIN_WAKER - Acts as an access control bit for the join handle waker. The
+//!    protocol for its usage is described below.
 //!
 //! The rest of the bits are used for the ref-count.
 //!
@@ -71,10 +72,38 @@
 //!    a lock for the stage field, and it can be accessed only by the thread
 //!    that set RUNNING to one.
 //!
-//!  * If JOIN_WAKER is zero, then the JoinHandle has exclusive access to the
-//!    join handle waker. If JOIN_WAKER and COMPLETE are both one, then the
-//!    thread that set COMPLETE to one has exclusive access to the join handle
-//!    waker.
+//!  * The waker field may be concurrently accessed by different threads: in one
+//!    thread the runtime may complete a task and *read* the waker field to
+//!    invoke the waker, and in another thread the task's JoinHandle may be
+//!    polled, and if the task hasn't yet completed, the JoinHandle may *write*
+//!    a waker to the waker field. The JOIN_WAKER bit ensures safe access by
+//!    multiple threads to the waker field using the following rules:
+//!
+//!    1. JOIN_WAKER is initialized to zero.
+//!
+//!    2. If JOIN_WAKER is zero, then the JoinHandle has exclusive (mutable)
+//!       access to the waker field.
+//!
+//!    3. If JOIN_WAKER is one, then the JoinHandle has shared (read-only)
+//!       access to the waker field.
+//!
+//!    4. If JOIN_WAKER is one and COMPLETE is one, then the runtime has shared
+//!       (read-only) access to the waker field.
+//!
+//!    5. If the JoinHandle needs to write to the waker field, then the
+//!       JoinHandle needs to (i) successfully set JOIN_WAKER to zero if it is
+//!       not already zero to gain exclusive access to the waker field per rule
+//!       2, (ii) write a waker, and (iii) successfully set JOIN_WAKER to one.
+//!
+//!    6. The JoinHandle can change JOIN_WAKER only if COMPLETE is zero (i.e.
+//!       the task hasn't yet completed).
+//!
+//!    Rule 6 implies that the steps (i) or (iii) of rule 5 may fail due to a
+//!    race. If step (i) fails, then the attempt to write a waker is aborted. If
+//!    step (iii) fails because COMPLETE is set to one by another thread after
+//!    step (i), then the waker field is cleared. Once COMPLETE is one (i.e.
+//!    task has completed), the JoinHandle will not modify JOIN_WAKER. After the
+//!    runtime sets COMPLETE to one, it invokes the waker if there is one.
 //!
 //! All other fields are immutable and can be accessed immutably without
 //! synchronization by anyone.
@@ -338,7 +367,7 @@ cfg_rt! {
             raw,
             _p: PhantomData,
         });
-        let join = JoinHandle::new(raw, id);
+        let join = JoinHandle::new(raw);
 
         (task, notified, join)
     }
@@ -533,55 +562,12 @@ impl fmt::Display for Id {
 }
 
 impl Id {
-    // When 64-bit atomics are available, use a static `AtomicU64` counter to
-    // generate task IDs.
-    //
-    // Note(eliza): we _could_ just use `crate::loom::AtomicU64`, which switches
-    // between an atomic and mutex-based implementation here, rather than having
-    // two separate functions for targets with and without 64-bit atomics.
-    // However, because we can't use the mutex-based implementation in a static
-    // initializer directly, the 32-bit impl also has to use a `OnceCell`, and I
-    // thought it was nicer to avoid the `OnceCell` overhead on 64-bit
-    // platforms...
-    cfg_has_atomic_u64! {
-        pub(crate) fn next() -> Self {
-            use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
-            static NEXT_ID: AtomicU64 = AtomicU64::new(1);
-            Self(NEXT_ID.fetch_add(1, Relaxed))
-        }
-    }
+    pub(crate) fn next() -> Self {
+        use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64};
 
-    cfg_not_has_atomic_u64! {
-        cfg_has_const_mutex_new! {
-            pub(crate) fn next() -> Self {
-                use crate::loom::sync::Mutex;
-                static NEXT_ID: Mutex<u64> = Mutex::const_new(1);
+        static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1);
 
-                let mut lock = NEXT_ID.lock();
-                let id = *lock;
-                *lock += 1;
-                Self(id)
-            }
-        }
-
-        cfg_not_has_const_mutex_new! {
-            pub(crate) fn next() -> Self {
-                use crate::util::once_cell::OnceCell;
-                use crate::loom::sync::Mutex;
-
-                fn init_next_id() -> Mutex<u64> {
-                    Mutex::new(1)
-                }
-
-                static NEXT_ID: OnceCell<Mutex<u64>> = OnceCell::new();
-
-                let next_id = NEXT_ID.get(init_next_id);
-                let mut lock = next_id.lock();
-                let id = *lock;
-                *lock += 1;
-                Self(id)
-            }
-        }
+        Self(NEXT_ID.fetch_add(1, Relaxed))
     }
 
     pub(crate) fn as_u64(&self) -> u64 {

tokio/src/runtime/task/raw.rs

@@ -14,45 +14,47 @@ pub(super) struct Vtable {
     /// Polls the future.
     pub(super) poll: unsafe fn(NonNull<Header>),
 
+    /// Schedules the task for execution on the runtime.
+    pub(super) schedule: unsafe fn(NonNull<Header>),
+
     /// Deallocates the memory.
     pub(super) dealloc: unsafe fn(NonNull<Header>),
 
     /// Reads the task output, if complete.
     pub(super) try_read_output: unsafe fn(NonNull<Header>, *mut (), &Waker),
 
-    /// Try to set the waker notified when the task is complete. Returns true if
-    /// the task has already completed. If this call returns false, then the
-    /// waker will not be notified.
-    pub(super) try_set_join_waker: unsafe fn(NonNull<Header>, &Waker) -> bool,
-
     /// The join handle has been dropped.
     pub(super) drop_join_handle_slow: unsafe fn(NonNull<Header>),
 
     /// An abort handle has been dropped.
     pub(super) drop_abort_handle: unsafe fn(NonNull<Header>),
 
-    /// The task is remotely aborted.
-    pub(super) remote_abort: unsafe fn(NonNull<Header>),
-
     /// Scheduler is being shutdown.
     pub(super) shutdown: unsafe fn(NonNull<Header>),
 
     /// The number of bytes that the `trailer` field is offset from the header.
     pub(super) trailer_offset: usize,
+
+    /// The number of bytes that the `scheduler` field is offset from the header.
+    pub(super) scheduler_offset: usize,
+
+    /// The number of bytes that the `id` field is offset from the header.
+    pub(super) id_offset: usize,
 }
 
 /// Get the vtable for the requested `T` and `S` generics.
 pub(super) fn vtable<T: Future, S: Schedule>() -> &'static Vtable {
     &Vtable {
         poll: poll::<T, S>,
+        schedule: schedule::<S>,
         dealloc: dealloc::<T, S>,
         try_read_output: try_read_output::<T, S>,
-        try_set_join_waker: try_set_join_waker::<T, S>,
         drop_join_handle_slow: drop_join_handle_slow::<T, S>,
         drop_abort_handle: drop_abort_handle::<T, S>,
-        remote_abort: remote_abort::<T, S>,
         shutdown: shutdown::<T, S>,
-        trailer_offset: TrailerOffsetHelper::<T, S>::OFFSET,
+        trailer_offset: OffsetHelper::<T, S>::TRAILER_OFFSET,
+        scheduler_offset: OffsetHelper::<T, S>::SCHEDULER_OFFSET,
+        id_offset: OffsetHelper::<T, S>::ID_OFFSET,
     }
 }
 
@@ -61,17 +63,31 @@ pub(super) fn vtable<T: Future, S: Schedule>() -> &'static Vtable {
 ///
 /// See this thread for more info:
 /// <https://users.rust-lang.org/t/custom-vtables-with-integers/78508>
-struct TrailerOffsetHelper<T, S>(T, S);
-impl<T: Future, S: Schedule> TrailerOffsetHelper<T, S> {
+struct OffsetHelper<T, S>(T, S);
+impl<T: Future, S: Schedule> OffsetHelper<T, S> {
     // Pass `size_of`/`align_of` as arguments rather than calling them directly
     // inside `get_trailer_offset` because trait bounds on generic parameters
     // of const fn are unstable on our MSRV.
-    const OFFSET: usize = get_trailer_offset(
+    const TRAILER_OFFSET: usize = get_trailer_offset(
         std::mem::size_of::<Header>(),
         std::mem::size_of::<Core<T, S>>(),
         std::mem::align_of::<Core<T, S>>(),
         std::mem::align_of::<Trailer>(),
     );
+
+    // The `scheduler` is the first field of `Core`, so it has the same
+    // offset as `Core`.
+    const SCHEDULER_OFFSET: usize = get_core_offset(
+        std::mem::size_of::<Header>(),
+        std::mem::align_of::<Core<T, S>>(),
+    );
+
+    const ID_OFFSET: usize = get_id_offset(
+        std::mem::size_of::<Header>(),
+        std::mem::align_of::<Core<T, S>>(),
+        std::mem::size_of::<S>(),
+        std::mem::align_of::<Id>(),
+    );
 }
 
 /// Compute the offset of the `Trailer` field in `Cell<T, S>` using the
@@ -101,6 +117,44 @@ const fn get_trailer_offset(
     offset
 }
 
+/// Compute the offset of the `Core<T, S>` field in `Cell<T, S>` using the
+/// `#[repr(C)]` algorithm.
+///
+/// Pseudo-code for the `#[repr(C)]` algorithm can be found here:
+/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs>
+const fn get_core_offset(header_size: usize, core_align: usize) -> usize {
+    let mut offset = header_size;
+
+    let core_misalign = offset % core_align;
+    if core_misalign > 0 {
+        offset += core_align - core_misalign;
+    }
+
+    offset
+}
+
+/// Compute the offset of the `Id` field in `Cell<T, S>` using the
+/// `#[repr(C)]` algorithm.
+///
+/// Pseudo-code for the `#[repr(C)]` algorithm can be found here:
+/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs>
+const fn get_id_offset(
+    header_size: usize,
+    core_align: usize,
+    scheduler_size: usize,
+    id_align: usize,
+) -> usize {
+    let mut offset = get_core_offset(header_size, core_align);
+    offset += scheduler_size;
+
+    let id_misalign = offset % id_align;
+    if id_misalign > 0 {
+        offset += id_align - id_misalign;
+    }
+
+    offset
+}
+
 impl RawTask {
     pub(super) fn new<T, S>(task: T, scheduler: S, id: Id) -> RawTask
     where
@@ -121,19 +175,36 @@ impl RawTask {
         self.ptr
     }
 
-    /// Returns a reference to the task's meta structure.
-    ///
-    /// Safe as `Header` is `Sync`.
+    pub(super) fn trailer_ptr(&self) -> NonNull<Trailer> {
+        unsafe { Header::get_trailer(self.ptr) }
+    }
+
+    /// Returns a reference to the task's header.
     pub(super) fn header(&self) -> &Header {
         unsafe { self.ptr.as_ref() }
     }
 
+    /// Returns a reference to the task's trailer.
+    pub(super) fn trailer(&self) -> &Trailer {
+        unsafe { &*self.trailer_ptr().as_ptr() }
+    }
+
+    /// Returns a reference to the task's state.
+    pub(super) fn state(&self) -> &State {
+        &self.header().state
+    }
+
     /// Safety: mutual exclusion is required to call this function.
     pub(super) fn poll(self) {
         let vtable = self.header().vtable;
         unsafe { (vtable.poll)(self.ptr) }
     }
 
+    pub(super) fn schedule(self) {
+        let vtable = self.header().vtable;
+        unsafe { (vtable.schedule)(self.ptr) }
+    }
+
     pub(super) fn dealloc(self) {
         let vtable = self.header().vtable;
         unsafe {
@@ -148,11 +219,6 @@ impl RawTask {
         (vtable.try_read_output)(self.ptr, dst, waker);
     }
 
-    pub(super) fn try_set_join_waker(self, waker: &Waker) -> bool {
-        let vtable = self.header().vtable;
-        unsafe { (vtable.try_set_join_waker)(self.ptr, waker) }
-    }
-
     pub(super) fn drop_join_handle_slow(self) {
         let vtable = self.header().vtable;
         unsafe { (vtable.drop_join_handle_slow)(self.ptr) }
@@ -168,11 +234,6 @@ impl RawTask {
         unsafe { (vtable.shutdown)(self.ptr) }
     }
 
-    pub(super) fn remote_abort(self) {
-        let vtable = self.header().vtable;
-        unsafe { (vtable.remote_abort)(self.ptr) }
-    }
-
     /// Increment the task's reference count.
     ///
     /// Currently, this is used only when creating an `AbortHandle`.
@@ -194,6 +255,15 @@ unsafe fn poll<T: Future, S: Schedule>(ptr: NonNull<Header>) {
     harness.poll();
 }
 
+unsafe fn schedule<S: Schedule>(ptr: NonNull<Header>) {
+    use crate::runtime::task::{Notified, Task};
+
+    let scheduler = Header::get_scheduler::<S>(ptr);
+    scheduler
+        .as_ref()
+        .schedule(Notified(Task::from_raw(ptr.cast())));
+}
+
 unsafe fn dealloc<T: Future, S: Schedule>(ptr: NonNull<Header>) {
     let harness = Harness::<T, S>::from_raw(ptr);
     harness.dealloc();
@@ -210,11 +280,6 @@ unsafe fn try_read_output<T: Future, S: Schedule>(
     harness.try_read_output(out, waker);
 }
 
-unsafe fn try_set_join_waker<T: Future, S: Schedule>(ptr: NonNull<Header>, waker: &Waker) -> bool {
-    let harness = Harness::<T, S>::from_raw(ptr);
-    harness.try_set_join_waker(waker)
-}
-
 unsafe fn drop_join_handle_slow<T: Future, S: Schedule>(ptr: NonNull<Header>) {
     let harness = Harness::<T, S>::from_raw(ptr);
     harness.drop_join_handle_slow()
@@ -225,11 +290,6 @@ unsafe fn drop_abort_handle<T: Future, S: Schedule>(ptr: NonNull<Header>) {
     harness.drop_reference();
 }
 
-unsafe fn remote_abort<T: Future, S: Schedule>(ptr: NonNull<Header>) {
-    let harness = Harness::<T, S>::from_raw(ptr);
-    harness.remote_abort()
-}
-
 unsafe fn shutdown<T: Future, S: Schedule>(ptr: NonNull<Header>) {
     let harness = Harness::<T, S>::from_raw(ptr);
     harness.shutdown()

tokio/src/runtime/task/state.rs

@@ -378,7 +378,7 @@ impl State {
     pub(super) fn set_join_waker(&self) -> UpdateResult {
         self.fetch_update(|curr| {
             assert!(curr.is_join_interested());
-            assert!(!curr.has_join_waker());
+            assert!(!curr.is_join_waker_set());
 
             if curr.is_complete() {
                 return None;
@@ -398,7 +398,7 @@ impl State {
     pub(super) fn unset_waker(&self) -> UpdateResult {
         self.fetch_update(|curr| {
             assert!(curr.is_join_interested());
-            assert!(curr.has_join_waker());
+            assert!(curr.is_join_waker_set());
 
             if curr.is_complete() {
                 return None;
@@ -546,7 +546,7 @@ impl Snapshot {
         self.0 &= !JOIN_INTEREST
     }
 
-    pub(super) fn has_join_waker(self) -> bool {
+    pub(super) fn is_join_waker_set(self) -> bool {
         self.0 & JOIN_WAKER == JOIN_WAKER
     }
 
@@ -588,7 +588,7 @@ impl fmt::Debug for Snapshot {
             .field("is_notified", &self.is_notified())
             .field("is_cancelled", &self.is_cancelled())
             .field("is_join_interested", &self.is_join_interested())
-            .field("has_join_waker", &self.has_join_waker())
+            .field("is_join_waker_set", &self.is_join_waker_set())
             .field("ref_count", &self.ref_count())
             .finish()
     }

tokio/src/runtime/task/waker.rs

@@ -1,6 +1,5 @@
 use crate::future::Future;
-use crate::runtime::task::harness::Harness;
-use crate::runtime::task::{Header, Schedule};
+use crate::runtime::task::{Header, RawTask, Schedule};
 
 use std::marker::PhantomData;
 use std::mem::ManuallyDrop;
@@ -28,7 +27,7 @@ where
     // point and not an *owned* waker, we must ensure that `drop` is never
     // called on this waker instance. This is done by wrapping it with
     // `ManuallyDrop` and then never calling drop.
-    let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker::<T, S>(*header))) };
+    let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker(*header))) };
 
     WakerRef {
         waker,
@@ -46,8 +45,8 @@ impl<S> ops::Deref for WakerRef<'_, S> {
 
 cfg_trace! {
     macro_rules! trace {
-        ($harness:expr, $op:expr) => {
-            if let Some(id) = $harness.id() {
+        ($header:expr, $op:expr) => {
+            if let Some(id) = Header::get_tracing_id(&$header) {
                 tracing::trace!(
                     target: "tokio::task::waker",
                     op = $op,
@@ -60,71 +59,46 @@ cfg_trace! {
 
 cfg_not_trace! {
     macro_rules! trace {
-        ($harness:expr, $op:expr) => {
+        ($header:expr, $op:expr) => {
             // noop
-            let _ = &$harness;
+            let _ = &$header;
         }
     }
 }
 
-unsafe fn clone_waker<T, S>(ptr: *const ()) -> RawWaker
-where
-    T: Future,
-    S: Schedule,
-{
-    let header = ptr as *const Header;
-    let ptr = NonNull::new_unchecked(ptr as *mut Header);
-    let harness = Harness::<T, S>::from_raw(ptr);
-    trace!(harness, "waker.clone");
-    (*header).state.ref_inc();
-    raw_waker::<T, S>(ptr)
+unsafe fn clone_waker(ptr: *const ()) -> RawWaker {
+    let header = NonNull::new_unchecked(ptr as *mut Header);
+    trace!(header, "waker.clone");
+    header.as_ref().state.ref_inc();
+    raw_waker(header)
 }
 
-unsafe fn drop_waker<T, S>(ptr: *const ())
-where
-    T: Future,
-    S: Schedule,
-{
+unsafe fn drop_waker(ptr: *const ()) {
     let ptr = NonNull::new_unchecked(ptr as *mut Header);
-    let harness = Harness::<T, S>::from_raw(ptr);
-    trace!(harness, "waker.drop");
-    harness.drop_reference();
+    trace!(ptr, "waker.drop");
+    let raw = RawTask::from_raw(ptr);
+    raw.drop_reference();
 }
 
-unsafe fn wake_by_val<T, S>(ptr: *const ())
-where
-    T: Future,
-    S: Schedule,
-{
+unsafe fn wake_by_val(ptr: *const ()) {
     let ptr = NonNull::new_unchecked(ptr as *mut Header);
-    let harness = Harness::<T, S>::from_raw(ptr);
-    trace!(harness, "waker.wake");
-    harness.wake_by_val();
+    trace!(ptr, "waker.wake");
+    let raw = RawTask::from_raw(ptr);
+    raw.wake_by_val();
 }
 
 // Wake without consuming the waker
-unsafe fn wake_by_ref<T, S>(ptr: *const ())
-where
-    T: Future,
-    S: Schedule,
-{
+unsafe fn wake_by_ref(ptr: *const ()) {
     let ptr = NonNull::new_unchecked(ptr as *mut Header);
-    let harness = Harness::<T, S>::from_raw(ptr);
-    trace!(harness, "waker.wake_by_ref");
-    harness.wake_by_ref();
+    trace!(ptr, "waker.wake_by_ref");
+    let raw = RawTask::from_raw(ptr);
+    raw.wake_by_ref();
 }
 
-fn raw_waker<T, S>(header: NonNull<Header>) -> RawWaker
-where
-    T: Future,
-    S: Schedule,
-{
+static WAKER_VTABLE: RawWakerVTable =
+    RawWakerVTable::new(clone_waker, wake_by_val, wake_by_ref, drop_waker);
+
+fn raw_waker(header: NonNull<Header>) -> RawWaker {
     let ptr = header.as_ptr() as *const ();
-    let vtable = &RawWakerVTable::new(
-        clone_waker::<T, S>,
-        wake_by_val::<T, S>,
-        wake_by_ref::<T, S>,
-        drop_waker::<T, S>,
-    );
-    RawWaker::new(ptr, vtable)
+    RawWaker::new(ptr, &WAKER_VTABLE)
 }

tokio/src/runtime/tests/loom_blocking.rs

@@ -73,6 +73,27 @@ fn spawn_mandatory_blocking_should_run_even_when_shutting_down_from_other_thread
     });
 }
 
+#[test]
+fn spawn_blocking_when_paused() {
+    use std::time::Duration;
+    loom::model(|| {
+        let rt = crate::runtime::Builder::new_current_thread()
+            .enable_time()
+            .start_paused(true)
+            .build()
+            .unwrap();
+        let handle = rt.handle();
+        let _enter = handle.enter();
+        let a = crate::task::spawn_blocking(|| {});
+        let b = crate::task::spawn_blocking(|| {});
+        rt.block_on(crate::time::timeout(Duration::from_millis(1), async move {
+            a.await.expect("blocking task should finish");
+            b.await.expect("blocking task should finish");
+        }))
+        .expect("timeout should not trigger");
+    });
+}
+
 fn mk_runtime(num_threads: usize) -> Runtime {
     runtime::Builder::new_multi_thread()
         .worker_threads(num_threads)

tokio/src/runtime/tests/loom_queue.rs

@@ -1,6 +1,6 @@
-use crate::runtime::blocking::NoopSchedule;
 use crate::runtime::scheduler::multi_thread::queue;
 use crate::runtime::task::Inject;
+use crate::runtime::tests::NoopSchedule;
 use crate::runtime::MetricsBatch;
 
 use loom::thread;

tokio/src/runtime/tests/loom_yield.rs

@@ -0,0 +1,37 @@
+use crate::runtime::park;
+use crate::runtime::tests::loom_oneshot as oneshot;
+use crate::runtime::{self, Runtime};
+
+#[test]
+fn yield_calls_park_before_scheduling_again() {
+    // Don't need to check all permutations
+    let mut loom = loom::model::Builder::default();
+    loom.max_permutations = Some(1);
+    loom.check(|| {
+        let rt = mk_runtime(2);
+        let (tx, rx) = oneshot::channel::<()>();
+
+        rt.spawn(async {
+            let tid = loom::thread::current().id();
+            let park_count = park::current_thread_park_count();
+
+            crate::task::yield_now().await;
+
+            if tid == loom::thread::current().id() {
+                let new_park_count = park::current_thread_park_count();
+                assert_eq!(park_count + 1, new_park_count);
+            }
+
+            tx.send(());
+        });
+
+        rx.recv();
+    });
+}
+
+fn mk_runtime(num_threads: usize) -> Runtime {
+    runtime::Builder::new_multi_thread()
+        .worker_threads(num_threads)
+        .build()
+        .unwrap()
+}

tokio/src/runtime/tests/mod.rs

@@ -2,11 +2,29 @@
 // other code when running loom tests.
 #![cfg_attr(loom, warn(dead_code, unreachable_pub))]
 
+use self::noop_scheduler::NoopSchedule;
 use self::unowned_wrapper::unowned;
 
+mod noop_scheduler {
+    use crate::runtime::task::{self, Task};
+
+    /// `task::Schedule` implementation that does nothing, for testing.
+    pub(crate) struct NoopSchedule;
+
+    impl task::Schedule for NoopSchedule {
+        fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> {
+            None
+        }
+
+        fn schedule(&self, _task: task::Notified<Self>) {
+            unreachable!();
+        }
+    }
+}
+
 mod unowned_wrapper {
-    use crate::runtime::blocking::NoopSchedule;
     use crate::runtime::task::{Id, JoinHandle, Notified};
+    use crate::runtime::tests::NoopSchedule;
 
     #[cfg(all(tokio_unstable, feature = "tracing"))]
     pub(crate) fn unowned<T>(task: T) -> (Notified<NoopSchedule>, JoinHandle<T::Output>)
@@ -41,6 +59,7 @@ cfg_loom! {
     mod loom_queue;
     mod loom_shutdown_join;
     mod loom_join_set;
+    mod loom_yield;
 }
 
 cfg_not_loom! {

tokio/src/runtime/tests/task.rs

@@ -1,5 +1,5 @@
-use crate::runtime::blocking::NoopSchedule;
 use crate::runtime::task::{self, unowned, Id, JoinHandle, OwnedTasks, Schedule, Task};
+use crate::runtime::tests::NoopSchedule;
 use crate::util::TryLock;
 
 use std::collections::VecDeque;

tokio/src/runtime/time/mod.rs

@@ -222,7 +222,7 @@ impl Driver {
             let handle = rt_handle.time();
             let clock = &handle.time_source.clock;
 
-            if clock.is_paused() {
+            if clock.can_auto_advance() {
                 self.park.park_timeout(rt_handle, Duration::from_secs(0));
 
                 // If the time driver was woken, then the park completed

tokio/src/signal/registry.rs

@@ -5,7 +5,6 @@ use crate::sync::watch;
 use crate::util::once_cell::OnceCell;
 
 use std::ops;
-use std::pin::Pin;
 use std::sync::atomic::{AtomicBool, Ordering};
 
 pub(crate) type EventId = usize;
@@ -162,14 +161,14 @@ where
     }
 }
 
-pub(crate) fn globals() -> Pin<&'static Globals>
+pub(crate) fn globals() -> &'static Globals
 where
     OsExtraData: 'static + Send + Sync + Init,
     OsStorage: 'static + Send + Sync + Init,
 {
     static GLOBALS: OnceCell<Globals> = OnceCell::new();
 
-    Pin::new(GLOBALS.get(globals_init))
+    GLOBALS.get(globals_init)
 }
 
 #[cfg(all(test, not(loom)))]

tokio/src/signal/unix.rs

@@ -14,7 +14,6 @@ use crate::sync::watch;
 
 use mio::net::UnixStream;
 use std::io::{self, Error, ErrorKind, Write};
-use std::pin::Pin;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Once;
 use std::task::{Context, Poll};
@@ -240,7 +239,7 @@ impl Default for SignalInfo {
 /// 2. Wake up the driver by writing a byte to a pipe
 ///
 /// Those two operations should both be async-signal safe.
-fn action(globals: Pin<&'static Globals>, signal: libc::c_int) {
+fn action(globals: &'static Globals, signal: libc::c_int) {
     globals.record_event(signal as EventId);
 
     // Send a wakeup, ignore any errors (anything reasonably possible is

tokio/src/signal/windows.rs

@@ -3,7 +3,7 @@
 //! This module is only defined on Windows and allows receiving "ctrl-c",
 //! "ctrl-break", "ctrl-logoff", "ctrl-shutdown", and "ctrl-close"
 //! notifications. These events are listened for via the `SetConsoleCtrlHandler`
-//! function which receives the corresponding winapi event type.
+//! function which receives the corresponding windows_sys event type.
 
 #![cfg(any(windows, docsrs))]
 #![cfg_attr(docsrs, doc(cfg(all(windows, feature = "signal"))))]

tokio/src/signal/windows/sys.rs

@@ -5,31 +5,30 @@ use std::sync::Once;
 use crate::signal::registry::{globals, EventId, EventInfo, Init, Storage};
 use crate::signal::RxFuture;
 
-use winapi::shared::minwindef::{BOOL, DWORD, FALSE, TRUE};
-use winapi::um::consoleapi::SetConsoleCtrlHandler;
-use winapi::um::wincon;
+use windows_sys::Win32::Foundation::BOOL;
+use windows_sys::Win32::System::Console as console;
 
 pub(super) fn ctrl_break() -> io::Result<RxFuture> {
-    new(wincon::CTRL_BREAK_EVENT)
+    new(console::CTRL_BREAK_EVENT)
 }
 
 pub(super) fn ctrl_close() -> io::Result<RxFuture> {
-    new(wincon::CTRL_CLOSE_EVENT)
+    new(console::CTRL_CLOSE_EVENT)
 }
 
 pub(super) fn ctrl_c() -> io::Result<RxFuture> {
-    new(wincon::CTRL_C_EVENT)
+    new(console::CTRL_C_EVENT)
 }
 
 pub(super) fn ctrl_logoff() -> io::Result<RxFuture> {
-    new(wincon::CTRL_LOGOFF_EVENT)
+    new(console::CTRL_LOGOFF_EVENT)
 }
 
 pub(super) fn ctrl_shutdown() -> io::Result<RxFuture> {
-    new(wincon::CTRL_SHUTDOWN_EVENT)
+    new(console::CTRL_SHUTDOWN_EVENT)
 }
 
-fn new(signum: DWORD) -> io::Result<RxFuture> {
+fn new(signum: u32) -> io::Result<RxFuture> {
     global_init()?;
     let rx = globals().register_listener(signum as EventId);
     Ok(RxFuture::new(rx))
@@ -58,12 +57,12 @@ impl Init for OsStorage {
 
 impl Storage for OsStorage {
     fn event_info(&self, id: EventId) -> Option<&EventInfo> {
-        match DWORD::try_from(id) {
-            Ok(wincon::CTRL_BREAK_EVENT) => Some(&self.ctrl_break),
-            Ok(wincon::CTRL_CLOSE_EVENT) => Some(&self.ctrl_close),
-            Ok(wincon::CTRL_C_EVENT) => Some(&self.ctrl_c),
-            Ok(wincon::CTRL_LOGOFF_EVENT) => Some(&self.ctrl_logoff),
-            Ok(wincon::CTRL_SHUTDOWN_EVENT) => Some(&self.ctrl_shutdown),
+        match u32::try_from(id) {
+            Ok(console::CTRL_BREAK_EVENT) => Some(&self.ctrl_break),
+            Ok(console::CTRL_CLOSE_EVENT) => Some(&self.ctrl_close),
+            Ok(console::CTRL_C_EVENT) => Some(&self.ctrl_c),
+            Ok(console::CTRL_LOGOFF_EVENT) => Some(&self.ctrl_logoff),
+            Ok(console::CTRL_SHUTDOWN_EVENT) => Some(&self.ctrl_shutdown),
             _ => None,
         }
     }
@@ -95,7 +94,7 @@ fn global_init() -> io::Result<()> {
     let mut init = None;
 
     INIT.call_once(|| unsafe {
-        let rc = SetConsoleCtrlHandler(Some(handler), TRUE);
+        let rc = console::SetConsoleCtrlHandler(Some(handler), 1);
         let ret = if rc == 0 {
             Err(io::Error::last_os_error())
         } else {
@@ -108,7 +107,7 @@ fn global_init() -> io::Result<()> {
     init.unwrap_or_else(|| Ok(()))
 }
 
-unsafe extern "system" fn handler(ty: DWORD) -> BOOL {
+unsafe extern "system" fn handler(ty: u32) -> BOOL {
     let globals = globals();
     globals.record_event(ty as EventId);
 
@@ -117,11 +116,11 @@ unsafe extern "system" fn handler(ty: DWORD) -> BOOL {
     // have the same restrictions as in Unix signal handlers, meaning we can
     // go ahead and perform the broadcast here.
     if globals.broadcast() {
-        TRUE
+        1
     } else {
         // No one is listening for this notification any more
         // let the OS fire the next (possibly the default) handler.
-        FALSE
+        0
     }
 }
 
@@ -145,7 +144,7 @@ mod tests {
         // like sending signals on Unix, so we'll stub out the actual OS
         // integration and test that our handling works.
         unsafe {
-            super::handler(wincon::CTRL_C_EVENT);
+            super::handler(console::CTRL_C_EVENT);
         }
 
         assert_ready_ok!(ctrl_c.poll());
@@ -162,7 +161,7 @@ mod tests {
             // like sending signals on Unix, so we'll stub out the actual OS
             // integration and test that our handling works.
             unsafe {
-                super::handler(wincon::CTRL_BREAK_EVENT);
+                super::handler(console::CTRL_BREAK_EVENT);
             }
 
             ctrl_break.recv().await.unwrap();
@@ -180,7 +179,7 @@ mod tests {
             // like sending signals on Unix, so we'll stub out the actual OS
             // integration and test that our handling works.
             unsafe {
-                super::handler(wincon::CTRL_CLOSE_EVENT);
+                super::handler(console::CTRL_CLOSE_EVENT);
             }
 
             ctrl_close.recv().await.unwrap();
@@ -198,7 +197,7 @@ mod tests {
             // like sending signals on Unix, so we'll stub out the actual OS
             // integration and test that our handling works.
             unsafe {
-                super::handler(wincon::CTRL_SHUTDOWN_EVENT);
+                super::handler(console::CTRL_SHUTDOWN_EVENT);
             }
 
             ctrl_shutdown.recv().await.unwrap();
@@ -216,7 +215,7 @@ mod tests {
             // like sending signals on Unix, so we'll stub out the actual OS
             // integration and test that our handling works.
             unsafe {
-                super::handler(wincon::CTRL_LOGOFF_EVENT);
+                super::handler(console::CTRL_LOGOFF_EVENT);
             }
 
             ctrl_logoff.recv().await.unwrap();

tokio/src/sync/mpsc/block.rs

@@ -1,6 +1,7 @@
 use crate::loom::cell::UnsafeCell;
 use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize};
 
+use std::alloc::Layout;
 use std::mem::MaybeUninit;
 use std::ops;
 use std::ptr::{self, NonNull};
@@ -10,6 +11,17 @@ use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Release};
 ///
 /// Each block in the list can hold up to `BLOCK_CAP` messages.
 pub(crate) struct Block<T> {
+    /// The header fields.
+    header: BlockHeader<T>,
+
+    /// Array containing values pushed into the block. Values are stored in a
+    /// continuous array in order to improve cache line behavior when reading.
+    /// The values must be manually dropped.
+    values: Values<T>,
+}
+
+/// Extra fields for a `Block<T>`.
+struct BlockHeader<T> {
     /// The start index of this block.
     ///
     /// Slots in this block have indices in `start_index .. start_index + BLOCK_CAP`.
@@ -24,11 +36,6 @@ pub(crate) struct Block<T> {
     /// The observed `tail_position` value *after* the block has been passed by
     /// `block_tail`.
     observed_tail_position: UnsafeCell<usize>,
-
-    /// Array containing values pushed into the block. Values are stored in a
-    /// continuous array in order to improve cache line behavior when reading.
-    /// The values must be manually dropped.
-    values: Values<T>,
 }
 
 pub(crate) enum Read<T> {
@@ -36,6 +43,7 @@ pub(crate) enum Read<T> {
     Closed,
 }
 
+#[repr(transparent)]
 struct Values<T>([UnsafeCell<MaybeUninit<T>>; BLOCK_CAP]);
 
 use super::BLOCK_CAP;
@@ -71,28 +79,56 @@ pub(crate) fn offset(slot_index: usize) -> usize {
     SLOT_MASK & slot_index
 }
 
+generate_addr_of_methods! {
+    impl<T> Block<T> {
+        unsafe fn addr_of_header(self: NonNull<Self>) -> NonNull<BlockHeader<T>> {
+            &self.header
+        }
+
+        unsafe fn addr_of_values(self: NonNull<Self>) -> NonNull<Values<T>> {
+            &self.values
+        }
+    }
+}
+
 impl<T> Block<T> {
-    pub(crate) fn new(start_index: usize) -> Block<T> {
-        Block {
-            // The absolute index in the channel of the first slot in the block.
-            start_index,
+    pub(crate) fn new(start_index: usize) -> Box<Block<T>> {
+        unsafe {
+            // Allocate the block on the heap.
+            // SAFETY: The size of the Block<T> is non-zero, since it is at least the size of the header.
+            let block = std::alloc::alloc(Layout::new::<Block<T>>()) as *mut Block<T>;
+            let block = match NonNull::new(block) {
+                Some(block) => block,
+                None => std::alloc::handle_alloc_error(Layout::new::<Block<T>>()),
+            };
 
-            // Pointer to the next block in the linked list.
-            next: AtomicPtr::new(ptr::null_mut()),
+            // Write the header to the block.
+            Block::addr_of_header(block).as_ptr().write(BlockHeader {
+                // The absolute index in the channel of the first slot in the block.
+                start_index,
 
-            ready_slots: AtomicUsize::new(0),
+                // Pointer to the next block in the linked list.
+                next: AtomicPtr::new(ptr::null_mut()),
 
-            observed_tail_position: UnsafeCell::new(0),
+                ready_slots: AtomicUsize::new(0),
 
-            // Value storage
-            values: unsafe { Values::uninitialized() },
+                observed_tail_position: UnsafeCell::new(0),
+            });
+
+            // Initialize the values array.
+            Values::initialize(Block::addr_of_values(block));
+
+            // Convert the pointer to a `Box`.
+            // Safety: The raw pointer was allocated using the global allocator, and with
+            // the layout for a `Block<T>`, so it's valid to convert it to box.
+            Box::from_raw(block.as_ptr())
         }
     }
 
     /// Returns `true` if the block matches the given index.
     pub(crate) fn is_at_index(&self, index: usize) -> bool {
         debug_assert!(offset(index) == 0);
-        self.start_index == index
+        self.header.start_index == index
     }
 
     /// Returns the number of blocks between `self` and the block at the
@@ -101,7 +137,7 @@ impl<T> Block<T> {
     /// `start_index` must represent a block *after* `self`.
     pub(crate) fn distance(&self, other_index: usize) -> usize {
         debug_assert!(offset(other_index) == 0);
-        other_index.wrapping_sub(self.start_index) / BLOCK_CAP
+        other_index.wrapping_sub(self.header.start_index) / BLOCK_CAP
     }
 
     /// Reads the value at the given offset.
@@ -116,7 +152,7 @@ impl<T> Block<T> {
     pub(crate) unsafe fn read(&self, slot_index: usize) -> Option<Read<T>> {
         let offset = offset(slot_index);
 
-        let ready_bits = self.ready_slots.load(Acquire);
+        let ready_bits = self.header.ready_slots.load(Acquire);
 
         if !is_ready(ready_bits, offset) {
             if is_tx_closed(ready_bits) {
@@ -156,7 +192,7 @@ impl<T> Block<T> {
 
     /// Signal to the receiver that the sender half of the list is closed.
     pub(crate) unsafe fn tx_close(&self) {
-        self.ready_slots.fetch_or(TX_CLOSED, Release);
+        self.header.ready_slots.fetch_or(TX_CLOSED, Release);
     }
 
     /// Resets the block to a blank state. This enables reusing blocks in the
@@ -169,9 +205,9 @@ impl<T> Block<T> {
     /// * All slots are empty.
     /// * The caller holds a unique pointer to the block.
     pub(crate) unsafe fn reclaim(&mut self) {
-        self.start_index = 0;
-        self.next = AtomicPtr::new(ptr::null_mut());
-        self.ready_slots = AtomicUsize::new(0);
+        self.header.start_index = 0;
+        self.header.next = AtomicPtr::new(ptr::null_mut());
+        self.header.ready_slots = AtomicUsize::new(0);
     }
 
     /// Releases the block to the rx half for freeing.
@@ -187,19 +223,20 @@ impl<T> Block<T> {
     pub(crate) unsafe fn tx_release(&self, tail_position: usize) {
         // Track the observed tail_position. Any sender targeting a greater
         // tail_position is guaranteed to not access this block.
-        self.observed_tail_position
+        self.header
+            .observed_tail_position
             .with_mut(|ptr| *ptr = tail_position);
 
         // Set the released bit, signalling to the receiver that it is safe to
         // free the block's memory as soon as all slots **prior** to
         // `observed_tail_position` have been filled.
-        self.ready_slots.fetch_or(RELEASED, Release);
+        self.header.ready_slots.fetch_or(RELEASED, Release);
     }
 
     /// Mark a slot as ready
     fn set_ready(&self, slot: usize) {
         let mask = 1 << slot;
-        self.ready_slots.fetch_or(mask, Release);
+        self.header.ready_slots.fetch_or(mask, Release);
     }
 
     /// Returns `true` when all slots have their `ready` bits set.
@@ -214,25 +251,31 @@ impl<T> Block<T> {
     /// single atomic cell. However, this could have negative impact on cache
     /// behavior as there would be many more mutations to a single slot.
     pub(crate) fn is_final(&self) -> bool {
-        self.ready_slots.load(Acquire) & READY_MASK == READY_MASK
+        self.header.ready_slots.load(Acquire) & READY_MASK == READY_MASK
     }
 
     /// Returns the `observed_tail_position` value, if set
     pub(crate) fn observed_tail_position(&self) -> Option<usize> {
-        if 0 == RELEASED & self.ready_slots.load(Acquire) {
+        if 0 == RELEASED & self.header.ready_slots.load(Acquire) {
             None
         } else {
-            Some(self.observed_tail_position.with(|ptr| unsafe { *ptr }))
+            Some(
+                self.header
+                    .observed_tail_position
+                    .with(|ptr| unsafe { *ptr }),
+            )
         }
     }
 
     /// Loads the next block
     pub(crate) fn load_next(&self, ordering: Ordering) -> Option<NonNull<Block<T>>> {
-        let ret = NonNull::new(self.next.load(ordering));
+        let ret = NonNull::new(self.header.next.load(ordering));
 
         debug_assert!(unsafe {
-            ret.map(|block| block.as_ref().start_index == self.start_index.wrapping_add(BLOCK_CAP))
-                .unwrap_or(true)
+            ret.map(|block| {
+                block.as_ref().header.start_index == self.header.start_index.wrapping_add(BLOCK_CAP)
+            })
+            .unwrap_or(true)
         });
 
         ret
@@ -260,9 +303,10 @@ impl<T> Block<T> {
         success: Ordering,
         failure: Ordering,
     ) -> Result<(), NonNull<Block<T>>> {
-        block.as_mut().start_index = self.start_index.wrapping_add(BLOCK_CAP);
+        block.as_mut().header.start_index = self.header.start_index.wrapping_add(BLOCK_CAP);
 
         let next_ptr = self
+            .header
             .next
             .compare_exchange(ptr::null_mut(), block.as_ptr(), success, failure)
             .unwrap_or_else(|x| x);
@@ -291,7 +335,7 @@ impl<T> Block<T> {
         // Create the new block. It is assumed that the block will become the
         // next one after `&self`. If this turns out to not be the case,
         // `start_index` is updated accordingly.
-        let new_block = Box::new(Block::new(self.start_index + BLOCK_CAP));
+        let new_block = Block::new(self.header.start_index + BLOCK_CAP);
 
         let mut new_block = unsafe { NonNull::new_unchecked(Box::into_raw(new_block)) };
 
@@ -308,7 +352,8 @@ impl<T> Block<T> {
         // `Release` ensures that the newly allocated block is available to
         // other threads acquiring the next pointer.
         let next = NonNull::new(
-            self.next
+            self.header
+                .next
                 .compare_exchange(ptr::null_mut(), new_block.as_ptr(), AcqRel, Acquire)
                 .unwrap_or_else(|x| x),
         );
@@ -360,19 +405,20 @@ fn is_tx_closed(bits: usize) -> bool {
 }
 
 impl<T> Values<T> {
-    unsafe fn uninitialized() -> Values<T> {
-        let mut vals = MaybeUninit::uninit();
-
+    /// Initialize a `Values` struct from a pointer.
+    ///
+    /// # Safety
+    ///
+    /// The raw pointer must be valid for writing a `Values<T>`.
+    unsafe fn initialize(_value: NonNull<Values<T>>) {
         // When fuzzing, `UnsafeCell` needs to be initialized.
         if_loom! {
-            let p = vals.as_mut_ptr() as *mut UnsafeCell<MaybeUninit<T>>;
+            let p = _value.as_ptr() as *mut UnsafeCell<MaybeUninit<T>>;
             for i in 0..BLOCK_CAP {
                 p.add(i)
                     .write(UnsafeCell::new(MaybeUninit::uninit()));
             }
         }
-
-        Values(vals.assume_init())
     }
 }
 
@@ -383,3 +429,20 @@ impl<T> ops::Index<usize> for Values<T> {
         self.0.index(index)
     }
 }
+
+#[cfg(all(test, not(loom)))]
+#[test]
+fn assert_no_stack_overflow() {
+    // https://github.com/tokio-rs/tokio/issues/5293
+
+    struct Foo {
+        _a: [u8; 2_000_000],
+    }
+
+    assert_eq!(
+        Layout::new::<MaybeUninit<Block<Foo>>>(),
+        Layout::new::<Block<Foo>>()
+    );
+
+    let _block = Block::<Foo>::new(0);
+}

tokio/src/sync/mpsc/list.rs

@@ -44,7 +44,7 @@ pub(crate) enum TryPopResult<T> {
 
 pub(crate) fn channel<T>() -> (Tx<T>, Rx<T>) {
     // Create the initial block shared between the tx and rx halves.
-    let initial_block = Box::new(Block::new(0));
+    let initial_block = Block::new(0);
     let initial_block_ptr = Box::into_raw(initial_block);
 
     let tx = Tx {

tokio/src/sync/watch.rs

@@ -9,10 +9,10 @@
 //! # Usage
 //!
 //! [`channel`] returns a [`Sender`] / [`Receiver`] pair. These are the producer
-//! and sender halves of the channel. The channel is created with an initial
+//! and consumer halves of the channel. The channel is created with an initial
 //! value. The **latest** value stored in the channel is accessed with
 //! [`Receiver::borrow()`]. Awaiting [`Receiver::changed()`] waits for a new
-//! value to sent by the [`Sender`] half.
+//! value to be sent by the [`Sender`] half.
 //!
 //! # Examples
 //!
@@ -90,10 +90,11 @@ pub struct Sender<T> {
 /// Returns a reference to the inner value.
 ///
 /// Outstanding borrows hold a read lock on the inner value. This means that
-/// long lived borrows could cause the produce half to block. It is recommended
-/// to keep the borrow as short lived as possible. Additionally, if you are
+/// long-lived borrows could cause the producer half to block. It is recommended
+/// to keep the borrow as short-lived as possible. Additionally, if you are
 /// running in an environment that allows `!Send` futures, you must ensure that
-/// the returned `Ref` type is never held alive across an `.await` point.
+/// the returned `Ref` type is never held alive across an `.await` point,
+/// otherwise, it can lead to a deadlock.
 ///
 /// The priority policy of the lock is dependent on the underlying lock
 /// implementation, and this type does not guarantee that any particular policy
@@ -350,11 +351,12 @@ impl<T> Receiver<T> {
     /// [`changed`] may return immediately even if you have already seen the
     /// value with a call to `borrow`.
     ///
-    /// Outstanding borrows hold a read lock. This means that long lived borrows
-    /// could cause the send half to block. It is recommended to keep the borrow
-    /// as short lived as possible. Additionally, if you are running in an
-    /// environment that allows `!Send` futures, you must ensure that the
-    /// returned `Ref` type is never held alive across an `.await` point.
+    /// Outstanding borrows hold a read lock on the inner value. This means that
+    /// long-lived borrows could cause the producer half to block. It is recommended
+    /// to keep the borrow as short-lived as possible. Additionally, if you are
+    /// running in an environment that allows `!Send` futures, you must ensure that
+    /// the returned `Ref` type is never held alive across an `.await` point,
+    /// otherwise, it can lead to a deadlock.
     ///
     /// The priority policy of the lock is dependent on the underlying lock
     /// implementation, and this type does not guarantee that any particular policy
@@ -401,11 +403,12 @@ impl<T> Receiver<T> {
     /// will not return immediately until the [`Sender`] has modified the shared
     /// value again.
     ///
-    /// Outstanding borrows hold a read lock. This means that long lived borrows
-    /// could cause the send half to block. It is recommended to keep the borrow
-    /// as short lived as possible. Additionally, if you are running in an
-    /// environment that allows `!Send` futures, you must ensure that the
-    /// returned `Ref` type is never held alive across an `.await` point.
+    /// Outstanding borrows hold a read lock on the inner value. This means that
+    /// long-lived borrows could cause the producer half to block. It is recommended
+    /// to keep the borrow as short-lived as possible. Additionally, if you are
+    /// running in an environment that allows `!Send` futures, you must ensure that
+    /// the returned `Ref` type is never held alive across an `.await` point,
+    /// otherwise, it can lead to a deadlock.
     ///
     /// The priority policy of the lock is dependent on the underlying lock
     /// implementation, and this type does not guarantee that any particular policy
@@ -794,9 +797,12 @@ impl<T> Sender<T> {
 
     /// Returns a reference to the most recently sent value
     ///
-    /// Outstanding borrows hold a read lock. This means that long lived borrows
-    /// could cause the send half to block. It is recommended to keep the borrow
-    /// as short lived as possible.
+    /// Outstanding borrows hold a read lock on the inner value. This means that
+    /// long-lived borrows could cause the producer half to block. It is recommended
+    /// to keep the borrow as short-lived as possible. Additionally, if you are
+    /// running in an environment that allows `!Send` futures, you must ensure that
+    /// the returned `Ref` type is never held alive across an `.await` point,
+    /// otherwise, it can lead to a deadlock.
     ///
     /// # Examples
     ///

tokio/src/task/yield_now.rs

@@ -1,3 +1,5 @@
+use crate::runtime::context;
+
 use std::future::Future;
 use std::pin::Pin;
 use std::task::{Context, Poll};
@@ -49,7 +51,17 @@ pub async fn yield_now() {
             }
 
             self.yielded = true;
-            cx.waker().wake_by_ref();
+
+            let defer = context::with_defer(|rt| {
+                rt.defer(cx.waker().clone());
+            });
+
+            if defer.is_none() {
+                //  Not currently in a runtime, just notify ourselves
+                //  immediately.
+                cx.waker().wake_by_ref();
+            }
+
             Poll::Pending
         }
     }

tokio/src/time/clock.rs

@@ -65,6 +65,9 @@ cfg_test_util! {
 
         /// Instant at which the clock was last unfrozen.
         unfrozen: Option<std::time::Instant>,
+
+        /// Number of `inhibit_auto_advance` calls still in effect.
+        auto_advance_inhibit_count: usize,
     }
 
     /// Pauses time.
@@ -187,6 +190,7 @@ cfg_test_util! {
                     enable_pausing,
                     base: now,
                     unfrozen: Some(now),
+                    auto_advance_inhibit_count: 0,
                 })),
             };
 
@@ -212,9 +216,20 @@ cfg_test_util! {
             inner.unfrozen = None;
         }
 
-        pub(crate) fn is_paused(&self) -> bool {
+        /// Temporarily stop auto-advancing the clock (see `tokio::time::pause`).
+        pub(crate) fn inhibit_auto_advance(&self) {
+            let mut inner = self.inner.lock();
+            inner.auto_advance_inhibit_count += 1;
+        }
+
+        pub(crate) fn allow_auto_advance(&self) {
+            let mut inner = self.inner.lock();
+            inner.auto_advance_inhibit_count -= 1;
+        }
+
+        pub(crate) fn can_auto_advance(&self) -> bool {
             let inner = self.inner.lock();
-            inner.unfrozen.is_none()
+            inner.unfrozen.is_none() && inner.auto_advance_inhibit_count == 0
         }
 
         #[track_caller]

tokio/src/time/sleep.rs

@@ -357,7 +357,7 @@ impl Sleep {
     fn reset_inner(self: Pin<&mut Self>, deadline: Instant) {
         let mut me = self.project();
         me.entry.as_mut().reset(deadline);
-        (*me.inner).deadline = deadline;
+        (me.inner).deadline = deadline;
 
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         {

tokio/src/util/linked_list.rs

@@ -126,7 +126,7 @@ impl<L: Link> LinkedList<L, L::Target> {
     pub(crate) fn push_front(&mut self, val: L::Handle) {
         // The value should not be dropped, it is being inserted into the list
         let val = ManuallyDrop::new(val);
-        let ptr = L::as_raw(&*val);
+        let ptr = L::as_raw(&val);
         assert_ne!(self.head, Some(ptr));
         unsafe {
             L::pointers(ptr).as_mut().set_next(self.head);

tokio/src/util/once_cell.rs

@@ -25,7 +25,7 @@ impl<T> OnceCell<T> {
     /// If the `init` closure panics, then the `OnceCell` is poisoned and all
     /// future calls to `get` will panic.
     #[inline]
-    pub(crate) fn get(&self, init: fn() -> T) -> &T {
+    pub(crate) fn get(&self, init: impl FnOnce() -> T) -> &T {
         if !self.once.is_completed() {
             self.do_init(init);
         }
@@ -41,7 +41,7 @@ impl<T> OnceCell<T> {
     }
 
     #[cold]
-    fn do_init(&self, init: fn() -> T) {
+    fn do_init(&self, init: impl FnOnce() -> T) {
         let value_ptr = self.value.get() as *mut T;
 
         self.once.call_once(|| {

tokio/src/util/rand.rs

@@ -46,7 +46,7 @@ cfg_rt! {
     }
 }
 
-/// A seed for random numnber generation.
+/// A seed for random number generation.
 ///
 /// In order to make certain functions within a runtime deterministic, a seed
 /// can be specified at the time of creation.

tokio/tests/_require_full.rs

@@ -1,2 +1,8 @@
-#![cfg(not(any(feature = "full", tokio_wasm)))]
+#[cfg(not(any(feature = "full", tokio_wasm)))]
 compile_error!("run main Tokio tests with `--features full`");
+
+// CI sets `--cfg tokio_no_parking_lot` when trying to run tests with
+// `parking_lot` disabled. This check prevents "silent failure" if `parking_lot`
+// accidentally gets enabled.
+#[cfg(all(tokio_no_parking_lot, feature = "parking_lot"))]
+compile_error!("parking_lot feature enabled when it should not be");