From 8170e2787c21dd3bf9dbfda3ab8c770c16d51180 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 2 Jul 2021 23:24:02 +0200 Subject: [PATCH 01/35] sync: fix watch wrapper (#3914) --- tokio-stream/Cargo.toml | 2 +- tokio-stream/src/wrappers/watch.rs | 4 ++-- tokio-stream/tests/watch.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tokio-stream/tests/watch.rs diff --git a/tokio-stream/Cargo.toml b/tokio-stream/Cargo.toml index 5c959172a..e169a2bec 100644 --- a/tokio-stream/Cargo.toml +++ b/tokio-stream/Cargo.toml @@ -30,7 +30,7 @@ signal = ["tokio/signal"] [dependencies] futures-core = { version = "0.3.0" } pin-project-lite = "0.2.0" -tokio = { version = "1.2.0", path = "../tokio", features = ["sync"] } +tokio = { version = "1.8.0", path = "../tokio", features = ["sync"] } tokio-util = { version = "0.6.3", path = "../tokio-util", optional = true } [dev-dependencies] diff --git a/tokio-stream/src/wrappers/watch.rs b/tokio-stream/src/wrappers/watch.rs index 64688aa3e..1daca1010 100644 --- a/tokio-stream/src/wrappers/watch.rs +++ b/tokio-stream/src/wrappers/watch.rs @@ -72,10 +72,10 @@ impl Stream for WatchStream { type Item = T; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let (result, rx) = ready!(self.inner.poll(cx)); + let (result, mut rx) = ready!(self.inner.poll(cx)); match result { Ok(_) => { - let received = (*rx.borrow()).clone(); + let received = (*rx.borrow_and_update()).clone(); self.inner.set(make_future(rx)); Poll::Ready(Some(received)) } diff --git a/tokio-stream/tests/watch.rs b/tokio-stream/tests/watch.rs new file mode 100644 index 000000000..92bcdf497 --- /dev/null +++ b/tokio-stream/tests/watch.rs @@ -0,0 +1,27 @@ +use tokio::sync::watch; +use tokio_stream::wrappers::WatchStream; +use tokio_stream::StreamExt; + +#[tokio::test] +async fn message_not_twice() { + let (tx, rx) = watch::channel("hello"); + + let mut counter = 0; + let mut stream = WatchStream::new(rx).map(move |payload| { + println!("{}", payload); + if payload == "goodbye" { + counter += 1; + } + if counter >= 2 { + panic!("too many goodbyes"); + } + }); + + let task = tokio::spawn(async move { while stream.next().await.is_some() {} }); + + // Send goodbye just once + tx.send("goodbye").unwrap(); + + drop(tx); + task.await.unwrap(); +} From 37e60fc7f9b6a1e9c89e1b05b8e38d88e4f829c9 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jul 2021 09:58:49 +0200 Subject: [PATCH 02/35] chore: fix new clippy complaint (#3915) --- tokio-stream/src/stream_ext/collect.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio-stream/src/stream_ext/collect.rs b/tokio-stream/src/stream_ext/collect.rs index 23f48b049..a33a6d669 100644 --- a/tokio-stream/src/stream_ext/collect.rs +++ b/tokio-stream/src/stream_ext/collect.rs @@ -113,7 +113,7 @@ impl> sealed::FromStreamPriv for String { } fn finalize(_: sealed::Internal, collection: &mut String) -> String { - mem::replace(collection, String::new()) + mem::take(collection) } } @@ -132,7 +132,7 @@ impl sealed::FromStreamPriv for Vec { } fn finalize(_: sealed::Internal, collection: &mut Vec) -> Vec { - mem::replace(collection, vec![]) + mem::take(collection) } } From 2e7de1ae1d4efb6d0fc6053a1fa22c1e0f91cc52 Mon Sep 17 00:00:00 2001 From: oiovoyo Date: Mon, 5 Jul 2021 22:10:55 +0800 Subject: [PATCH 03/35] stream: modify HashMap to StreamMap in example. (#3925) --- tokio-stream/src/stream_map.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tokio-stream/src/stream_map.rs b/tokio-stream/src/stream_map.rs index 7fc136ff7..9dc529ac0 100644 --- a/tokio-stream/src/stream_map.rs +++ b/tokio-stream/src/stream_map.rs @@ -364,11 +364,11 @@ impl StreamMap { /// # Examples /// /// ``` - /// use std::collections::HashMap; + /// use tokio_stream::{StreamMap, pending}; /// - /// let mut a = HashMap::new(); + /// let mut a = StreamMap::new(); /// assert!(a.is_empty()); - /// a.insert(1, "a"); + /// a.insert(1, pending::()); /// assert!(!a.is_empty()); /// ``` pub fn is_empty(&self) -> bool { From 8148b2107c754cf5460d37ddedfff841de97007f Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Tue, 6 Jul 2021 10:15:15 +0100 Subject: [PATCH 04/35] net: make ucred return pid field for OpenBSD (#3919) --- tokio/src/net/unix/ucred.rs | 66 +++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/tokio/src/net/unix/ucred.rs b/tokio/src/net/unix/ucred.rs index 6310183d7..49c714284 100644 --- a/tokio/src/net/unix/ucred.rs +++ b/tokio/src/net/unix/ucred.rs @@ -31,15 +31,13 @@ impl UCred { } } -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] pub(crate) use self::impl_linux::get_peer_cred; -#[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd" -))] +#[cfg(any(target_os = "netbsd"))] +pub(crate) use self::impl_netbsd::get_peer_cred; + +#[cfg(any(target_os = "dragonfly", target_os = "freebsd"))] pub(crate) use self::impl_bsd::get_peer_cred; #[cfg(any(target_os = "macos", target_os = "ios"))] @@ -48,13 +46,16 @@ pub(crate) use self::impl_macos::get_peer_cred; #[cfg(any(target_os = "solaris", target_os = "illumos"))] pub(crate) use self::impl_solaris::get_peer_cred; -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] pub(crate) mod impl_linux { use crate::net::unix::UnixStream; use libc::{c_void, getsockopt, socklen_t, SOL_SOCKET, SO_PEERCRED}; use std::{io, mem}; + #[cfg(target_os = "openbsd")] + use libc::sockpeercred as ucred; + #[cfg(any(target_os = "linux", target_os = "android"))] use libc::ucred; pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result { @@ -97,12 +98,49 @@ pub(crate) mod impl_linux { } } -#[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd" -))] +#[cfg(any(target_os = "netbsd"))] +pub(crate) mod impl_netbsd { + use crate::net::unix::UnixStream; + + use libc::{c_void, getsockopt, socklen_t, unpcbid, LOCAL_PEEREID, SOL_SOCKET}; + use std::io; + use std::mem::size_of; + use std::os::unix::io::AsRawFd; + + pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result { + unsafe { + let raw_fd = sock.as_raw_fd(); + + let mut unpcbid = unpcbid { + unp_pid: 0, + unp_euid: 0, + unp_egid: 0, + }; + + let unpcbid_size = size_of::(); + let mut unpcbid_size = unpcbid_size as socklen_t; + + let ret = getsockopt( + raw_fd, + SOL_SOCKET, + LOCAL_PEEREID, + &mut unpcbid as *mut unpcbid as *mut c_void, + &mut unpcbid_size, + ); + if ret == 0 && unpcbid_size as usize == size_of::() { + Ok(super::UCred { + uid: unpcbid.unp_euid, + gid: unpcbid.unp_egid, + pid: Some(unpcbid.unp_pid), + }) + } else { + Err(io::Error::last_os_error()) + } + } + } +} + +#[cfg(any(target_os = "dragonfly", target_os = "freebsd"))] pub(crate) mod impl_bsd { use crate::net::unix::UnixStream; From 80f0801e1905737228c58b45f09394a2f7378685 Mon Sep 17 00:00:00 2001 From: Blas Rodriguez Irizar Date: Tue, 6 Jul 2021 15:01:13 +0200 Subject: [PATCH 05/35] tokio-macros: compat with clippy::unwrap_used (#3926) --- tokio-macros/src/entry.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tokio-macros/src/entry.rs b/tokio-macros/src/entry.rs index ddc19585d..c5db13cc4 100644 --- a/tokio-macros/src/entry.rs +++ b/tokio-macros/src/entry.rs @@ -201,12 +201,15 @@ fn parse_knobs( for arg in args { match arg { syn::NestedMeta::Meta(syn::Meta::NameValue(namevalue)) => { - let ident = namevalue.path.get_ident(); - if ident.is_none() { - let msg = "Must have specified ident"; - return Err(syn::Error::new_spanned(namevalue, msg)); - } - match ident.unwrap().to_string().to_lowercase().as_str() { + let ident = namevalue + .path + .get_ident() + .ok_or_else(|| { + syn::Error::new_spanned(&namevalue, "Must have specified ident") + })? + .to_string() + .to_lowercase(); + match ident.as_str() { "worker_threads" => { config.set_worker_threads( namevalue.lit.clone(), @@ -239,12 +242,11 @@ fn parse_knobs( } } syn::NestedMeta::Meta(syn::Meta::Path(path)) => { - let ident = path.get_ident(); - if ident.is_none() { - let msg = "Must have specified ident"; - return Err(syn::Error::new_spanned(path, msg)); - } - let name = ident.unwrap().to_string().to_lowercase(); + let name = path + .get_ident() + .ok_or_else(|| syn::Error::new_spanned(&path, "Must have specified ident"))? + .to_string() + .to_lowercase(); let msg = match name.as_str() { "threaded_scheduler" | "multi_thread" => { format!( @@ -326,11 +328,11 @@ fn parse_knobs( #rt .enable_all() .build() - .unwrap() + .expect("Failed building the Runtime") .block_on(async #body) } }) - .unwrap(); + .expect("Parsing failure"); input.block.brace_token = brace_token; let result = quote! { From 4818c2ed052549825da7bd59df2cf0da808e66c4 Mon Sep 17 00:00:00 2001 From: Blas Rodriguez Irizar Date: Tue, 6 Jul 2021 16:25:13 +0200 Subject: [PATCH 06/35] fs: document performance considerations (#3920) --- tokio-macros/src/lib.rs | 2 +- tokio-stream/src/lib.rs | 2 +- tokio-test/src/lib.rs | 2 +- tokio-util/src/lib.rs | 2 +- tokio/src/fs/read.rs | 4 ++++ tokio/src/fs/read_dir.rs | 5 +++++ tokio/src/fs/read_to_string.rs | 4 ++++ tokio/src/fs/write.rs | 4 ++++ tokio/src/lib.rs | 2 +- 9 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tokio-macros/src/lib.rs b/tokio-macros/src/lib.rs index 18de77281..df18616e9 100644 --- a/tokio-macros/src/lib.rs +++ b/tokio-macros/src/lib.rs @@ -5,7 +5,7 @@ rust_2018_idioms, unreachable_pub )] -#![cfg_attr(docsrs, deny(broken_intra_doc_links))] +#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) diff --git a/tokio-stream/src/lib.rs b/tokio-stream/src/lib.rs index af99488b1..b7f232fda 100644 --- a/tokio-stream/src/lib.rs +++ b/tokio-stream/src/lib.rs @@ -10,7 +10,7 @@ unreachable_pub )] #![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(docsrs, deny(broken_intra_doc_links))] +#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) diff --git a/tokio-test/src/lib.rs b/tokio-test/src/lib.rs index 4e6b86904..7bba3eeda 100644 --- a/tokio-test/src/lib.rs +++ b/tokio-test/src/lib.rs @@ -4,7 +4,7 @@ rust_2018_idioms, unreachable_pub )] -#![cfg_attr(docsrs, deny(broken_intra_doc_links))] +#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) diff --git a/tokio-util/src/lib.rs b/tokio-util/src/lib.rs index 4d0832fc4..0b3e59623 100644 --- a/tokio-util/src/lib.rs +++ b/tokio-util/src/lib.rs @@ -5,7 +5,7 @@ rust_2018_idioms, unreachable_pub )] -#![cfg_attr(docsrs, deny(broken_intra_doc_links))] +#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) diff --git a/tokio/src/fs/read.rs b/tokio/src/fs/read.rs index 2d80eb5bd..ada5ba391 100644 --- a/tokio/src/fs/read.rs +++ b/tokio/src/fs/read.rs @@ -13,8 +13,12 @@ use std::{io, path::Path}; /// buffer based on the file size when available, so it is generally faster than /// reading into a vector created with `Vec::new()`. /// +/// This operation is implemented by running the equivalent blocking operation +/// on a separate thread pool using [`spawn_blocking`]. +/// /// [`File::open`]: super::File::open /// [`read_to_end`]: crate::io::AsyncReadExt::read_to_end +/// [`spawn_blocking`]: crate::task::spawn_blocking /// /// # Errors /// diff --git a/tokio/src/fs/read_dir.rs b/tokio/src/fs/read_dir.rs index aedaf7b92..9a0334306 100644 --- a/tokio/src/fs/read_dir.rs +++ b/tokio/src/fs/read_dir.rs @@ -13,6 +13,11 @@ use std::task::Poll; /// Returns a stream over the entries within a directory. /// /// This is an async version of [`std::fs::read_dir`](std::fs::read_dir) +/// +/// This operation is implemented by running the equivalent blocking +/// operation on a separate thread pool using [`spawn_blocking`]. +/// +/// [`spawn_blocking`]: crate::task::spawn_blocking pub async fn read_dir(path: impl AsRef) -> io::Result { let path = path.as_ref().to_owned(); let std = asyncify(|| std::fs::read_dir(path)).await?; diff --git a/tokio/src/fs/read_to_string.rs b/tokio/src/fs/read_to_string.rs index 4f37986d6..26228d98c 100644 --- a/tokio/src/fs/read_to_string.rs +++ b/tokio/src/fs/read_to_string.rs @@ -7,6 +7,10 @@ use std::{io, path::Path}; /// /// This is the async equivalent of [`std::fs::read_to_string`][std]. /// +/// This operation is implemented by running the equivalent blocking operation +/// on a separate thread pool using [`spawn_blocking`]. +/// +/// [`spawn_blocking`]: crate::task::spawn_blocking /// [std]: fn@std::fs::read_to_string /// /// # Examples diff --git a/tokio/src/fs/write.rs b/tokio/src/fs/write.rs index 0ed908271..28606fb36 100644 --- a/tokio/src/fs/write.rs +++ b/tokio/src/fs/write.rs @@ -7,6 +7,10 @@ use std::{io, path::Path}; /// /// This is the async equivalent of [`std::fs::write`][std]. /// +/// This operation is implemented by running the equivalent blocking operation +/// on a separate thread pool using [`spawn_blocking`]. +/// +/// [`spawn_blocking`]: crate::task::spawn_blocking /// [std]: fn@std::fs::write /// /// # Examples diff --git a/tokio/src/lib.rs b/tokio/src/lib.rs index c74b9643f..fbf28ff8d 100644 --- a/tokio/src/lib.rs +++ b/tokio/src/lib.rs @@ -10,7 +10,7 @@ unreachable_pub )] #![deny(unused_must_use)] -#![cfg_attr(docsrs, deny(broken_intra_doc_links))] +#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) From be26ca762521a0f5db58fe3814d738719ac20b77 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Jul 2021 10:17:47 +0200 Subject: [PATCH 07/35] sync: wrap state in helper struct (#3922) --- tokio/src/sync/watch.rs | 96 ++++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/tokio/src/sync/watch.rs b/tokio/src/sync/watch.rs index 7852b0cb1..3d52e4b0a 100644 --- a/tokio/src/sync/watch.rs +++ b/tokio/src/sync/watch.rs @@ -56,7 +56,7 @@ use crate::sync::notify::Notify; use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::atomic::Ordering::{Relaxed, SeqCst}; +use crate::loom::sync::atomic::Ordering::Relaxed; use crate::loom::sync::{Arc, RwLock, RwLockReadGuard}; use std::ops; @@ -74,7 +74,7 @@ pub struct Receiver { shared: Arc>, /// Last observed version - version: usize, + version: Version, } /// Sends values to the associated [`Receiver`](struct@Receiver). @@ -104,7 +104,7 @@ struct Shared { /// /// The lowest bit represents a "closed" state. The rest of the bits /// represent the current version. - version: AtomicUsize, + state: AtomicState, /// Tracks the number of `Receiver` instances ref_count_rx: AtomicUsize, @@ -152,7 +152,69 @@ pub mod error { impl std::error::Error for RecvError {} } -const CLOSED: usize = 1; +use self::state::{AtomicState, Version}; +mod state { + use crate::loom::sync::atomic::AtomicUsize; + use crate::loom::sync::atomic::Ordering::SeqCst; + + const CLOSED: usize = 1; + + /// The version part of the state. The lowest bit is always zero. + #[derive(Copy, Clone, Debug, Eq, PartialEq)] + pub(super) struct Version(usize); + + /// Snapshot of the state. The first bit is used as the CLOSED bit. + /// The remaining bits are used as the version. + #[derive(Copy, Clone, Debug)] + pub(super) struct StateSnapshot(usize); + + /// The state stored in an atomic integer. + #[derive(Debug)] + pub(super) struct AtomicState(AtomicUsize); + + impl Version { + /// Get the initial version when creating the channel. + pub(super) fn initial() -> Self { + Version(0) + } + } + + impl StateSnapshot { + /// Extract the version from the state. + pub(super) fn version(self) -> Version { + Version(self.0 & !CLOSED) + } + + /// Is the closed bit set? + pub(super) fn is_closed(self) -> bool { + (self.0 & CLOSED) == CLOSED + } + } + + impl AtomicState { + /// Create a new `AtomicState` that is not closed and which has the + /// version set to `Version::initial()`. + pub(super) fn new() -> Self { + AtomicState(AtomicUsize::new(0)) + } + + /// Load the current value of the state. + pub(super) fn load(&self) -> StateSnapshot { + StateSnapshot(self.0.load(SeqCst)) + } + + /// Increment the version counter. + pub(super) fn increment_version(&self) { + // Increment by two to avoid touching the CLOSED bit. + self.0.fetch_add(2, SeqCst); + } + + /// Set the closed bit in the state. + pub(super) fn set_closed(&self) { + self.0.fetch_or(CLOSED, SeqCst); + } + } +} /// Creates a new watch channel, returning the "send" and "receive" handles. /// @@ -184,7 +246,7 @@ const CLOSED: usize = 1; pub fn channel(init: T) -> (Sender, Receiver) { let shared = Arc::new(Shared { value: RwLock::new(init), - version: AtomicUsize::new(0), + state: AtomicState::new(), ref_count_rx: AtomicUsize::new(1), notify_rx: Notify::new(), notify_tx: Notify::new(), @@ -194,13 +256,16 @@ pub fn channel(init: T) -> (Sender, Receiver) { shared: shared.clone(), }; - let rx = Receiver { shared, version: 0 }; + let rx = Receiver { + shared, + version: Version::initial(), + }; (tx, rx) } impl Receiver { - fn from_shared(version: usize, shared: Arc>) -> Self { + fn from_shared(version: Version, shared: Arc>) -> Self { // No synchronization necessary as this is only used as a counter and // not memory access. shared.ref_count_rx.fetch_add(1, Relaxed); @@ -247,7 +312,7 @@ impl Receiver { /// [`changed`]: Receiver::changed pub fn borrow_and_update(&mut self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - self.version = self.shared.version.load(SeqCst) & !CLOSED; + self.version = self.shared.state.load().version(); Ref { inner } } @@ -315,11 +380,11 @@ impl Receiver { fn maybe_changed( shared: &Shared, - version: &mut usize, + version: &mut Version, ) -> Option> { // Load the version from the state - let state = shared.version.load(SeqCst); - let new_version = state & !CLOSED; + let state = shared.state.load(); + let new_version = state.version(); if *version != new_version { // Observe the new version and return @@ -327,7 +392,7 @@ fn maybe_changed( return Some(Ok(())); } - if CLOSED == state & CLOSED { + if state.is_closed() { // All receivers have dropped. return Some(Err(error::RecvError(()))); } @@ -368,8 +433,7 @@ impl Sender { let mut lock = self.shared.value.write().unwrap(); *lock = value; - // Update the version. 2 is used so that the CLOSED bit is not set. - self.shared.version.fetch_add(2, SeqCst); + self.shared.state.increment_version(); // Release the write lock. // @@ -463,7 +527,7 @@ impl Sender { cfg_signal_internal! { pub(crate) fn subscribe(&self) -> Receiver { let shared = self.shared.clone(); - let version = shared.version.load(SeqCst); + let version = shared.state.load().version(); Receiver::from_shared(version, shared) } @@ -494,7 +558,7 @@ impl Sender { impl Drop for Sender { fn drop(&mut self) { - self.shared.version.fetch_or(CLOSED, SeqCst); + self.shared.state.set_closed(); self.shared.notify_rx.notify_waiters(); } } From 51fad066e2f155bb220aa8486ce51bd6bcbec1d8 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Jul 2021 10:52:53 +0200 Subject: [PATCH 08/35] bench: update spawn benchmarks (#3927) --- benches/spawn.rs | 94 +++++++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/benches/spawn.rs b/benches/spawn.rs index 72a403575..7d4b81373 100644 --- a/benches/spawn.rs +++ b/benches/spawn.rs @@ -2,63 +2,83 @@ //! This essentially measure the time to enqueue a task in the local and remote //! case. +#[macro_use] +extern crate bencher; + use bencher::{black_box, Bencher}; async fn work() -> usize { let val = 1 + 1; + tokio::task::yield_now().await; black_box(val) } -fn basic_scheduler_local_spawn(bench: &mut Bencher) { +fn basic_scheduler_spawn(bench: &mut Bencher) { let runtime = tokio::runtime::Builder::new_current_thread() .build() .unwrap(); - runtime.block_on(async { - bench.iter(|| { - let h = tokio::spawn(work()); - black_box(h); - }) - }); -} - -fn threaded_scheduler_local_spawn(bench: &mut Bencher) { - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - runtime.block_on(async { - bench.iter(|| { - let h = tokio::spawn(work()); - black_box(h); - }) - }); -} - -fn basic_scheduler_remote_spawn(bench: &mut Bencher) { - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - bench.iter(|| { - let h = runtime.spawn(work()); - black_box(h); + runtime.block_on(async { + let h = tokio::spawn(work()); + assert_eq!(h.await.unwrap(), 2); + }); }); } -fn threaded_scheduler_remote_spawn(bench: &mut Bencher) { - let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap(); - +fn basic_scheduler_spawn10(bench: &mut Bencher) { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); bench.iter(|| { - let h = runtime.spawn(work()); - black_box(h); + runtime.block_on(async { + let mut handles = Vec::with_capacity(10); + for _ in 0..10 { + handles.push(tokio::spawn(work())); + } + for handle in handles { + assert_eq!(handle.await.unwrap(), 2); + } + }); + }); +} + +fn threaded_scheduler_spawn(bench: &mut Bencher) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + bench.iter(|| { + runtime.block_on(async { + let h = tokio::spawn(work()); + assert_eq!(h.await.unwrap(), 2); + }); + }); +} + +fn threaded_scheduler_spawn10(bench: &mut Bencher) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + bench.iter(|| { + runtime.block_on(async { + let mut handles = Vec::with_capacity(10); + for _ in 0..10 { + handles.push(tokio::spawn(work())); + } + for handle in handles { + assert_eq!(handle.await.unwrap(), 2); + } + }); }); } bencher::benchmark_group!( spawn, - basic_scheduler_local_spawn, - threaded_scheduler_local_spawn, - basic_scheduler_remote_spawn, - threaded_scheduler_remote_spawn + basic_scheduler_spawn, + basic_scheduler_spawn10, + threaded_scheduler_spawn, + threaded_scheduler_spawn10, ); bencher::benchmark_main!(spawn); From e2589a0e401e27457618920e66caa0f14e9a69ad Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Jul 2021 10:53:57 +0200 Subject: [PATCH 09/35] runtime: add OwnedTasks (#3909) --- tokio/src/runtime/basic_scheduler.rs | 98 +++----------- tokio/src/runtime/task/core.rs | 11 -- tokio/src/runtime/task/list.rs | 33 +++++ tokio/src/runtime/task/mod.rs | 15 +-- tokio/src/runtime/task/stack.rs | 83 ------------ tokio/src/runtime/tests/task.rs | 39 +----- tokio/src/runtime/thread_pool/worker.rs | 164 ++++-------------------- tokio/src/util/linked_list.rs | 49 ------- 8 files changed, 87 insertions(+), 405 deletions(-) create mode 100644 tokio/src/runtime/task/list.rs delete mode 100644 tokio/src/runtime/task/stack.rs diff --git a/tokio/src/runtime/basic_scheduler.rs b/tokio/src/runtime/basic_scheduler.rs index 13dfb6973..9efe3844a 100644 --- a/tokio/src/runtime/basic_scheduler.rs +++ b/tokio/src/runtime/basic_scheduler.rs @@ -2,16 +2,14 @@ use crate::future::poll_fn; use crate::loom::sync::atomic::AtomicBool; use crate::loom::sync::Mutex; use crate::park::{Park, Unpark}; -use crate::runtime::task::{self, JoinHandle, Schedule, Task}; +use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task}; use crate::sync::notify::Notify; -use crate::util::linked_list::{Link, LinkedList}; use crate::util::{waker_ref, Wake, WakerRef}; use std::cell::RefCell; use std::collections::VecDeque; use std::fmt; use std::future::Future; -use std::ptr::NonNull; use std::sync::atomic::Ordering::{AcqRel, Acquire, Release}; use std::sync::Arc; use std::task::Poll::{Pending, Ready}; @@ -57,9 +55,6 @@ pub(crate) struct Spawner { } struct Tasks { - /// Collection of all active tasks spawned onto this executor. - owned: LinkedList>, > as Link>::Target>, - /// Local run queue. /// /// Tasks notified from the current thread are pushed into this queue. @@ -69,23 +64,23 @@ struct Tasks { /// A remote scheduler entry. /// /// These are filled in by remote threads sending instructions to the scheduler. -enum Entry { +enum RemoteMsg { /// A remote thread wants to spawn a task. Schedule(task::Notified>), - /// A remote thread wants a task to be released by the scheduler. We only - /// have access to its header. - Release(NonNull), } // Safety: Used correctly, the task header is "thread safe". Ultimately the task // is owned by the current thread executor, for which this instruction is being // sent. -unsafe impl Send for Entry {} +unsafe impl Send for RemoteMsg {} /// Scheduler state shared between threads. struct Shared { /// Remote run queue. None if the `Runtime` has been dropped. - queue: Mutex>>, + queue: Mutex>>, + + /// Collection of all active tasks spawned onto this executor. + owned: OwnedTasks>, /// Unpark the blocked thread. unpark: Box, @@ -125,6 +120,7 @@ impl BasicScheduler

{ let spawner = Spawner { shared: Arc::new(Shared { queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), + owned: OwnedTasks::new(), unpark: unpark as Box, woken: AtomicBool::new(false), }), @@ -132,7 +128,6 @@ impl BasicScheduler

{ let inner = Mutex::new(Some(Inner { tasks: Some(Tasks { - owned: LinkedList::new(), queue: VecDeque::with_capacity(INITIAL_CAPACITY), }), spawner: spawner.clone(), @@ -227,7 +222,7 @@ impl Inner

{ .borrow_mut() .queue .pop_front() - .map(Entry::Schedule) + .map(RemoteMsg::Schedule) }) } else { context @@ -235,7 +230,7 @@ impl Inner

{ .borrow_mut() .queue .pop_front() - .map(Entry::Schedule) + .map(RemoteMsg::Schedule) .or_else(|| scheduler.spawner.pop()) }; @@ -251,26 +246,7 @@ impl Inner

{ }; match entry { - Entry::Schedule(task) => crate::coop::budget(|| task.run()), - Entry::Release(ptr) => { - // Safety: the task header is only legally provided - // internally in the header, so we know that it is a - // valid (or in particular *allocated*) header that - // is part of the linked list. - unsafe { - let removed = context.tasks.borrow_mut().owned.remove(ptr); - - // TODO: This seems like it should hold, because - // there doesn't seem to be an avenue for anyone - // else to fiddle with the owned tasks - // collection *after* a remote thread has marked - // it as released, and at that point, the only - // location at which it can be removed is here - // or in the Drop implementation of the - // scheduler. - debug_assert!(removed.is_some()); - } - } + RemoteMsg::Schedule(task) => crate::coop::budget(|| task.run()), } } @@ -335,14 +311,7 @@ impl Drop for BasicScheduler

{ }; enter(&mut inner, |scheduler, context| { - // Loop required here to ensure borrow is dropped between iterations - #[allow(clippy::while_let_loop)] - loop { - let task = match context.tasks.borrow_mut().owned.pop_back() { - Some(task) => task, - None => break, - }; - + while let Some(task) = context.shared.owned.pop_back() { task.shutdown(); } @@ -358,13 +327,9 @@ impl Drop for BasicScheduler

{ if let Some(remote_queue) = remote_queue.take() { for entry in remote_queue { match entry { - Entry::Schedule(task) => { + RemoteMsg::Schedule(task) => { task.shutdown(); } - Entry::Release(..) => { - // Do nothing, each entry in the linked list was *just* - // dropped by the scheduler above. - } } } } @@ -375,7 +340,7 @@ impl Drop for BasicScheduler

{ // The assert below is unrelated to this mutex. drop(remote_queue); - assert!(context.tasks.borrow().owned.is_empty()); + assert!(context.shared.owned.is_empty()); }); } } @@ -400,7 +365,7 @@ impl Spawner { handle } - fn pop(&self) -> Option { + fn pop(&self) -> Option { match self.shared.queue.lock().as_mut() { Some(queue) => queue.pop_front(), None => None, @@ -430,39 +395,14 @@ impl Schedule for Arc { fn bind(task: Task) -> Arc { CURRENT.with(|maybe_cx| { let cx = maybe_cx.expect("scheduler context missing"); - cx.tasks.borrow_mut().owned.push_front(task); + cx.shared.owned.push_front(task); cx.shared.clone() }) } fn release(&self, task: &Task) -> Option> { - CURRENT.with(|maybe_cx| { - let ptr = NonNull::from(task.header()); - - if let Some(cx) = maybe_cx { - // safety: the task is inserted in the list in `bind`. - unsafe { cx.tasks.borrow_mut().owned.remove(ptr) } - } else { - // By sending an `Entry::Release` to the runtime, we ask the - // runtime to remove this task from the linked list in - // `Tasks::owned`. - // - // If the queue is `None`, then the task was already removed - // from that list in the destructor of `BasicScheduler`. We do - // not do anything in this case for the same reason that - // `Entry::Release` messages are ignored in the remote queue - // drain loop of `BasicScheduler`'s destructor. - if let Some(queue) = self.queue.lock().as_mut() { - queue.push_back(Entry::Release(ptr)); - } - - self.unpark.unpark(); - // Returning `None` here prevents the task plumbing from being - // freed. It is then up to the scheduler through the queue we - // just added to, or its Drop impl to free the task. - None - } - }) + // SAFETY: Inserted into the list in bind above. + unsafe { self.owned.remove(task) } } fn schedule(&self, task: task::Notified) { @@ -473,7 +413,7 @@ impl Schedule for Arc { _ => { let mut guard = self.queue.lock(); if let Some(queue) = guard.as_mut() { - queue.push_back(Entry::Schedule(task)); + queue.push_back(RemoteMsg::Schedule(task)); drop(guard); self.unpark.unpark(); } else { diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index 428c921fe..e4624c7b7 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -66,9 +66,6 @@ pub(crate) struct Header { /// Pointer to next task, used with the injection queue pub(crate) queue_next: UnsafeCell>>, - /// Pointer to the next task in the transfer stack - pub(super) stack_next: UnsafeCell>>, - /// Table of function pointers for executing actions on the task. pub(super) vtable: &'static Vtable, @@ -104,7 +101,6 @@ impl Cell { state, owned: UnsafeCell::new(linked_list::Pointers::new()), queue_next: UnsafeCell::new(None), - stack_next: UnsafeCell::new(None), vtable: raw::vtable::(), #[cfg(all(tokio_unstable, feature = "tracing"))] id, @@ -299,13 +295,6 @@ impl CoreStage { cfg_rt_multi_thread! { impl Header { - pub(crate) fn shutdown(&self) { - use crate::runtime::task::RawTask; - - let task = unsafe { RawTask::from_raw(self.into()) }; - task.shutdown(); - } - pub(crate) unsafe fn set_next(&self, next: Option>) { self.queue_next.with_mut(|ptr| *ptr = next); } diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs new file mode 100644 index 000000000..45e22a72a --- /dev/null +++ b/tokio/src/runtime/task/list.rs @@ -0,0 +1,33 @@ +use crate::loom::sync::Mutex; +use crate::runtime::task::Task; +use crate::util::linked_list::{Link, LinkedList}; + +pub(crate) struct OwnedTasks { + list: Mutex, as Link>::Target>>, +} + +impl OwnedTasks { + pub(crate) fn new() -> Self { + Self { + list: Mutex::new(LinkedList::new()), + } + } + + pub(crate) fn push_front(&self, task: Task) { + self.list.lock().push_front(task); + } + + pub(crate) fn pop_back(&self) -> Option> { + self.list.lock().pop_back() + } + + /// The caller must ensure that if the provided task is stored in a + /// linked list, then it is in this linked list. + pub(crate) unsafe fn remove(&self, task: &Task) -> Option> { + self.list.lock().remove(task.header().into()) + } + + pub(crate) fn is_empty(&self) -> bool { + self.list.lock().is_empty() + } +} diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 58b8c2a15..6b1b8c638 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -13,6 +13,9 @@ mod join; #[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411 pub use self::join::JoinHandle; +mod list; +pub(super) use self::list::OwnedTasks; + mod raw; use self::raw::RawTask; @@ -21,11 +24,6 @@ use self::state::State; mod waker; -cfg_rt_multi_thread! { - mod stack; - pub(crate) use self::stack::TransferStack; -} - use crate::future::Future; use crate::util::linked_list; @@ -62,11 +60,10 @@ pub(crate) trait Schedule: Sync + Sized + 'static { fn bind(task: Task) -> Self; /// The task has completed work and is ready to be released. The scheduler - /// is free to drop it whenever. + /// should release it immediately and return it. The task module will batch + /// the ref-dec with setting other options. /// - /// If the scheduler can immediately release the task, it should return - /// it as part of the function. This enables the task module to batch - /// the ref-dec with other options. + /// If the scheduler has already released the task, then None is returned. fn release(&self, task: &Task) -> Option>; /// Schedule the task diff --git a/tokio/src/runtime/task/stack.rs b/tokio/src/runtime/task/stack.rs deleted file mode 100644 index 9dd8d3f43..000000000 --- a/tokio/src/runtime/task/stack.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::loom::sync::atomic::AtomicPtr; -use crate::runtime::task::{Header, Task}; - -use std::marker::PhantomData; -use std::ptr::{self, NonNull}; -use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; - -/// Concurrent stack of tasks, used to pass ownership of a task from one worker -/// to another. -pub(crate) struct TransferStack { - head: AtomicPtr

, - _p: PhantomData, -} - -impl TransferStack { - pub(crate) fn new() -> TransferStack { - TransferStack { - head: AtomicPtr::new(ptr::null_mut()), - _p: PhantomData, - } - } - - pub(crate) fn push(&self, task: Task) { - let task = task.into_raw(); - - // We don't care about any memory associated w/ setting the `head` - // field, just the current value. - // - // The compare-exchange creates a release sequence. - let mut curr = self.head.load(Relaxed); - - loop { - unsafe { - task.as_ref() - .stack_next - .with_mut(|ptr| *ptr = NonNull::new(curr)) - }; - - let res = self - .head - .compare_exchange(curr, task.as_ptr() as *mut _, Release, Relaxed); - - match res { - Ok(_) => return, - Err(actual) => { - curr = actual; - } - } - } - } - - pub(crate) fn drain(&self) -> impl Iterator> { - struct Iter(Option>, PhantomData); - - impl Iterator for Iter { - type Item = Task; - - fn next(&mut self) -> Option> { - let task = self.0?; - - // Move the cursor forward - self.0 = unsafe { task.as_ref().stack_next.with(|ptr| *ptr) }; - - // Return the task - unsafe { Some(Task::from_raw(task)) } - } - } - - impl Drop for Iter { - fn drop(&mut self) { - use std::process; - - if self.0.is_some() { - // we have bugs - process::abort(); - } - } - } - - let ptr = self.head.swap(ptr::null_mut(), Acquire); - Iter(NonNull::new(ptr), PhantomData) - } -} diff --git a/tokio/src/runtime/tests/task.rs b/tokio/src/runtime/tests/task.rs index 7c2012523..1f3e89d76 100644 --- a/tokio/src/runtime/tests/task.rs +++ b/tokio/src/runtime/tests/task.rs @@ -1,5 +1,4 @@ -use crate::runtime::task::{self, Schedule, Task}; -use crate::util::linked_list::{Link, LinkedList}; +use crate::runtime::task::{self, OwnedTasks, Schedule, Task}; use crate::util::TryLock; use std::collections::VecDeque; @@ -51,10 +50,9 @@ fn with(f: impl FnOnce(Runtime)) { let _reset = Reset; let rt = Runtime(Arc::new(Inner { - released: task::TransferStack::new(), + owned: OwnedTasks::new(), core: TryLock::new(Core { queue: VecDeque::new(), - tasks: LinkedList::new(), }), })); @@ -66,13 +64,12 @@ fn with(f: impl FnOnce(Runtime)) { struct Runtime(Arc); struct Inner { - released: task::TransferStack, core: TryLock, + owned: OwnedTasks, } struct Core { queue: VecDeque>, - tasks: LinkedList, as Link>::Target>, } static CURRENT: TryLock> = TryLock::new(None); @@ -91,8 +88,6 @@ impl Runtime { task.run(); } - self.0.maintenance(); - n } @@ -107,7 +102,7 @@ impl Runtime { fn shutdown(&self) { let mut core = self.0.core.try_lock().unwrap(); - for task in core.tasks.iter() { + while let Some(task) = self.0.owned.pop_back() { task.shutdown(); } @@ -117,40 +112,20 @@ impl Runtime { drop(core); - while !self.0.core.try_lock().unwrap().tasks.is_empty() { - self.0.maintenance(); - } - } -} - -impl Inner { - fn maintenance(&self) { - use std::mem::ManuallyDrop; - - for task in self.released.drain() { - let task = ManuallyDrop::new(task); - - // safety: see worker.rs - unsafe { - let ptr = task.header().into(); - self.core.try_lock().unwrap().tasks.remove(ptr); - } - } + assert!(self.0.owned.is_empty()); } } impl Schedule for Runtime { fn bind(task: Task) -> Runtime { let rt = CURRENT.try_lock().unwrap().as_ref().unwrap().clone(); - rt.0.core.try_lock().unwrap().tasks.push_front(task); + rt.0.owned.push_front(task); rt } fn release(&self, task: &Task) -> Option> { // safety: copying worker.rs - let task = unsafe { Task::from_raw(task.header().into()) }; - self.0.released.push(task); - None + unsafe { self.0.owned.remove(task) } } fn schedule(&self, task: task::Notified) { diff --git a/tokio/src/runtime/thread_pool/worker.rs b/tokio/src/runtime/thread_pool/worker.rs index 70cbddbd0..4ae0f5a25 100644 --- a/tokio/src/runtime/thread_pool/worker.rs +++ b/tokio/src/runtime/thread_pool/worker.rs @@ -11,9 +11,9 @@ use crate::park::{Park, Unpark}; use crate::runtime; use crate::runtime::enter::EnterContext; use crate::runtime::park::{Parker, Unparker}; +use crate::runtime::task::OwnedTasks; use crate::runtime::thread_pool::{AtomicCell, Idle}; use crate::runtime::{queue, task}; -use crate::util::linked_list::{Link, LinkedList}; use crate::util::FastRand; use std::cell::RefCell; @@ -53,9 +53,6 @@ struct Core { /// True if the scheduler is being shutdown is_shutdown: bool, - /// Tasks owned by the core - tasks: LinkedList::Target>, - /// Parker /// /// Stored in an `Option` as the parker is added / removed to make the @@ -78,6 +75,9 @@ pub(super) struct Shared { /// Coordinates idle workers idle: Idle, + /// Collection of all active tasks spawned onto this executor. + owned: OwnedTasks>, + /// Cores that have observed the shutdown signal /// /// The core is **not** placed back in the worker to avoid it from being @@ -91,10 +91,6 @@ struct Remote { /// Steal tasks from this worker. steal: queue::Steal>, - /// Transfers tasks to be released. Any worker pushes tasks, only the owning - /// worker pops. - pending_drop: task::TransferStack>, - /// Unparks the associated worker thread unpark: Unparker, } @@ -142,22 +138,18 @@ pub(super) fn create(size: usize, park: Parker) -> (Arc, Launch) { run_queue, is_searching: false, is_shutdown: false, - tasks: LinkedList::new(), park: Some(park), rand: FastRand::new(seed()), })); - remotes.push(Remote { - steal, - pending_drop: task::TransferStack::new(), - unpark, - }); + remotes.push(Remote { steal, unpark }); } let shared = Arc::new(Shared { remotes: remotes.into_boxed_slice(), inject: queue::Inject::new(), idle: Idle::new(size), + owned: OwnedTasks::new(), shutdown_cores: Mutex::new(vec![]), }); @@ -203,18 +195,20 @@ where CURRENT.with(|maybe_cx| { match (crate::runtime::enter::context(), maybe_cx.is_some()) { (EnterContext::Entered { .. }, true) => { - // We are on a thread pool runtime thread, so we just need to set up blocking. + // We are on a thread pool runtime thread, so we just need to + // set up blocking. had_entered = true; } (EnterContext::Entered { allow_blocking }, false) => { - // We are on an executor, but _not_ on the thread pool. - // That is _only_ okay if we are in a thread pool runtime's block_on method: + // We are on an executor, but _not_ on the thread pool. That is + // _only_ okay if we are in a thread pool runtime's block_on + // method: if allow_blocking { had_entered = true; return; } else { - // This probably means we are on the basic_scheduler or in a LocalSet, - // where it is _not_ okay to block. + // This probably means we are on the basic_scheduler or in a + // LocalSet, where it is _not_ okay to block. panic!("can call blocking only when running on the multi-threaded runtime"); } } @@ -538,42 +532,25 @@ impl Core { true } - /// Runs maintenance work such as free pending tasks and check the pool's - /// state. + /// Runs maintenance work such as checking the pool's state. fn maintenance(&mut self, worker: &Worker) { - self.drain_pending_drop(worker); - if !self.is_shutdown { // Check if the scheduler has been shutdown self.is_shutdown = worker.inject().is_closed(); } } - // Signals all tasks to shut down, and waits for them to complete. Must run - // before we enter the single-threaded phase of shutdown processing. + /// Signals all tasks to shut down, and waits for them to complete. Must run + /// before we enter the single-threaded phase of shutdown processing. fn pre_shutdown(&mut self, worker: &Worker) { // Signal to all tasks to shut down. - for header in self.tasks.iter() { + while let Some(header) = worker.shared.owned.pop_back() { header.shutdown(); } - - loop { - self.drain_pending_drop(worker); - - if self.tasks.is_empty() { - break; - } - - // Wait until signalled - let park = self.park.as_mut().expect("park missing"); - park.park().expect("park failed"); - } } // Shutdown the core fn shutdown(&mut self) { - assert!(self.tasks.is_empty()); - // Take the core let mut park = self.park.take().expect("park missing"); @@ -582,24 +559,6 @@ impl Core { park.shutdown(); } - - fn drain_pending_drop(&mut self, worker: &Worker) { - use std::mem::ManuallyDrop; - - for task in worker.remote().pending_drop.drain() { - let task = ManuallyDrop::new(task); - - // safety: tasks are only pushed into the `pending_drop` stacks that - // are associated with the list they are inserted into. When a task - // is pushed into `pending_drop`, the ref-inc is skipped, so we must - // not ref-dec here. - // - // See `bind` and `release` implementations. - unsafe { - self.tasks.remove(task.header().into()); - } - } - } } impl Worker { @@ -607,15 +566,6 @@ impl Worker { fn inject(&self) -> &queue::Inject> { &self.shared.inject } - - /// Return a reference to this worker's remote data - fn remote(&self) -> &Remote { - &self.shared.remotes[self.index] - } - - fn eq(&self, other: &Worker) -> bool { - self.shared.ptr_eq(&other.shared) && self.index == other.index - } } impl task::Schedule for Arc { @@ -624,12 +574,7 @@ impl task::Schedule for Arc { let cx = maybe_cx.expect("scheduler context missing"); // Track the task - cx.core - .borrow_mut() - .as_mut() - .expect("scheduler core missing") - .tasks - .push_front(task); + cx.worker.shared.owned.push_front(task); // Return a clone of the worker cx.worker.clone() @@ -637,75 +582,8 @@ impl task::Schedule for Arc { } fn release(&self, task: &Task) -> Option { - use std::ptr::NonNull; - - enum Immediate { - // Task has been synchronously removed from the Core owned by the - // current thread - Removed(Option), - // Task is owned by another thread, so we need to notify it to clean - // up the task later. - MaybeRemote, - } - - let immediate = CURRENT.with(|maybe_cx| { - let cx = match maybe_cx { - Some(cx) => cx, - None => return Immediate::MaybeRemote, - }; - - if !self.eq(&cx.worker) { - // Task owned by another core, so we need to notify it. - return Immediate::MaybeRemote; - } - - let mut maybe_core = cx.core.borrow_mut(); - - if let Some(core) = &mut *maybe_core { - // Directly remove the task - // - // safety: the task is inserted in the list in `bind`. - unsafe { - let ptr = NonNull::from(task.header()); - return Immediate::Removed(core.tasks.remove(ptr)); - } - } - - Immediate::MaybeRemote - }); - - // Checks if we were called from within a worker, allowing for immediate - // removal of a scheduled task. Else we have to go through the slower - // process below where we remotely mark a task as dropped. - match immediate { - Immediate::Removed(task) => return task, - Immediate::MaybeRemote => (), - }; - - // Track the task to be released by the worker that owns it - // - // Safety: We get a new handle without incrementing the ref-count. - // A ref-count is held by the "owned" linked list and it is only - // ever removed from that list as part of the release process: this - // method or popping the task from `pending_drop`. Thus, we can rely - // on the ref-count held by the linked-list to keep the memory - // alive. - // - // When the task is removed from the stack, it is forgotten instead - // of dropped. - let task = unsafe { Task::from_raw(task.header().into()) }; - - self.remote().pending_drop.push(task); - - // The worker core has been handed off to another thread. In the - // event that the scheduler is currently shutting down, the thread - // that owns the task may be waiting on the release to complete - // shutdown. - if self.inject().is_closed() { - self.remote().unpark.unpark(); - } - - None + // SAFETY: Inserted into owned in bind. + unsafe { self.shared.owned.remove(task) } } fn schedule(&self, task: Notified) { @@ -825,6 +703,8 @@ impl Shared { return; } + debug_assert!(self.owned.is_empty()); + for mut core in cores.drain(..) { core.shutdown(); } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index a74f56215..1eab81c31 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -236,37 +236,6 @@ impl Default for LinkedList { } } -// ===== impl Iter ===== - -cfg_rt_multi_thread! { - pub(crate) struct Iter<'a, T: Link> { - curr: Option>, - _p: core::marker::PhantomData<&'a T>, - } - - impl LinkedList { - pub(crate) fn iter(&self) -> Iter<'_, L> { - Iter { - curr: self.head, - _p: core::marker::PhantomData, - } - } - } - - impl<'a, T: Link> Iterator for Iter<'a, T> { - type Item = &'a T::Target; - - fn next(&mut self) -> Option<&'a T::Target> { - let curr = self.curr?; - // safety: the pointer references data contained by the list - self.curr = unsafe { T::pointers(curr).as_ref() }.get_next(); - - // safety: the value is still owned by the linked list. - Some(unsafe { &*curr.as_ptr() }) - } - } -} - // ===== impl DrainFilter ===== cfg_io_readiness! { @@ -645,24 +614,6 @@ mod tests { } } - #[test] - fn iter() { - let a = entry(5); - let b = entry(7); - - let mut list = LinkedList::<&Entry, <&Entry as Link>::Target>::new(); - - assert_eq!(0, list.iter().count()); - - list.push_front(a.as_ref()); - list.push_front(b.as_ref()); - - let mut i = list.iter(); - assert_eq!(7, i.next().unwrap().val); - assert_eq!(5, i.next().unwrap().val); - assert!(i.next().is_none()); - } - proptest::proptest! { #[test] fn fuzz_linked_list(ops: Vec) { From c505a2f81aa7a73d95bd2b0c56a5a5205054656e Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Jul 2021 10:58:02 +0200 Subject: [PATCH 10/35] chore: prepare tokio-macros 1.3.0 (#3931) --- tokio-macros/CHANGELOG.md | 6 ++++++ tokio-macros/Cargo.toml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tokio-macros/CHANGELOG.md b/tokio-macros/CHANGELOG.md index 6df34c0c0..0d58f9737 100644 --- a/tokio-macros/CHANGELOG.md +++ b/tokio-macros/CHANGELOG.md @@ -1,3 +1,9 @@ +# 1.3.0 (July 7, 2021) + +- macros: don't trigger `clippy::unwrap_used` ([#3926]) + +[#3926]: https://github.com/tokio-rs/tokio/pull/3926 + # 1.2.0 (May 14, 2021) - macros: forward input arguments in `#[tokio::test]` ([#3691]) diff --git a/tokio-macros/Cargo.toml b/tokio-macros/Cargo.toml index 875ceb624..5399bc6c0 100644 --- a/tokio-macros/Cargo.toml +++ b/tokio-macros/Cargo.toml @@ -6,13 +6,13 @@ name = "tokio-macros" # - Cargo.toml # - Update CHANGELOG.md. # - Create "tokio-macros-1.0.x" git tag. -version = "1.2.0" +version = "1.3.0" edition = "2018" authors = ["Tokio Contributors "] license = "MIT" repository = "https://github.com/tokio-rs/tokio" homepage = "https://tokio.rs" -documentation = "https://docs.rs/tokio-macros/1.2.0/tokio_macros" +documentation = "https://docs.rs/tokio-macros/1.3.0/tokio_macros" description = """ Tokio's proc macros. """ From 5d61c997e9281e00f35c7ea426786996945f47f5 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Jul 2021 10:58:51 +0200 Subject: [PATCH 11/35] chore: prepare tokio-stream 0.1.7 (#3923) --- tokio-stream/CHANGELOG.md | 10 ++++++++++ tokio-stream/Cargo.toml | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tokio-stream/CHANGELOG.md b/tokio-stream/CHANGELOG.md index 0c5980418..a0cdef0f6 100644 --- a/tokio-stream/CHANGELOG.md +++ b/tokio-stream/CHANGELOG.md @@ -1,3 +1,13 @@ +# 0.1.7 (July 7, 2021) + +### Fixed + +- sync: fix watch wrapper ([#3914]) +- time: fix `Timeout::size_hint` ([#3902]) + +[#3902]: https://github.com/tokio-rs/tokio/pull/3902 +[#3914]: https://github.com/tokio-rs/tokio/pull/3914 + # 0.1.6 (May 14, 2021) ### Added diff --git a/tokio-stream/Cargo.toml b/tokio-stream/Cargo.toml index e169a2bec..911657c37 100644 --- a/tokio-stream/Cargo.toml +++ b/tokio-stream/Cargo.toml @@ -6,13 +6,13 @@ name = "tokio-stream" # - Cargo.toml # - Update CHANGELOG.md. # - Create "tokio-stream-0.1.x" git tag. -version = "0.1.6" +version = "0.1.7" edition = "2018" authors = ["Tokio Contributors "] license = "MIT" repository = "https://github.com/tokio-rs/tokio" homepage = "https://tokio.rs" -documentation = "https://docs.rs/tokio-stream/0.1.6/tokio_stream" +documentation = "https://docs.rs/tokio-stream/0.1.7/tokio_stream" description = """ Utilities to work with `Stream` and `tokio`. """ From c6fbb9aeb1f8f52055d0fc506f8ac2ffac700bfe Mon Sep 17 00:00:00 2001 From: Moritz Gunz Date: Thu, 8 Jul 2021 12:54:31 +0200 Subject: [PATCH 12/35] task: expose nameable future for TaskLocal::scope (#3273) --- tokio/src/task/mod.rs | 5 +++++ tokio/src/task/task_local.rs | 33 ++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tokio/src/task/mod.rs b/tokio/src/task/mod.rs index ae4c35c9c..ea9878736 100644 --- a/tokio/src/task/mod.rs +++ b/tokio/src/task/mod.rs @@ -304,4 +304,9 @@ cfg_rt! { mod builder; pub use builder::Builder; } + + /// Task-related futures. + pub mod futures { + pub use super::task_local::TaskLocalFuture; + } } diff --git a/tokio/src/task/task_local.rs b/tokio/src/task/task_local.rs index 6571ffd7b..b55c6abc4 100644 --- a/tokio/src/task/task_local.rs +++ b/tokio/src/task/task_local.rs @@ -115,7 +115,7 @@ impl LocalKey { /// }).await; /// # } /// ``` - pub async fn scope(&'static self, value: T, f: F) -> F::Output + pub fn scope(&'static self, value: T, f: F) -> TaskLocalFuture where F: Future, { @@ -124,7 +124,6 @@ impl LocalKey { slot: Some(value), future: f, } - .await } /// Sets a value `T` as the task-local value for the closure `F`. @@ -206,7 +205,31 @@ impl fmt::Debug for LocalKey { } pin_project! { - struct TaskLocalFuture { + /// A future that sets a value `T` of a task local for the future `F` during + /// its execution. + /// + /// The value of the task-local must be `'static` and will be dropped on the + /// completion of the future. + /// + /// Created by the function [`LocalKey::scope`](self::LocalKey::scope). + /// + /// ### Examples + /// + /// ``` + /// # async fn dox() { + /// tokio::task_local! { + /// static NUMBER: u32; + /// } + /// + /// NUMBER.scope(1, async move { + /// println!("task local value: {}", NUMBER.get()); + /// }).await; + /// # } + /// ``` + pub struct TaskLocalFuture + where + T: 'static + { local: &'static LocalKey, slot: Option, #[pin] @@ -252,10 +275,6 @@ impl Future for TaskLocalFuture { } } -// Required to make `pin_project` happy. -trait StaticLifetime: 'static {} -impl StaticLifetime for T {} - /// An error returned by [`LocalKey::try_with`](method@LocalKey::try_with). #[derive(Clone, Copy, Eq, PartialEq)] pub struct AccessError { From c306bf853a1f8423b154f17fa47926f04eecd9b4 Mon Sep 17 00:00:00 2001 From: ty Date: Thu, 8 Jul 2021 19:02:40 +0800 Subject: [PATCH 13/35] net: allow customized I/O operations for TcpStream (#3888) --- tokio/src/net/tcp/stream.rs | 35 ++++++++++++++ tokio/src/net/udp.rs | 35 ++++++++++++++ tokio/src/net/unix/datagram/socket.rs | 35 ++++++++++++++ tokio/src/net/unix/stream.rs | 35 ++++++++++++++ tokio/src/net/windows/named_pipe.rs | 70 +++++++++++++++++++++++++++ 5 files changed, 210 insertions(+) diff --git a/tokio/src/net/tcp/stream.rs b/tokio/src/net/tcp/stream.rs index 0277a360d..daf3dccde 100644 --- a/tokio/src/net/tcp/stream.rs +++ b/tokio/src/net/tcp/stream.rs @@ -936,6 +936,41 @@ impl TcpStream { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(bufs)) } + /// Try to perform IO operation from the socket using a user-provided IO operation. + /// + /// If the socket is ready, the provided closure is called. The + /// closure should attempt to perform IO operation from the socket by manually calling the + /// appropriate syscall. If the operation fails because the socket is not + /// actually ready, then the closure should return a `WouldBlock` error and + /// the readiness flag is cleared. The return value of the closure is + /// then returned by `try_io`. + /// + /// If the socket is not ready, then the closure is not called + /// and a `WouldBlock` error is returned. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the read operation using any of the + /// methods defined on the Tokio `TcpStream` type, as this will mess with + /// the readiness flag and can cause the socket to behave incorrectly. + /// + /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: TcpStream::readable() + /// [`writable()`]: TcpStream::writable() + /// [`ready()`]: TcpStream::ready() + pub fn try_io( + &self, + interest: Interest, + f: impl FnOnce() -> io::Result, + ) -> io::Result { + self.io.registration().try_io(interest, f) + } + /// Receives data on the socket from the remote address to which it is /// connected, without removing that data from the queue. On success, /// returns the number of bytes peeked. diff --git a/tokio/src/net/udp.rs b/tokio/src/net/udp.rs index 301a85cc0..a47f071d7 100644 --- a/tokio/src/net/udp.rs +++ b/tokio/src/net/udp.rs @@ -1170,6 +1170,41 @@ impl UdpSocket { .try_io(Interest::READABLE, || self.io.recv_from(buf)) } + /// Try to perform IO operation from the socket using a user-provided IO operation. + /// + /// If the socket is ready, the provided closure is called. The + /// closure should attempt to perform IO operation from the socket by manually calling the + /// appropriate syscall. If the operation fails because the socket is not + /// actually ready, then the closure should return a `WouldBlock` error and + /// the readiness flag is cleared. The return value of the closure is + /// then returned by `try_io`. + /// + /// If the socket is not ready, then the closure is not called + /// and a `WouldBlock` error is returned. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the read operation using any of the + /// methods defined on the Tokio `UdpSocket` type, as this will mess with + /// the readiness flag and can cause the socket to behave incorrectly. + /// + /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: UdpSocket::readable() + /// [`writable()`]: UdpSocket::writable() + /// [`ready()`]: UdpSocket::ready() + pub fn try_io( + &self, + interest: Interest, + f: impl FnOnce() -> io::Result, + ) -> io::Result { + self.io.registration().try_io(interest, f) + } + /// Receives data from the socket, without removing it from the input queue. /// On success, returns the number of bytes read and the address from whence /// the data came. diff --git a/tokio/src/net/unix/datagram/socket.rs b/tokio/src/net/unix/datagram/socket.rs index 2d2177803..c7c6568f2 100644 --- a/tokio/src/net/unix/datagram/socket.rs +++ b/tokio/src/net/unix/datagram/socket.rs @@ -1143,6 +1143,41 @@ impl UnixDatagram { Ok((n, SocketAddr(addr))) } + /// Try to perform IO operation from the socket using a user-provided IO operation. + /// + /// If the socket is ready, the provided closure is called. The + /// closure should attempt to perform IO operation from the socket by manually calling the + /// appropriate syscall. If the operation fails because the socket is not + /// actually ready, then the closure should return a `WouldBlock` error and + /// the readiness flag is cleared. The return value of the closure is + /// then returned by `try_io`. + /// + /// If the socket is not ready, then the closure is not called + /// and a `WouldBlock` error is returned. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the read operation using any of the + /// methods defined on the Tokio `UnixDatagram` type, as this will mess with + /// the readiness flag and can cause the socket to behave incorrectly. + /// + /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: UnixDatagram::readable() + /// [`writable()`]: UnixDatagram::writable() + /// [`ready()`]: UnixDatagram::ready() + pub fn try_io( + &self, + interest: Interest, + f: impl FnOnce() -> io::Result, + ) -> io::Result { + self.io.registration().try_io(interest, f) + } + /// Returns the local address that this socket is bound to. /// /// # Examples diff --git a/tokio/src/net/unix/stream.rs b/tokio/src/net/unix/stream.rs index 4baac6062..39f6ee258 100644 --- a/tokio/src/net/unix/stream.rs +++ b/tokio/src/net/unix/stream.rs @@ -653,6 +653,41 @@ impl UnixStream { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } + /// Try to perform IO operation from the socket using a user-provided IO operation. + /// + /// If the socket is ready, the provided closure is called. The + /// closure should attempt to perform IO operation from the socket by manually calling the + /// appropriate syscall. If the operation fails because the socket is not + /// actually ready, then the closure should return a `WouldBlock` error and + /// the readiness flag is cleared. The return value of the closure is + /// then returned by `try_io`. + /// + /// If the socket is not ready, then the closure is not called + /// and a `WouldBlock` error is returned. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the read operation using any of the + /// methods defined on the Tokio `UnixStream` type, as this will mess with + /// the readiness flag and can cause the socket to behave incorrectly. + /// + /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: UnixStream::readable() + /// [`writable()`]: UnixStream::writable() + /// [`ready()`]: UnixStream::ready() + pub fn try_io( + &self, + interest: Interest, + f: impl FnOnce() -> io::Result, + ) -> io::Result { + self.io.registration().try_io(interest, f) + } + /// Creates new `UnixStream` from a `std::os::unix::net::UnixStream`. /// /// This function is intended to be used to wrap a UnixStream from the diff --git a/tokio/src/net/windows/named_pipe.rs b/tokio/src/net/windows/named_pipe.rs index b9f7d49d7..4e7458b58 100644 --- a/tokio/src/net/windows/named_pipe.rs +++ b/tokio/src/net/windows/named_pipe.rs @@ -723,6 +723,41 @@ impl NamedPipeServer { .registration() .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } + + /// Try to perform IO operation from the socket using a user-provided IO operation. + /// + /// If the socket is ready, the provided closure is called. The + /// closure should attempt to perform IO operation from the socket by manually calling the + /// appropriate syscall. If the operation fails because the socket is not + /// actually ready, then the closure should return a `WouldBlock` error and + /// the readiness flag is cleared. The return value of the closure is + /// then returned by `try_io`. + /// + /// If the socket is not ready, then the closure is not called + /// and a `WouldBlock` error is returned. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the read operation using any of the + /// methods defined on the Tokio `NamedPipeServer` type, as this will mess with + /// the readiness flag and can cause the socket to behave incorrectly. + /// + /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: NamedPipeServer::readable() + /// [`writable()`]: NamedPipeServer::writable() + /// [`ready()`]: NamedPipeServer::ready() + pub fn try_io( + &self, + interest: Interest, + f: impl FnOnce() -> io::Result, + ) -> io::Result { + self.io.registration().try_io(interest, f) + } } impl AsyncRead for NamedPipeServer { @@ -1343,6 +1378,41 @@ impl NamedPipeClient { .registration() .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } + + /// Try to perform IO operation from the socket using a user-provided IO operation. + /// + /// If the socket is ready, the provided closure is called. The + /// closure should attempt to perform IO operation from the socket by manually calling the + /// appropriate syscall. If the operation fails because the socket is not + /// actually ready, then the closure should return a `WouldBlock` error and + /// the readiness flag is cleared. The return value of the closure is + /// then returned by `try_io`. + /// + /// If the socket is not ready, then the closure is not called + /// and a `WouldBlock` error is returned. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the read operation using any of the + /// methods defined on the Tokio `NamedPipeClient` type, as this will mess with + /// the readiness flag and can cause the socket to behave incorrectly. + /// + /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: NamedPipeClient::readable() + /// [`writable()`]: NamedPipeClient::writable() + /// [`ready()`]: NamedPipeClient::ready() + pub fn try_io( + &self, + interest: Interest, + f: impl FnOnce() -> io::Result, + ) -> io::Result { + self.io.registration().try_io(interest, f) + } } impl AsyncRead for NamedPipeClient { From 127983e5b4ac36e780fbdafddca1da349cdb6453 Mon Sep 17 00:00:00 2001 From: Alan Somers Date: Sun, 11 Jul 2021 01:53:04 -0600 Subject: [PATCH 14/35] tests: update Nix to 0.22.0 (#3951) --- tokio/Cargo.toml | 2 +- tokio/tests/io_async_fd.rs | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index 2945b6a38..fd8b5dbd8 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -109,7 +109,7 @@ signal-hook-registry = { version = "1.1.1", optional = true } [target.'cfg(unix)'.dev-dependencies] libc = { version = "0.2.42" } -nix = { version = "0.19.0" } +nix = { version = "0.22.0" } [target.'cfg(windows)'.dependencies.winapi] version = "0.3.8" diff --git a/tokio/tests/io_async_fd.rs b/tokio/tests/io_async_fd.rs index d1586bb36..dc21e426f 100644 --- a/tokio/tests/io_async_fd.rs +++ b/tokio/tests/io_async_fd.rs @@ -13,7 +13,6 @@ use std::{ task::{Context, Waker}, }; -use nix::errno::Errno; use nix::unistd::{close, read, write}; use futures::{poll, FutureExt}; @@ -56,10 +55,6 @@ impl TestWaker { } } -fn is_blocking(e: &nix::Error) -> bool { - Some(Errno::EAGAIN) == e.as_errno() -} - #[derive(Debug)] struct FileDescriptor { fd: RawFd, @@ -73,11 +68,7 @@ impl AsRawFd for FileDescriptor { impl Read for &FileDescriptor { fn read(&mut self, buf: &mut [u8]) -> io::Result { - match read(self.fd, buf) { - Ok(n) => Ok(n), - Err(e) if is_blocking(&e) => Err(ErrorKind::WouldBlock.into()), - Err(e) => Err(io::Error::new(ErrorKind::Other, e)), - } + read(self.fd, buf).map_err(io::Error::from) } } @@ -89,11 +80,7 @@ impl Read for FileDescriptor { impl Write for &FileDescriptor { fn write(&mut self, buf: &[u8]) -> io::Result { - match write(self.fd, buf) { - Ok(n) => Ok(n), - Err(e) if is_blocking(&e) => Err(ErrorKind::WouldBlock.into()), - Err(e) => Err(io::Error::new(ErrorKind::Other, e)), - } + write(self.fd, buf).map_err(io::Error::from) } fn flush(&mut self) -> io::Result<()> { From 3b38ebd7f5d50611193f942c31c57262f263be33 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 12 Jul 2021 10:31:36 +0200 Subject: [PATCH 15/35] runtime: move inject queue to `tokio::runtime::task` (#3939) --- tokio/src/runtime/queue.rs | 280 +++++------------------- tokio/src/runtime/task/inject.rs | 221 +++++++++++++++++++ tokio/src/runtime/task/mod.rs | 9 +- tokio/src/runtime/tests/loom_queue.rs | 10 +- tokio/src/runtime/tests/queue.rs | 12 +- tokio/src/runtime/thread_pool/worker.rs | 8 +- 6 files changed, 298 insertions(+), 242 deletions(-) create mode 100644 tokio/src/runtime/task/inject.rs diff --git a/tokio/src/runtime/queue.rs b/tokio/src/runtime/queue.rs index 818ef7bac..6e91dfa23 100644 --- a/tokio/src/runtime/queue.rs +++ b/tokio/src/runtime/queue.rs @@ -1,13 +1,12 @@ //! Run-queue structures to support a work-stealing scheduler use crate::loom::cell::UnsafeCell; -use crate::loom::sync::atomic::{AtomicU16, AtomicU32, AtomicUsize}; -use crate::loom::sync::{Arc, Mutex}; -use crate::runtime::task; +use crate::loom::sync::atomic::{AtomicU16, AtomicU32}; +use crate::loom::sync::Arc; +use crate::runtime::task::{self, Inject}; -use std::marker::PhantomData; use std::mem::MaybeUninit; -use std::ptr::{self, NonNull}; +use std::ptr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; /// Producer handle. May only be used from a single thread. @@ -18,19 +17,6 @@ pub(super) struct Local { /// Consumer handle. May be used from many threads. pub(super) struct Steal(Arc>); -/// Growable, MPMC queue used to inject new tasks into the scheduler and as an -/// overflow queue when the local, fixed-size, array queue overflows. -pub(super) struct Inject { - /// Pointers to the head and tail of the queue - pointers: Mutex, - - /// Number of pending tasks in the queue. This helps prevent unnecessary - /// locking in the hot path. - len: AtomicUsize, - - _p: PhantomData, -} - pub(super) struct Inner { /// Concurrently updated by many threads. /// @@ -49,24 +35,11 @@ pub(super) struct Inner { tail: AtomicU16, /// Elements - buffer: Box<[UnsafeCell>>]>, -} - -struct Pointers { - /// True if the queue is closed - is_closed: bool, - - /// Linked-list head - head: Option>, - - /// Linked-list tail - tail: Option>, + buffer: Box<[UnsafeCell>>; LOCAL_QUEUE_CAPACITY]>, } unsafe impl Send for Inner {} unsafe impl Sync for Inner {} -unsafe impl Send for Inject {} -unsafe impl Sync for Inject {} #[cfg(not(loom))] const LOCAL_QUEUE_CAPACITY: usize = 256; @@ -79,6 +52,17 @@ const LOCAL_QUEUE_CAPACITY: usize = 4; const MASK: usize = LOCAL_QUEUE_CAPACITY - 1; +// Constructing the fixed size array directly is very awkward. The only way to +// do it is to repeat `UnsafeCell::new(MaybeUninit::uninit())` 256 times, as +// the contents are not Copy. The trick with defining a const doesn't work for +// generic types. +fn make_fixed_size(buffer: Box<[T]>) -> Box<[T; LOCAL_QUEUE_CAPACITY]> { + assert_eq!(buffer.len(), LOCAL_QUEUE_CAPACITY); + + // safety: We check that the length is correct. + unsafe { Box::from_raw(Box::into_raw(buffer).cast()) } +} + /// Create a new local run-queue pub(super) fn local() -> (Steal, Local) { let mut buffer = Vec::with_capacity(LOCAL_QUEUE_CAPACITY); @@ -90,7 +74,7 @@ pub(super) fn local() -> (Steal, Local) { let inner = Arc::new(Inner { head: AtomicU32::new(0), tail: AtomicU16::new(0), - buffer: buffer.into(), + buffer: make_fixed_size(buffer.into_boxed_slice()), }); let local = Local { @@ -109,10 +93,7 @@ impl Local { } /// Pushes a task to the back of the local queue, skipping the LIFO slot. - pub(super) fn push_back(&mut self, mut task: task::Notified, inject: &Inject) - where - T: crate::runtime::task::Schedule, - { + pub(super) fn push_back(&mut self, mut task: task::Notified, inject: &Inject) { let tail = loop { let head = self.inner.head.load(Acquire); let (steal, real) = unpack(head); @@ -179,9 +160,12 @@ impl Local { tail: u16, inject: &Inject, ) -> Result<(), task::Notified> { - const BATCH_LEN: usize = LOCAL_QUEUE_CAPACITY / 2 + 1; + /// How many elements are we taking from the local queue. + /// + /// This is one less than the number of tasks pushed to the inject + /// queue as we are also inserting the `task` argument. + const NUM_TASKS_TAKEN: u16 = (LOCAL_QUEUE_CAPACITY / 2) as u16; - let n = (LOCAL_QUEUE_CAPACITY / 2) as u16; assert_eq!( tail.wrapping_sub(head) as usize, LOCAL_QUEUE_CAPACITY, @@ -207,7 +191,10 @@ impl Local { .head .compare_exchange( prev, - pack(head.wrapping_add(n), head.wrapping_add(n)), + pack( + head.wrapping_add(NUM_TASKS_TAKEN), + head.wrapping_add(NUM_TASKS_TAKEN), + ), Release, Relaxed, ) @@ -219,41 +206,41 @@ impl Local { return Err(task); } - // link the tasks - for i in 0..n { - let j = i + 1; + /// An iterator that takes elements out of the run queue. + struct BatchTaskIter<'a, T: 'static> { + buffer: &'a [UnsafeCell>>; LOCAL_QUEUE_CAPACITY], + head: u32, + i: u32, + } + impl<'a, T: 'static> Iterator for BatchTaskIter<'a, T> { + type Item = task::Notified; - let i_idx = i.wrapping_add(head) as usize & MASK; - let j_idx = j.wrapping_add(head) as usize & MASK; + #[inline] + fn next(&mut self) -> Option> { + if self.i == u32::from(NUM_TASKS_TAKEN) { + None + } else { + let i_idx = self.i.wrapping_add(self.head) as usize & MASK; + let slot = &self.buffer[i_idx]; - // Get the next pointer - let next = if j == n { - // The last task in the local queue being moved - task.header().into() - } else { - // safety: The above CAS prevents a stealer from accessing these - // tasks and we are the only producer. - self.inner.buffer[j_idx].with(|ptr| unsafe { - let value = (*ptr).as_ptr(); - (*value).header().into() - }) - }; + // safety: Our CAS from before has assumed exclusive ownership + // of the task pointers in this range. + let task = slot.with(|ptr| unsafe { ptr::read((*ptr).as_ptr()) }); - // safety: the above CAS prevents a stealer from accessing these - // tasks and we are the only producer. - self.inner.buffer[i_idx].with_mut(|ptr| unsafe { - let ptr = (*ptr).as_ptr(); - (*ptr).header().set_next(Some(next)) - }); + self.i += 1; + Some(task) + } + } } - // safety: the above CAS prevents a stealer from accessing these tasks - // and we are the only producer. - let head = self.inner.buffer[head as usize & MASK] - .with(|ptr| unsafe { ptr::read((*ptr).as_ptr()) }); - - // Push the tasks onto the inject queue - inject.push_batch(head, task, BATCH_LEN); + // safety: The CAS above ensures that no consumer will look at these + // values again, and we are the only producer. + let batch_iter = BatchTaskIter { + buffer: &*self.inner.buffer, + head: head as u32, + i: 0, + }; + inject.push_batch(batch_iter.chain(std::iter::once(task))); Ok(()) } @@ -473,159 +460,6 @@ impl Inner { } } -impl Inject { - pub(super) fn new() -> Inject { - Inject { - pointers: Mutex::new(Pointers { - is_closed: false, - head: None, - tail: None, - }), - len: AtomicUsize::new(0), - _p: PhantomData, - } - } - - pub(super) fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Close the injection queue, returns `true` if the queue is open when the - /// transition is made. - pub(super) fn close(&self) -> bool { - let mut p = self.pointers.lock(); - - if p.is_closed { - return false; - } - - p.is_closed = true; - true - } - - pub(super) fn is_closed(&self) -> bool { - self.pointers.lock().is_closed - } - - pub(super) fn len(&self) -> usize { - self.len.load(Acquire) - } - - /// Pushes a value into the queue. - /// - /// Returns `Err(task)` if pushing fails due to the queue being shutdown. - /// The caller is expected to call `shutdown()` on the task **if and only - /// if** it is a newly spawned task. - pub(super) fn push(&self, task: task::Notified) -> Result<(), task::Notified> - where - T: crate::runtime::task::Schedule, - { - // Acquire queue lock - let mut p = self.pointers.lock(); - - if p.is_closed { - return Err(task); - } - - // safety: only mutated with the lock held - let len = unsafe { self.len.unsync_load() }; - let task = task.into_raw(); - - // The next pointer should already be null - debug_assert!(get_next(task).is_none()); - - if let Some(tail) = p.tail { - set_next(tail, Some(task)); - } else { - p.head = Some(task); - } - - p.tail = Some(task); - - self.len.store(len + 1, Release); - Ok(()) - } - - pub(super) fn push_batch( - &self, - batch_head: task::Notified, - batch_tail: task::Notified, - num: usize, - ) { - let batch_head = batch_head.into_raw(); - let batch_tail = batch_tail.into_raw(); - - debug_assert!(get_next(batch_tail).is_none()); - - let mut p = self.pointers.lock(); - - if let Some(tail) = p.tail { - set_next(tail, Some(batch_head)); - } else { - p.head = Some(batch_head); - } - - p.tail = Some(batch_tail); - - // Increment the count. - // - // safety: All updates to the len atomic are guarded by the mutex. As - // such, a non-atomic load followed by a store is safe. - let len = unsafe { self.len.unsync_load() }; - - self.len.store(len + num, Release); - } - - pub(super) fn pop(&self) -> Option> { - // Fast path, if len == 0, then there are no values - if self.is_empty() { - return None; - } - - let mut p = self.pointers.lock(); - - // It is possible to hit null here if another thread popped the last - // task between us checking `len` and acquiring the lock. - let task = p.head?; - - p.head = get_next(task); - - if p.head.is_none() { - p.tail = None; - } - - set_next(task, None); - - // Decrement the count. - // - // safety: All updates to the len atomic are guarded by the mutex. As - // such, a non-atomic load followed by a store is safe. - self.len - .store(unsafe { self.len.unsync_load() } - 1, Release); - - // safety: a `Notified` is pushed into the queue and now it is popped! - Some(unsafe { task::Notified::from_raw(task) }) - } -} - -impl Drop for Inject { - fn drop(&mut self) { - if !std::thread::panicking() { - assert!(self.pop().is_none(), "queue not empty"); - } - } -} - -fn get_next(header: NonNull) -> Option> { - unsafe { header.as_ref().queue_next.with(|ptr| *ptr) } -} - -fn set_next(header: NonNull, val: Option>) { - unsafe { - header.as_ref().set_next(val); - } -} - /// Split the head value into the real head and the index a stealer is working /// on. fn unpack(n: u32) -> (u16, u16) { diff --git a/tokio/src/runtime/task/inject.rs b/tokio/src/runtime/task/inject.rs new file mode 100644 index 000000000..8ca3187a7 --- /dev/null +++ b/tokio/src/runtime/task/inject.rs @@ -0,0 +1,221 @@ +//! Inject queue used to send wakeups to a work-stealing scheduler + +use crate::loom::sync::atomic::AtomicUsize; +use crate::loom::sync::Mutex; +use crate::runtime::task; + +use std::marker::PhantomData; +use std::ptr::NonNull; +use std::sync::atomic::Ordering::{Acquire, Release}; + +/// Growable, MPMC queue used to inject new tasks into the scheduler and as an +/// overflow queue when the local, fixed-size, array queue overflows. +pub(crate) struct Inject { + /// Pointers to the head and tail of the queue + pointers: Mutex, + + /// Number of pending tasks in the queue. This helps prevent unnecessary + /// locking in the hot path. + len: AtomicUsize, + + _p: PhantomData, +} + +struct Pointers { + /// True if the queue is closed + is_closed: bool, + + /// Linked-list head + head: Option>, + + /// Linked-list tail + tail: Option>, +} + +unsafe impl Send for Inject {} +unsafe impl Sync for Inject {} + +impl Inject { + pub(crate) fn new() -> Inject { + Inject { + pointers: Mutex::new(Pointers { + is_closed: false, + head: None, + tail: None, + }), + len: AtomicUsize::new(0), + _p: PhantomData, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Close the injection queue, returns `true` if the queue is open when the + /// transition is made. + pub(crate) fn close(&self) -> bool { + let mut p = self.pointers.lock(); + + if p.is_closed { + return false; + } + + p.is_closed = true; + true + } + + pub(crate) fn is_closed(&self) -> bool { + self.pointers.lock().is_closed + } + + pub(crate) fn len(&self) -> usize { + self.len.load(Acquire) + } + + /// Pushes a value into the queue. + /// + /// Returns `Err(task)` if pushing fails due to the queue being shutdown. + /// The caller is expected to call `shutdown()` on the task **if and only + /// if** it is a newly spawned task. + pub(crate) fn push(&self, task: task::Notified) -> Result<(), task::Notified> { + // Acquire queue lock + let mut p = self.pointers.lock(); + + if p.is_closed { + return Err(task); + } + + // safety: only mutated with the lock held + let len = unsafe { self.len.unsync_load() }; + let task = task.into_raw(); + + // The next pointer should already be null + debug_assert!(get_next(task).is_none()); + + if let Some(tail) = p.tail { + set_next(tail, Some(task)); + } else { + p.head = Some(task); + } + + p.tail = Some(task); + + self.len.store(len + 1, Release); + Ok(()) + } + + /// Pushes several values into the queue. + /// + /// SAFETY: The caller should ensure that we have exclusive access to the + /// `queue_next` field in the provided tasks. + #[inline] + pub(crate) fn push_batch(&self, mut iter: I) + where + I: Iterator>, + { + let first = match iter.next() { + Some(first) => first.into_raw(), + None => return, + }; + + // Link up all the tasks. + let mut prev = first; + let mut counter = 1; + + // We are going to be called with an `std::iter::Chain`, and that + // iterator overrides `for_each` to something that is easier for the + // compiler to optimize than a loop. + iter.map(|next| next.into_raw()).for_each(|next| { + // safety: The caller guarantees exclusive access to this field. + set_next(prev, Some(next)); + prev = next; + counter += 1; + }); + + // Now that the tasks are linked together, insert them into the + // linked list. + self.push_batch_inner(first, prev, counter); + } + + /// Insert several tasks that have been linked together into the queue. + /// + /// The provided head and tail may be be the same task. In this case, a + /// single task is inserted. + #[inline] + fn push_batch_inner( + &self, + batch_head: NonNull, + batch_tail: NonNull, + num: usize, + ) { + debug_assert!(get_next(batch_tail).is_none()); + + let mut p = self.pointers.lock(); + + if let Some(tail) = p.tail { + set_next(tail, Some(batch_head)); + } else { + p.head = Some(batch_head); + } + + p.tail = Some(batch_tail); + + // Increment the count. + // + // safety: All updates to the len atomic are guarded by the mutex. As + // such, a non-atomic load followed by a store is safe. + let len = unsafe { self.len.unsync_load() }; + + self.len.store(len + num, Release); + } + + pub(crate) fn pop(&self) -> Option> { + // Fast path, if len == 0, then there are no values + if self.is_empty() { + return None; + } + + let mut p = self.pointers.lock(); + + // It is possible to hit null here if another thread popped the last + // task between us checking `len` and acquiring the lock. + let task = p.head?; + + p.head = get_next(task); + + if p.head.is_none() { + p.tail = None; + } + + set_next(task, None); + + // Decrement the count. + // + // safety: All updates to the len atomic are guarded by the mutex. As + // such, a non-atomic load followed by a store is safe. + self.len + .store(unsafe { self.len.unsync_load() } - 1, Release); + + // safety: a `Notified` is pushed into the queue and now it is popped! + Some(unsafe { task::Notified::from_raw(task) }) + } +} + +impl Drop for Inject { + fn drop(&mut self) { + if !std::thread::panicking() { + assert!(self.pop().is_none(), "queue not empty"); + } + } +} + +fn get_next(header: NonNull) -> Option> { + unsafe { header.as_ref().queue_next.with(|ptr| *ptr) } +} + +fn set_next(header: NonNull, val: Option>) { + unsafe { + header.as_ref().set_next(val); + } +} diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 6b1b8c638..a4d4146fe 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -9,6 +9,11 @@ pub use self::error::JoinError; mod harness; use self::harness::Harness; +cfg_rt_multi_thread! { + mod inject; + pub(super) use self::inject::Inject; +} + mod join; #[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411 pub use self::join::JoinHandle; @@ -134,10 +139,6 @@ cfg_rt_multi_thread! { pub(crate) unsafe fn from_raw(ptr: NonNull
) -> Notified { Notified(Task::from_raw(ptr)) } - - pub(crate) fn header(&self) -> &Header { - self.0.header() - } } impl Task { diff --git a/tokio/src/runtime/tests/loom_queue.rs b/tokio/src/runtime/tests/loom_queue.rs index 34da7fd66..977c9159b 100644 --- a/tokio/src/runtime/tests/loom_queue.rs +++ b/tokio/src/runtime/tests/loom_queue.rs @@ -1,5 +1,5 @@ use crate::runtime::queue; -use crate::runtime::task::{self, Schedule, Task}; +use crate::runtime::task::{self, Inject, Schedule, Task}; use loom::thread; @@ -7,7 +7,7 @@ use loom::thread; fn basic() { loom::model(|| { let (steal, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); let th = thread::spawn(move || { let (_, mut local) = queue::local(); @@ -61,7 +61,7 @@ fn basic() { fn steal_overflow() { loom::model(|| { let (steal, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); let th = thread::spawn(move || { let (_, mut local) = queue::local(); @@ -129,7 +129,7 @@ fn multi_stealer() { loom::model(|| { let (steal, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); // Push work for _ in 0..NUM_TASKS { @@ -166,7 +166,7 @@ fn chained_steal() { loom::model(|| { let (s1, mut l1) = queue::local(); let (s2, mut l2) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); // Load up some tasks for _ in 0..4 { diff --git a/tokio/src/runtime/tests/queue.rs b/tokio/src/runtime/tests/queue.rs index b2962f154..e08dc6d99 100644 --- a/tokio/src/runtime/tests/queue.rs +++ b/tokio/src/runtime/tests/queue.rs @@ -1,5 +1,5 @@ use crate::runtime::queue; -use crate::runtime::task::{self, Schedule, Task}; +use crate::runtime::task::{self, Inject, Schedule, Task}; use std::thread; use std::time::Duration; @@ -7,7 +7,7 @@ use std::time::Duration; #[test] fn fits_256() { let (_, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); for _ in 0..256 { let (task, _) = super::joinable::<_, Runtime>(async {}); @@ -22,7 +22,7 @@ fn fits_256() { #[test] fn overflow() { let (_, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); for _ in 0..257 { let (task, _) = super::joinable::<_, Runtime>(async {}); @@ -46,7 +46,7 @@ fn overflow() { fn steal_batch() { let (steal1, mut local1) = queue::local(); let (_, mut local2) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); for _ in 0..4 { let (task, _) = super::joinable::<_, Runtime>(async {}); @@ -78,7 +78,7 @@ fn stress1() { for _ in 0..NUM_ITER { let (steal, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); let th = thread::spawn(move || { let (_, mut local) = queue::local(); @@ -134,7 +134,7 @@ fn stress2() { for _ in 0..NUM_ITER { let (steal, mut local) = queue::local(); - let inject = queue::Inject::new(); + let inject = Inject::new(); let th = thread::spawn(move || { let (_, mut local) = queue::local(); diff --git a/tokio/src/runtime/thread_pool/worker.rs b/tokio/src/runtime/thread_pool/worker.rs index 4ae0f5a25..e91e2ea4b 100644 --- a/tokio/src/runtime/thread_pool/worker.rs +++ b/tokio/src/runtime/thread_pool/worker.rs @@ -11,7 +11,7 @@ use crate::park::{Park, Unpark}; use crate::runtime; use crate::runtime::enter::EnterContext; use crate::runtime::park::{Parker, Unparker}; -use crate::runtime::task::OwnedTasks; +use crate::runtime::task::{Inject, OwnedTasks}; use crate::runtime::thread_pool::{AtomicCell, Idle}; use crate::runtime::{queue, task}; use crate::util::FastRand; @@ -70,7 +70,7 @@ pub(super) struct Shared { remotes: Box<[Remote]>, /// Submit work to the scheduler while **not** currently on a worker thread. - inject: queue::Inject>, + inject: Inject>, /// Coordinates idle workers idle: Idle, @@ -147,7 +147,7 @@ pub(super) fn create(size: usize, park: Parker) -> (Arc, Launch) { let shared = Arc::new(Shared { remotes: remotes.into_boxed_slice(), - inject: queue::Inject::new(), + inject: Inject::new(), idle: Idle::new(size), owned: OwnedTasks::new(), shutdown_cores: Mutex::new(vec![]), @@ -563,7 +563,7 @@ impl Core { impl Worker { /// Returns a reference to the scheduler's injection queue - fn inject(&self) -> &queue::Inject> { + fn inject(&self) -> &Inject> { &self.shared.inject } } From 80d8d40a341e37f6c416cc46d30dd3e7d601c972 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 12 Jul 2021 11:19:14 +0200 Subject: [PATCH 16/35] sync: clean up OnceCell (#3945) --- tokio/src/sync/once_cell.rs | 365 ++++++++++++++++++++---------------- 1 file changed, 208 insertions(+), 157 deletions(-) diff --git a/tokio/src/sync/once_cell.rs b/tokio/src/sync/once_cell.rs index ce55d9e35..91705a558 100644 --- a/tokio/src/sync/once_cell.rs +++ b/tokio/src/sync/once_cell.rs @@ -1,4 +1,4 @@ -use super::Semaphore; +use super::{Semaphore, SemaphorePermit, TryAcquireError}; use crate::loom::cell::UnsafeCell; use std::error::Error; use std::fmt; @@ -8,15 +8,30 @@ use std::ops::Drop; use std::ptr; use std::sync::atomic::{AtomicBool, Ordering}; -/// A thread-safe cell which can be written to only once. +// This file contains an implementation of an OnceCell. The principle +// behind the safety the of the cell is that any thread with an `&OnceCell` may +// access the `value` field according the following rules: +// +// 1. When `value_set` is false, the `value` field may be modified by the +// thread holding the permit on the semaphore. +// 2. When `value_set` is true, the `value` field may be accessed immutably by +// any thread. +// +// It is an invariant that if the semaphore is closed, then `value_set` is true. +// The reverse does not necessarily hold — but if not, the semaphore may not +// have any available permits. +// +// A thread with a `&mut OnceCell` may modify the value in any way it wants as +// long as the invariants are upheld. + +/// A thread-safe cell that can be written to only once. /// -/// Provides the functionality to either set the value, in case `OnceCell` -/// is uninitialized, or get the already initialized value by using an async -/// function via [`OnceCell::get_or_init`]. -/// -/// [`OnceCell::get_or_init`]: crate::sync::OnceCell::get_or_init +/// A `OnceCell` is typically used for global variables that need to be +/// initialized once on first use, but need no further changes. The `OnceCell` +/// in Tokio allows the initialization procedure to be asynchronous. /// /// # Examples +/// /// ``` /// use tokio::sync::OnceCell; /// @@ -28,8 +43,28 @@ use std::sync::atomic::{AtomicBool, Ordering}; /// /// #[tokio::main] /// async fn main() { -/// let result1 = ONCE.get_or_init(some_computation).await; -/// assert_eq!(*result1, 2); +/// let result = ONCE.get_or_init(some_computation).await; +/// assert_eq!(*result, 2); +/// } +/// ``` +/// +/// It is often useful to write a wrapper method for accessing the value. +/// +/// ``` +/// use tokio::sync::OnceCell; +/// +/// static ONCE: OnceCell = OnceCell::const_new(); +/// +/// async fn get_global_integer() -> &'static u32 { +/// ONCE.get_or_init(|| async { +/// 1 + 1 +/// }).await +/// } +/// +/// #[tokio::main] +/// async fn main() { +/// let result = get_global_integer().await; +/// assert_eq!(*result, 2); /// } /// ``` pub struct OnceCell { @@ -68,7 +103,7 @@ impl Eq for OnceCell {} impl Drop for OnceCell { fn drop(&mut self) { - if self.initialized() { + if self.initialized_mut() { unsafe { self.value .with_mut(|ptr| ptr::drop_in_place((&mut *ptr).as_mut_ptr())); @@ -90,7 +125,7 @@ impl From for OnceCell { } impl OnceCell { - /// Creates a new uninitialized OnceCell instance. + /// Creates a new empty `OnceCell` instance. pub fn new() -> Self { OnceCell { value_set: AtomicBool::new(false), @@ -99,8 +134,9 @@ impl OnceCell { } } - /// Creates a new initialized OnceCell instance if `value` is `Some`, otherwise - /// has the same functionality as [`OnceCell::new`]. + /// Creates a new `OnceCell` that contains the provided value, if any. + /// + /// If the `Option` is `None`, this is equivalent to `OnceCell::new`. /// /// [`OnceCell::new`]: crate::sync::OnceCell::new pub fn new_with(value: Option) -> Self { @@ -111,8 +147,31 @@ impl OnceCell { } } - /// Creates a new uninitialized OnceCell instance. - #[cfg(all(feature = "parking_lot", not(all(loom, test)),))] + /// Creates a new empty `OnceCell` instance. + /// + /// Equivalent to `OnceCell::new`, except that it can be used in static + /// variables. + /// + /// # Example + /// + /// ``` + /// use tokio::sync::OnceCell; + /// + /// static ONCE: OnceCell = OnceCell::const_new(); + /// + /// async fn get_global_integer() -> &'static u32 { + /// ONCE.get_or_init(|| async { + /// 1 + 1 + /// }).await + /// } + /// + /// #[tokio::main] + /// async fn main() { + /// let result = get_global_integer().await; + /// assert_eq!(*result, 2); + /// } + /// ``` + #[cfg(all(feature = "parking_lot", not(all(loom, test))))] #[cfg_attr(docsrs, doc(cfg(feature = "parking_lot")))] pub const fn const_new() -> Self { OnceCell { @@ -122,33 +181,48 @@ impl OnceCell { } } - /// Whether the value of the OnceCell is set or not. + /// Returns `true` if the `OnceCell` currently contains a value, and `false` + /// otherwise. pub fn initialized(&self) -> bool { + // Using acquire ordering so any threads that read a true from this + // atomic is able to read the value. self.value_set.load(Ordering::Acquire) } - // SAFETY: safe to call only once self.initialized() is true + /// Returns `true` if the `OnceCell` currently contains a value, and `false` + /// otherwise. + fn initialized_mut(&mut self) -> bool { + *self.value_set.get_mut() + } + + // SAFETY: The OnceCell must not be empty. unsafe fn get_unchecked(&self) -> &T { &*self.value.with(|ptr| (*ptr).as_ptr()) } - // SAFETY: safe to call only once self.initialized() is true. Safe because - // because of the mutable reference. + // SAFETY: The OnceCell must not be empty. unsafe fn get_unchecked_mut(&mut self) -> &mut T { &mut *self.value.with_mut(|ptr| (*ptr).as_mut_ptr()) } - // SAFETY: safe to call only once a permit on the semaphore has been - // acquired - unsafe fn set_value(&self, value: T) { - self.value.with_mut(|ptr| (*ptr).as_mut_ptr().write(value)); + fn set_value(&self, value: T, permit: SemaphorePermit<'_>) -> &T { + // SAFETY: We are holding the only permit on the semaphore. + unsafe { + self.value.with_mut(|ptr| (*ptr).as_mut_ptr().write(value)); + } + + // Using release ordering so any threads that read a true from this + // atomic is able to read the value we just stored. self.value_set.store(true, Ordering::Release); self.semaphore.close(); + permit.forget(); + + // SAFETY: We just initialized the cell. + unsafe { self.get_unchecked() } } - /// Tries to get a reference to the value of the OnceCell. - /// - /// Returns None if the value of the OnceCell hasn't previously been initialized. + /// Returns a reference to the value currently stored in the `OnceCell`, or + /// `None` if the `OnceCell` is empty. pub fn get(&self) -> Option<&T> { if self.initialized() { Some(unsafe { self.get_unchecked() }) @@ -157,179 +231,161 @@ impl OnceCell { } } - /// Tries to return a mutable reference to the value of the cell. + /// Returns a mutable reference to the value currently stored in the + /// `OnceCell`, or `None` if the `OnceCell` is empty. /// - /// Returns None if the cell hasn't previously been initialized. + /// Since this call borrows the `OnceCell` mutably, it is safe to mutate the + /// value inside the `OnceCell` — the mutable borrow statically guarantees + /// no other references exist. pub fn get_mut(&mut self) -> Option<&mut T> { - if self.initialized() { + if self.initialized_mut() { Some(unsafe { self.get_unchecked_mut() }) } else { None } } - /// Sets the value of the OnceCell to the argument value. + /// Set the value of the `OnceCell` to the given value if the `OnceCell` is + /// empty. /// - /// If the value of the OnceCell was already set prior to this call - /// then [`SetError::AlreadyInitializedError`] is returned. If another thread - /// is initializing the cell while this method is called, - /// [`SetError::InitializingError`] is returned. In order to wait - /// for an ongoing initialization to finish, call - /// [`OnceCell::get_or_init`] instead. + /// If the `OnceCell` already has a value, this call will fail with an + /// [`SetError::AlreadyInitializedError`]. + /// + /// If the `OnceCell` is empty, but some other task is currently trying to + /// set the value, this call will fail with [`SetError::InitializingError`]. /// /// [`SetError::AlreadyInitializedError`]: crate::sync::SetError::AlreadyInitializedError /// [`SetError::InitializingError`]: crate::sync::SetError::InitializingError - /// ['OnceCell::get_or_init`]: crate::sync::OnceCell::get_or_init pub fn set(&self, value: T) -> Result<(), SetError> { - if !self.initialized() { - // Another thread might be initializing the cell, in which case `try_acquire` will - // return an error - match self.semaphore.try_acquire() { - Ok(_permit) => { - if !self.initialized() { - // SAFETY: There is only one permit on the semaphore, hence only one - // mutable reference is created - unsafe { self.set_value(value) }; - - return Ok(()); - } else { - unreachable!( - "acquired the permit after OnceCell value was already initialized." - ); - } - } - _ => { - // Couldn't acquire the permit, look if initializing process is already completed - if !self.initialized() { - return Err(SetError::InitializingError(value)); - } - } - } + if self.initialized() { + return Err(SetError::AlreadyInitializedError(value)); } - Err(SetError::AlreadyInitializedError(value)) + // Another task might be initializing the cell, in which case + // `try_acquire` will return an error. If we succeed to acquire the + // permit, then we can set the value. + match self.semaphore.try_acquire() { + Ok(permit) => { + debug_assert!(!self.initialized()); + self.set_value(value, permit); + Ok(()) + } + Err(TryAcquireError::NoPermits) => { + // Some other task is holding the permit. That task is + // currently trying to initialize the value. + Err(SetError::InitializingError(value)) + } + Err(TryAcquireError::Closed) => { + // The semaphore was closed. Some other task has initialized + // the value. + Err(SetError::AlreadyInitializedError(value)) + } + } } - /// Tries to initialize the value of the OnceCell using the async function `f`. - /// If the value of the OnceCell was already initialized prior to this call, - /// a reference to that initialized value is returned. If some other thread - /// initiated the initialization prior to this call and the initialization - /// hasn't completed, this call waits until the initialization is finished. + /// Get the value currently in the `OnceCell`, or initialize it with the + /// given asynchronous operation. /// - /// This will deadlock if `f` tries to initialize the cell itself. + /// If some other task is currently working on initializing the `OnceCell`, + /// this call will wait for that other task to finish, then return the value + /// that the other task produced. + /// + /// If the provided operation is cancelled or panics, the initialization + /// attempt is cancelled. If there are other tasks waiting for the value to + /// be initialized, one of them will start another attempt at initializing + /// the value. + /// + /// This will deadlock if `f` tries to initialize the cell recursively. pub async fn get_or_init(&self, f: F) -> &T where F: FnOnce() -> Fut, Fut: Future, { if self.initialized() { - // SAFETY: once the value is initialized, no mutable references are given out, so - // we can give out arbitrarily many immutable references + // SAFETY: The OnceCell has been fully initialized. unsafe { self.get_unchecked() } } else { - // After acquire().await we have either acquired a permit while self.value - // is still uninitialized, or the current thread is awoken after another thread - // has initialized the value and closed the semaphore, in which case self.initialized - // is true and we don't set the value here + // Here we try to acquire the semaphore permit. Holding the permit + // will allow us to set the value of the OnceCell, and prevents + // other tasks from initializing the OnceCell while we are holding + // it. match self.semaphore.acquire().await { - Ok(_permit) => { - if !self.initialized() { - // If `f()` panics or `select!` is called, this `get_or_init` call - // is aborted and the semaphore permit is dropped. - let value = f().await; + Ok(permit) => { + debug_assert!(!self.initialized()); - // SAFETY: There is only one permit on the semaphore, hence only one - // mutable reference is created - unsafe { self.set_value(value) }; + // If `f()` panics or `select!` is called, this + // `get_or_init` call is aborted and the semaphore permit is + // dropped. + let value = f().await; - // SAFETY: once the value is initialized, no mutable references are given out, so - // we can give out arbitrarily many immutable references - unsafe { self.get_unchecked() } - } else { - unreachable!("acquired semaphore after value was already initialized."); - } + self.set_value(value, permit) } Err(_) => { - if self.initialized() { - // SAFETY: once the value is initialized, no mutable references are given out, so - // we can give out arbitrarily many immutable references - unsafe { self.get_unchecked() } - } else { - unreachable!( - "Semaphore closed, but the OnceCell has not been initialized." - ); - } + debug_assert!(self.initialized()); + + // SAFETY: The semaphore has been closed. This only happens + // when the OnceCell is fully initialized. + unsafe { self.get_unchecked() } } } } } - /// Tries to initialize the value of the OnceCell using the async function `f`. - /// If the value of the OnceCell was already initialized prior to this call, - /// a reference to that initialized value is returned. If some other thread - /// initiated the initialization prior to this call and the initialization - /// hasn't completed, this call waits until the initialization is finished. - /// If the function argument `f` returns an error, `get_or_try_init` - /// returns that error, otherwise the result of `f` will be stored in the cell. + /// Get the value currently in the `OnceCell`, or initialize it with the + /// given asynchronous operation. /// - /// This will deadlock if `f` tries to initialize the cell itself. + /// If some other task is currently working on initializing the `OnceCell`, + /// this call will wait for that other task to finish, then return the value + /// that the other task produced. + /// + /// If the provided operation returns an error, is cancelled or panics, the + /// initialization attempt is cancelled. If there are other tasks waiting + /// for the value to be initialized, one of them will start another attempt + /// at initializing the value. + /// + /// This will deadlock if `f` tries to initialize the cell recursively. pub async fn get_or_try_init(&self, f: F) -> Result<&T, E> where F: FnOnce() -> Fut, Fut: Future>, { if self.initialized() { - // SAFETY: once the value is initialized, no mutable references are given out, so - // we can give out arbitrarily many immutable references + // SAFETY: The OnceCell has been fully initialized. unsafe { Ok(self.get_unchecked()) } } else { - // After acquire().await we have either acquired a permit while self.value - // is still uninitialized, or the current thread is awoken after another thread - // has initialized the value and closed the semaphore, in which case self.initialized - // is true and we don't set the value here + // Here we try to acquire the semaphore permit. Holding the permit + // will allow us to set the value of the OnceCell, and prevents + // other tasks from initializing the OnceCell while we are holding + // it. match self.semaphore.acquire().await { - Ok(_permit) => { - if !self.initialized() { - // If `f()` panics or `select!` is called, this `get_or_try_init` call - // is aborted and the semaphore permit is dropped. - let value = f().await; + Ok(permit) => { + debug_assert!(!self.initialized()); - match value { - Ok(value) => { - // SAFETY: There is only one permit on the semaphore, hence only one - // mutable reference is created - unsafe { self.set_value(value) }; + // If `f()` panics or `select!` is called, this + // `get_or_try_init` call is aborted and the semaphore + // permit is dropped. + let value = f().await; - // SAFETY: once the value is initialized, no mutable references are given out, so - // we can give out arbitrarily many immutable references - unsafe { Ok(self.get_unchecked()) } - } - Err(e) => Err(e), - } - } else { - unreachable!("acquired semaphore after value was already initialized."); + match value { + Ok(value) => Ok(self.set_value(value, permit)), + Err(e) => Err(e), } } Err(_) => { - if self.initialized() { - // SAFETY: once the value is initialized, no mutable references are given out, so - // we can give out arbitrarily many immutable references - unsafe { Ok(self.get_unchecked()) } - } else { - unreachable!( - "Semaphore closed, but the OnceCell has not been initialized." - ); - } + debug_assert!(self.initialized()); + + // SAFETY: The semaphore has been closed. This only happens + // when the OnceCell is fully initialized. + unsafe { Ok(self.get_unchecked()) } } } } } - /// Moves the value out of the cell, destroying the cell in the process. - /// - /// Returns `None` if the cell is uninitialized. + /// Take the value from the cell, destroying the cell in the process. + /// Returns `None` if the cell is empty. pub fn into_inner(mut self) -> Option { - if self.initialized() { + if self.initialized_mut() { // Set to uninitialized for the destructor of `OnceCell` to work properly *self.value_set.get_mut() = false; Some(unsafe { self.value.with(|ptr| ptr::read(ptr).assume_init()) }) @@ -338,20 +394,18 @@ impl OnceCell { } } - /// Takes ownership of the current value, leaving the cell uninitialized. - /// - /// Returns `None` if the cell is uninitialized. + /// Takes ownership of the current value, leaving the cell empty. Returns + /// `None` if the cell is empty. pub fn take(&mut self) -> Option { std::mem::take(self).into_inner() } } -// Since `get` gives us access to immutable references of the -// OnceCell, OnceCell can only be Sync if T is Sync, otherwise -// OnceCell would allow sharing references of !Sync values across -// threads. We need T to be Send in order for OnceCell to by Sync -// because we can use `set` on `&OnceCell` to send -// values (of type T) across threads. +// Since `get` gives us access to immutable references of the OnceCell, OnceCell +// can only be Sync if T is Sync, otherwise OnceCell would allow sharing +// references of !Sync values across threads. We need T to be Send in order for +// OnceCell to by Sync because we can use `set` on `&OnceCell` to send values +// (of type T) across threads. unsafe impl Sync for OnceCell {} // Access to OnceCell's value is guarded by the semaphore permit @@ -359,20 +413,17 @@ unsafe impl Sync for OnceCell {} // it's safe to send it to another thread unsafe impl Send for OnceCell {} -/// Errors that can be returned from [`OnceCell::set`] +/// Errors that can be returned from [`OnceCell::set`]. /// /// [`OnceCell::set`]: crate::sync::OnceCell::set #[derive(Debug, PartialEq)] pub enum SetError { - /// Error resulting from [`OnceCell::set`] calls if the cell was previously initialized. + /// The cell was already initialized when [`OnceCell::set`] was called. /// /// [`OnceCell::set`]: crate::sync::OnceCell::set AlreadyInitializedError(T), - /// Error resulting from [`OnceCell::set`] calls when the cell is currently being - /// initialized during the calls to that method. - /// - /// [`OnceCell::set`]: crate::sync::OnceCell::set + /// The cell is currently being initialized. InitializingError(T), } From ccd495647d04cd71e9e9bfe21ac61289ec35968e Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 12 Jul 2021 11:21:10 +0200 Subject: [PATCH 17/35] chore: update nightly version (#3953) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bfb3b727..dd38a52c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ name: CI env: RUSTFLAGS: -Dwarnings RUST_BACKTRACE: 1 - nightly: nightly-2021-04-25 + nightly: nightly-2021-07-09 minrust: 1.45.2 jobs: From 3fd88dac644728a9ae9ffa8bc518b090d90ed51b Mon Sep 17 00:00:00 2001 From: Alan Somers Date: Mon, 12 Jul 2021 03:36:50 -0600 Subject: [PATCH 18/35] net: fix the uds_datagram tests with the latest nightly stdlib (#3952) --- tokio/tests/uds_datagram.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tokio/tests/uds_datagram.rs b/tokio/tests/uds_datagram.rs index 10314bebf..4d2846865 100644 --- a/tokio/tests/uds_datagram.rs +++ b/tokio/tests/uds_datagram.rs @@ -87,9 +87,12 @@ async fn try_send_recv_never_block() -> io::Result<()> { dgram1.writable().await.unwrap(); match dgram1.try_send(payload) { - Err(err) => match err.kind() { - io::ErrorKind::WouldBlock | io::ErrorKind::Other => break, - _ => unreachable!("unexpected error {:?}", err), + Err(err) => match (err.kind(), err.raw_os_error()) { + (io::ErrorKind::WouldBlock, _) => break, + (_, Some(libc::ENOBUFS)) => break, + _ => { + panic!("unexpected error {:?}", err); + } }, Ok(len) => { assert_eq!(len, payload.len()); @@ -291,9 +294,12 @@ async fn try_recv_buf_never_block() -> io::Result<()> { dgram1.writable().await.unwrap(); match dgram1.try_send(payload) { - Err(err) => match err.kind() { - io::ErrorKind::WouldBlock | io::ErrorKind::Other => break, - _ => unreachable!("unexpected error {:?}", err), + Err(err) => match (err.kind(), err.raw_os_error()) { + (io::ErrorKind::WouldBlock, _) => break, + (_, Some(libc::ENOBUFS)) => break, + _ => { + panic!("unexpected error {:?}", err); + } }, Ok(len) => { assert_eq!(len, payload.len()); From 6610ba9bd6b6a48d59f80573b5fa307972ace55a Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 12 Jul 2021 14:42:00 +0200 Subject: [PATCH 19/35] runtime: use OwnedTasks in LocalSet (#3950) --- tokio/src/runtime/task/mod.rs | 2 +- tokio/src/task/local.rs | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index a4d4146fe..5e2477906 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -19,7 +19,7 @@ mod join; pub use self::join::JoinHandle; mod list; -pub(super) use self::list::OwnedTasks; +pub(crate) use self::list::OwnedTasks; mod raw; use self::raw::RawTask; diff --git a/tokio/src/task/local.rs b/tokio/src/task/local.rs index 49b0ec6c4..01af63fc9 100644 --- a/tokio/src/task/local.rs +++ b/tokio/src/task/local.rs @@ -1,7 +1,6 @@ //! Runs `!Send` futures on the current thread. -use crate::runtime::task::{self, JoinHandle, Task}; +use crate::runtime::task::{self, JoinHandle, OwnedTasks, Task}; use crate::sync::AtomicWaker; -use crate::util::linked_list::{Link, LinkedList}; use std::cell::{Cell, RefCell}; use std::collections::VecDeque; @@ -233,7 +232,7 @@ struct Context { struct Tasks { /// Collection of all active tasks spawned onto this executor. - owned: LinkedList>, > as Link>::Target>, + owned: OwnedTasks>, /// Local run queue sender and receiver. queue: VecDeque>>, @@ -334,7 +333,7 @@ impl LocalSet { tick: Cell::new(0), context: Context { tasks: RefCell::new(Tasks { - owned: LinkedList::new(), + owned: OwnedTasks::new(), queue: VecDeque::with_capacity(INITIAL_CAPACITY), }), shared: Arc::new(Shared { @@ -682,17 +681,14 @@ impl task::Schedule for Arc { } fn release(&self, task: &Task) -> Option> { - use std::ptr::NonNull; - CURRENT.with(|maybe_cx| { let cx = maybe_cx.expect("scheduler context missing"); assert!(cx.shared.ptr_eq(self)); - let ptr = NonNull::from(task.header()); // safety: task must be contained by list. It is inserted into the // list in `bind`. - unsafe { cx.tasks.borrow_mut().owned.remove(ptr) } + unsafe { cx.tasks.borrow_mut().owned.remove(&task) } }) } From 7a11cfd77a00d383f090a876768f34c66d5dc538 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 12 Jul 2021 16:38:19 +0200 Subject: [PATCH 20/35] chore: update async_send_sync test (#3943) --- tokio/src/task/task_local.rs | 10 +- tokio/tests/async_send_sync.rs | 837 ++++++++++++++++++++++----------- 2 files changed, 568 insertions(+), 279 deletions(-) diff --git a/tokio/src/task/task_local.rs b/tokio/src/task/task_local.rs index b55c6abc4..0bb90a190 100644 --- a/tokio/src/task/task_local.rs +++ b/tokio/src/task/task_local.rs @@ -2,6 +2,7 @@ use pin_project_lite::pin_project; use std::cell::RefCell; use std::error::Error; use std::future::Future; +use std::marker::PhantomPinned; use std::pin::Pin; use std::task::{Context, Poll}; use std::{fmt, thread}; @@ -123,6 +124,7 @@ impl LocalKey { local: &self, slot: Some(value), future: f, + _pinned: PhantomPinned, } } @@ -147,12 +149,14 @@ impl LocalKey { where F: FnOnce() -> R, { - let mut scope = TaskLocalFuture { + let scope = TaskLocalFuture { local: &self, slot: Some(value), future: (), + _pinned: PhantomPinned, }; - Pin::new(&mut scope).with_task(|_| f()) + crate::pin!(scope); + scope.with_task(|_| f()) } /// Accesses the current task-local and runs the provided closure. @@ -234,6 +238,8 @@ pin_project! { slot: Option, #[pin] future: F, + #[pin] + _pinned: PhantomPinned, } } diff --git a/tokio/tests/async_send_sync.rs b/tokio/tests/async_send_sync.rs index 01e608186..12d132393 100644 --- a/tokio/tests/async_send_sync.rs +++ b/tokio/tests/async_send_sync.rs @@ -4,13 +4,30 @@ use std::cell::Cell; use std::future::Future; -use std::io::{Cursor, SeekFrom}; +use std::io::SeekFrom; use std::net::SocketAddr; use std::pin::Pin; use std::rc::Rc; use tokio::net::TcpStream; use tokio::time::{Duration, Instant}; +// The names of these structs behaves better when sorted. +// Send: Yes, Sync: Yes +#[derive(Clone)] +struct YY {} + +// Send: Yes, Sync: No +#[derive(Clone)] +struct YN { + _value: Cell, +} + +// Send: No, Sync: No +#[derive(Clone)] +struct NN { + _value: Rc, +} + #[allow(dead_code)] type BoxFutureSync = std::pin::Pin + Send + Sync>>; #[allow(dead_code)] @@ -19,11 +36,11 @@ type BoxFutureSend = std::pin::Pin + type BoxFuture = std::pin::Pin>>; #[allow(dead_code)] -type BoxAsyncRead = std::pin::Pin>; +type BoxAsyncRead = std::pin::Pin>; #[allow(dead_code)] -type BoxAsyncSeek = std::pin::Pin>; +type BoxAsyncSeek = std::pin::Pin>; #[allow(dead_code)] -type BoxAsyncWrite = std::pin::Pin>; +type BoxAsyncWrite = std::pin::Pin>; #[allow(dead_code)] fn require_send(_t: &T) {} @@ -59,310 +76,576 @@ macro_rules! into_todo { x }}; } -macro_rules! assert_value { - ($type:ty: Send & Sync) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f: $type = todo!(); - require_send(&f); - require_sync(&f); - }; + +macro_rules! async_assert_fn_send { + (Send & $(!)?Sync & $(!)?Unpin, $value:expr) => { + require_send(&$value); }; - ($type:ty: !Send & Sync) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f: $type = todo!(); - AmbiguousIfSend::some_item(&f); - require_sync(&f); - }; - }; - ($type:ty: Send & !Sync) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f: $type = todo!(); - require_send(&f); - AmbiguousIfSync::some_item(&f); - }; - }; - ($type:ty: !Send & !Sync) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f: $type = todo!(); - AmbiguousIfSend::some_item(&f); - AmbiguousIfSync::some_item(&f); - }; - }; - ($type:ty: Unpin) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f: $type = todo!(); - require_unpin(&f); - }; + (!Send & $(!)?Sync & $(!)?Unpin, $value:expr) => { + AmbiguousIfSend::some_item(&$value); }; } +macro_rules! async_assert_fn_sync { + ($(!)?Send & Sync & $(!)?Unpin, $value:expr) => { + require_sync(&$value); + }; + ($(!)?Send & !Sync & $(!)?Unpin, $value:expr) => { + AmbiguousIfSync::some_item(&$value); + }; +} +macro_rules! async_assert_fn_unpin { + ($(!)?Send & $(!)?Sync & Unpin, $value:expr) => { + require_unpin(&$value); + }; + ($(!)?Send & $(!)?Sync & !Unpin, $value:expr) => { + AmbiguousIfUnpin::some_item(&$value); + }; +} + macro_rules! async_assert_fn { - ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): Send & Sync) => { + ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): $($tok:tt)*) => { #[allow(unreachable_code)] #[allow(unused_variables)] const _: fn() = || { let f = $($f $(::<$($generic),*>)? )::+( $( into_todo!($arg) ),* ); - require_send(&f); - require_sync(&f); + async_assert_fn_send!($($tok)*, f); + async_assert_fn_sync!($($tok)*, f); + async_assert_fn_unpin!($($tok)*, f); }; }; - ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): Send & !Sync) => { +} +macro_rules! assert_value { + ($type:ty: $($tok:tt)*) => { #[allow(unreachable_code)] #[allow(unused_variables)] const _: fn() = || { - let f = $($f $(::<$($generic),*>)? )::+( $( into_todo!($arg) ),* ); - require_send(&f); - AmbiguousIfSync::some_item(&f); - }; - }; - ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): !Send & Sync) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f = $($f $(::<$($generic),*>)? )::+( $( into_todo!($arg) ),* ); - AmbiguousIfSend::some_item(&f); - require_sync(&f); - }; - }; - ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): !Send & !Sync) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f = $($f $(::<$($generic),*>)? )::+( $( into_todo!($arg) ),* ); - AmbiguousIfSend::some_item(&f); - AmbiguousIfSync::some_item(&f); - }; - }; - ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): !Unpin) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f = $($f $(::<$($generic),*>)? )::+( $( into_todo!($arg) ),* ); - AmbiguousIfUnpin::some_item(&f); - }; - }; - ($($f:ident $(< $($generic:ty),* > )? )::+($($arg:ty),*): Unpin) => { - #[allow(unreachable_code)] - #[allow(unused_variables)] - const _: fn() = || { - let f = $($f $(::<$($generic),*>)? )::+( $( into_todo!($arg) ),* ); - require_unpin(&f); + let f: $type = todo!(); + async_assert_fn_send!($($tok)*, f); + async_assert_fn_sync!($($tok)*, f); + async_assert_fn_unpin!($($tok)*, f); }; }; } -async_assert_fn!(tokio::io::copy(&mut TcpStream, &mut TcpStream): Send & Sync); -async_assert_fn!(tokio::io::empty(): Send & Sync); -async_assert_fn!(tokio::io::repeat(u8): Send & Sync); -async_assert_fn!(tokio::io::sink(): Send & Sync); -async_assert_fn!(tokio::io::split(TcpStream): Send & Sync); -async_assert_fn!(tokio::io::stderr(): Send & Sync); -async_assert_fn!(tokio::io::stdin(): Send & Sync); -async_assert_fn!(tokio::io::stdout(): Send & Sync); -async_assert_fn!(tokio::io::Split>>::next_segment(_): Send & Sync); +assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin); +assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin); +assert_value!(tokio::fs::File: Send & Sync & Unpin); +assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin); +assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin); -async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync); -async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync); -async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync); -async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync); -async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync); -async_assert_fn!(tokio::fs::metadata(&str): Send & Sync); -async_assert_fn!(tokio::fs::read(&str): Send & Sync); -async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync); -async_assert_fn!(tokio::fs::read_link(&str): Send & Sync); -async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync); -async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync); -async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync); -async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync); -async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync); -async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync); -async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync); -async_assert_fn!(tokio::fs::write(&str, Vec): Send & Sync); -async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync); -async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync); -async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync); -async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync); +async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::write(&str, Vec): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::open(&str): Send & Sync); -async_assert_fn!(tokio::fs::File::create(&str): Send & Sync); -async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync); -async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync); -async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync); -async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync); -async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync); -async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync); -async_assert_fn!(tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync); - -async_assert_fn!(tokio::net::lookup_host(SocketAddr): Send & Sync); -async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync); -async_assert_fn!(tokio::net::TcpListener::accept(_): Send & Sync); -async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync); -async_assert_fn!(tokio::net::TcpStream::peek(_, &mut [u8]): Send & Sync); -async_assert_fn!(tokio::net::tcp::ReadHalf::peek(_, &mut [u8]): Send & Sync); -async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync); -async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync); -async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync); -async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync); -async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync); -async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync); +assert_value!(tokio::net::TcpListener: Send & Sync & Unpin); +assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin); +assert_value!(tokio::net::TcpStream: Send & Sync & Unpin); +assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin); +assert_value!(tokio::net::tcp::OwnedReadHalf: Send & Sync & Unpin); +assert_value!(tokio::net::tcp::OwnedWriteHalf: Send & Sync & Unpin); +assert_value!(tokio::net::tcp::ReadHalf<'_>: Send & Sync & Unpin); +assert_value!(tokio::net::tcp::ReuniteError: Send & Sync & Unpin); +assert_value!(tokio::net::tcp::WriteHalf<'_>: Send & Sync & Unpin); +async_assert_fn!(tokio::net::TcpListener::accept(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::TcpStream::peek(_, &mut [u8]): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::TcpStream::readable(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::TcpStream::ready(_, tokio::io::Interest): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::TcpStream::writable(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::lookup_host(SocketAddr): Send & Sync & !Unpin); +async_assert_fn!(tokio::net::tcp::ReadHalf::peek(_, &mut [u8]): Send & Sync & !Unpin); #[cfg(unix)] mod unix_datagram { use super::*; - async_assert_fn!(tokio::net::UnixListener::bind(&str): Send & Sync); - async_assert_fn!(tokio::net::UnixListener::accept(_): Send & Sync); - async_assert_fn!(tokio::net::UnixDatagram::send(_, &[u8]): Send & Sync); - async_assert_fn!(tokio::net::UnixDatagram::recv(_, &mut [u8]): Send & Sync); - async_assert_fn!(tokio::net::UnixDatagram::send_to(_, &[u8], &str): Send & Sync); - async_assert_fn!(tokio::net::UnixDatagram::recv_from(_, &mut [u8]): Send & Sync); - async_assert_fn!(tokio::net::UnixStream::connect(&str): Send & Sync); + use tokio::net::*; + assert_value!(UnixDatagram: Send & Sync & Unpin); + assert_value!(UnixListener: Send & Sync & Unpin); + assert_value!(UnixStream: Send & Sync & Unpin); + assert_value!(unix::OwnedReadHalf: Send & Sync & Unpin); + assert_value!(unix::OwnedWriteHalf: Send & Sync & Unpin); + assert_value!(unix::ReadHalf<'_>: Send & Sync & Unpin); + assert_value!(unix::ReuniteError: Send & Sync & Unpin); + assert_value!(unix::SocketAddr: Send & Sync & Unpin); + assert_value!(unix::UCred: Send & Sync & Unpin); + assert_value!(unix::WriteHalf<'_>: Send & Sync & Unpin); + async_assert_fn!(UnixDatagram::readable(_): Send & Sync & !Unpin); + async_assert_fn!(UnixDatagram::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(UnixDatagram::recv(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(UnixDatagram::recv_from(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(UnixDatagram::send(_, &[u8]): Send & Sync & !Unpin); + async_assert_fn!(UnixDatagram::send_to(_, &[u8], &str): Send & Sync & !Unpin); + async_assert_fn!(UnixDatagram::writable(_): Send & Sync & !Unpin); + async_assert_fn!(UnixListener::accept(_): Send & Sync & !Unpin); + async_assert_fn!(UnixStream::connect(&str): Send & Sync & !Unpin); + async_assert_fn!(UnixStream::readable(_): Send & Sync & !Unpin); + async_assert_fn!(UnixStream::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(UnixStream::writable(_): Send & Sync & !Unpin); } -async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync); -async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync); +#[cfg(windows)] +mod windows_named_pipe { + use super::*; + use tokio::net::windows::named_pipe::*; + assert_value!(ClientOptions: Send & Sync & Unpin); + assert_value!(NamedPipeClient: Send & Sync & Unpin); + assert_value!(NamedPipeServer: Send & Sync & Unpin); + assert_value!(PipeEnd: Send & Sync & Unpin); + assert_value!(PipeInfo: Send & Sync & Unpin); + assert_value!(PipeMode: Send & Sync & Unpin); + assert_value!(ServerOptions: Send & Sync & Unpin); + async_assert_fn!(NamedPipeClient::readable(_): Send & Sync & !Unpin); + async_assert_fn!(NamedPipeClient::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(NamedPipeClient::writable(_): Send & Sync & !Unpin); + async_assert_fn!(NamedPipeServer::connect(_): Send & Sync & !Unpin); + async_assert_fn!(NamedPipeServer::readable(_): Send & Sync & !Unpin); + async_assert_fn!(NamedPipeServer::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(NamedPipeServer::writable(_): Send & Sync & !Unpin); +} + +assert_value!(tokio::process::Child: Send & Sync & Unpin); +assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin); +assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin); +assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin); +assert_value!(tokio::process::Command: Send & Sync & Unpin); +async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin); + +async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin); #[cfg(unix)] -async_assert_fn!(tokio::signal::unix::Signal::recv(_): Send & Sync); +mod unix_signal { + use super::*; + assert_value!(tokio::signal::unix::Signal: Send & Sync & Unpin); + assert_value!(tokio::signal::unix::SignalKind: Send & Sync & Unpin); + async_assert_fn!(tokio::signal::unix::Signal::recv(_): Send & Sync & !Unpin); +} +#[cfg(windows)] +mod windows_signal { + use super::*; + assert_value!(tokio::signal::windows::CtrlC: Send & Sync & Unpin); + assert_value!(tokio::signal::windows::CtrlBreak: Send & Sync & Unpin); + async_assert_fn!(tokio::signal::windows::CtrlC::recv(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::signal::windows::CtrlBreak::recv(_): Send & Sync & !Unpin); +} -async_assert_fn!(tokio::sync::Barrier::wait(_): Send & Sync); -async_assert_fn!(tokio::sync::Mutex::lock(_): Send & Sync); -async_assert_fn!(tokio::sync::Mutex>::lock(_): Send & Sync); -async_assert_fn!(tokio::sync::Mutex>::lock(_): !Send & !Sync); -async_assert_fn!(tokio::sync::Mutex::lock_owned(_): Send & Sync); -async_assert_fn!(tokio::sync::Mutex>::lock_owned(_): Send & Sync); -async_assert_fn!(tokio::sync::Mutex>::lock_owned(_): !Send & !Sync); -async_assert_fn!(tokio::sync::Notify::notified(_): Send & Sync); -async_assert_fn!(tokio::sync::RwLock::read(_): Send & Sync); -async_assert_fn!(tokio::sync::RwLock::write(_): Send & Sync); -async_assert_fn!(tokio::sync::RwLock>::read(_): !Send & !Sync); -async_assert_fn!(tokio::sync::RwLock>::write(_): !Send & !Sync); -async_assert_fn!(tokio::sync::RwLock>::read(_): !Send & !Sync); -async_assert_fn!(tokio::sync::RwLock>::write(_): !Send & !Sync); -async_assert_fn!(tokio::sync::Semaphore::acquire(_): Send & Sync); +assert_value!(tokio::sync::AcquireError: Send & Sync & Unpin); +assert_value!(tokio::sync::Barrier: Send & Sync & Unpin); +assert_value!(tokio::sync::BarrierWaitResult: Send & Sync & Unpin); +assert_value!(tokio::sync::MappedMutexGuard<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::MappedMutexGuard<'_, YN>: Send & !Sync & Unpin); +assert_value!(tokio::sync::MappedMutexGuard<'_, YY>: Send & Sync & Unpin); +assert_value!(tokio::sync::Mutex: !Send & !Sync & Unpin); +assert_value!(tokio::sync::Mutex: Send & Sync & Unpin); +assert_value!(tokio::sync::Mutex: Send & Sync & Unpin); +assert_value!(tokio::sync::MutexGuard<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::MutexGuard<'_, YN>: Send & !Sync & Unpin); +assert_value!(tokio::sync::MutexGuard<'_, YY>: Send & Sync & Unpin); +assert_value!(tokio::sync::Notify: Send & Sync & Unpin); +assert_value!(tokio::sync::OnceCell: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OnceCell: Send & !Sync & Unpin); +assert_value!(tokio::sync::OnceCell: Send & Sync & Unpin); +assert_value!(tokio::sync::OwnedMutexGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMutexGuard: Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMutexGuard: Send & Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockMappedWriteGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockMappedWriteGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockMappedWriteGuard: Send & Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockReadGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockReadGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockReadGuard: Send & Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockWriteGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockWriteGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedRwLockWriteGuard: Send & Sync & Unpin); +assert_value!(tokio::sync::OwnedSemaphorePermit: Send & Sync & Unpin); +assert_value!(tokio::sync::RwLock: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLock: Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLock: Send & Sync & Unpin); +assert_value!(tokio::sync::RwLockMappedWriteGuard<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLockMappedWriteGuard<'_, YN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLockMappedWriteGuard<'_, YY>: Send & Sync & Unpin); +assert_value!(tokio::sync::RwLockReadGuard<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLockReadGuard<'_, YN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLockReadGuard<'_, YY>: Send & Sync & Unpin); +assert_value!(tokio::sync::RwLockWriteGuard<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLockWriteGuard<'_, YN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::RwLockWriteGuard<'_, YY>: Send & Sync & Unpin); +assert_value!(tokio::sync::Semaphore: Send & Sync & Unpin); +assert_value!(tokio::sync::SemaphorePermit<'_>: Send & Sync & Unpin); +assert_value!(tokio::sync::TryAcquireError: Send & Sync & Unpin); +assert_value!(tokio::sync::TryLockError: Send & Sync & Unpin); +assert_value!(tokio::sync::broadcast::Receiver: !Send & !Sync & Unpin); +assert_value!(tokio::sync::broadcast::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::broadcast::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::broadcast::Sender: !Send & !Sync & Unpin); +assert_value!(tokio::sync::broadcast::Sender: Send & Sync & Unpin); +assert_value!(tokio::sync::broadcast::Sender: Send & Sync & Unpin); +assert_value!(tokio::sync::futures::Notified<'_>: Send & Sync & !Unpin); +assert_value!(tokio::sync::mpsc::OwnedPermit: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::OwnedPermit: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::OwnedPermit: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::Permit<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::Permit<'_, YN>: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::Permit<'_, YY>: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::Receiver: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::Sender: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::Sender: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::Sender: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::UnboundedReceiver: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::UnboundedReceiver: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::UnboundedReceiver: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::UnboundedSender: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::UnboundedSender: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::UnboundedSender: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::SendError: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::SendError: Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::SendError: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::SendTimeoutError: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::SendTimeoutError: Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::SendTimeoutError: Send & Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::TrySendError: !Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::TrySendError: Send & !Sync & Unpin); +assert_value!(tokio::sync::mpsc::error::TrySendError: Send & Sync & Unpin); +assert_value!(tokio::sync::oneshot::Receiver: !Send & !Sync & Unpin); +assert_value!(tokio::sync::oneshot::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::oneshot::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::oneshot::Sender: !Send & !Sync & Unpin); +assert_value!(tokio::sync::oneshot::Sender: Send & Sync & Unpin); +assert_value!(tokio::sync::oneshot::Sender: Send & Sync & Unpin); +assert_value!(tokio::sync::watch::Receiver: !Send & !Sync & Unpin); +assert_value!(tokio::sync::watch::Receiver: !Send & !Sync & Unpin); +assert_value!(tokio::sync::watch::Receiver: Send & Sync & Unpin); +assert_value!(tokio::sync::watch::Ref<'_, NN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::watch::Ref<'_, YN>: !Send & !Sync & Unpin); +assert_value!(tokio::sync::watch::Ref<'_, YY>: !Send & Sync & Unpin); +assert_value!(tokio::sync::watch::Sender: !Send & !Sync & Unpin); +assert_value!(tokio::sync::watch::Sender: !Send & !Sync & Unpin); +assert_value!(tokio::sync::watch::Sender: Send & Sync & Unpin); +async_assert_fn!(tokio::sync::Barrier::wait(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Mutex::lock(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::Mutex::lock_owned(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::Mutex::lock(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Mutex::lock_owned(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Mutex::lock(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Mutex::lock_owned(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Notify::notified(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin + Send + Sync>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin + Send>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin> + Send + Sync>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin> + Send>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin>>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin + Send + Sync>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin + Send>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin> + Send + Sync>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin> + Send>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin>>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin + Send + Sync>>): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin + Send>>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_init( _, fn() -> Pin>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin> + Send + Sync>>): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin> + Send>>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::OnceCell::get_or_try_init( _, fn() -> Pin>>>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::RwLock::read(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::RwLock::write(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::RwLock::read(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::RwLock::write(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::RwLock::read(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::RwLock::write(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Semaphore::acquire(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Semaphore::acquire_many(_, u32): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Semaphore::acquire_many_owned(_, u32): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::Semaphore::acquire_owned(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::broadcast::Receiver::recv(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::broadcast::Receiver::recv(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::broadcast::Receiver::recv(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Receiver::recv(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Receiver::recv(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Receiver::recv(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::closed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::reserve(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::reserve_owned(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::send(_, NN): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::send_timeout(_, NN, Duration): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::reserve(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::reserve_owned(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::send(_, YN): Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::send_timeout(_, YN, Duration): Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::reserve(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::reserve_owned(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::send(_, YY): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::Sender::send_timeout(_, YY, Duration): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::UnboundedReceiver::recv(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::UnboundedReceiver::recv(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::UnboundedReceiver::recv(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::UnboundedSender::closed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::UnboundedSender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::mpsc::UnboundedSender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::oneshot::Sender::closed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::oneshot::Sender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::oneshot::Sender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::watch::Receiver::changed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::watch::Receiver::changed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::watch::Receiver::changed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::sync::watch::Sender::closed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::watch::Sender::closed(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::sync::watch::Sender::closed(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::sync::broadcast::Receiver::recv(_): Send & Sync); -async_assert_fn!(tokio::sync::broadcast::Receiver>::recv(_): Send & Sync); -async_assert_fn!(tokio::sync::broadcast::Receiver>::recv(_): !Send & !Sync); +async_assert_fn!(tokio::task::LocalKey::scope(_, u32, BoxFutureSync<()>): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey::scope(_, u32, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey::scope(_, u32, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFutureSync<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey>::scope(_, Rc, BoxFutureSync<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey>::scope(_, Rc, BoxFutureSend<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalKey>::scope(_, Rc, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::LocalSet::run_until(_, BoxFutureSync<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::unconstrained(BoxFuture<()>): !Send & !Sync & Unpin); +async_assert_fn!(tokio::task::unconstrained(BoxFutureSend<()>): Send & !Sync & Unpin); +async_assert_fn!(tokio::task::unconstrained(BoxFutureSync<()>): Send & Sync & Unpin); +assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin); +assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); +assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); +assert_value!(tokio::task::JoinHandle: !Send & !Sync & Unpin); -async_assert_fn!(tokio::sync::mpsc::Receiver::recv(_): Send & Sync); -async_assert_fn!(tokio::sync::mpsc::Receiver>::recv(_): Send & Sync); -async_assert_fn!(tokio::sync::mpsc::Receiver>::recv(_): !Send & !Sync); -async_assert_fn!(tokio::sync::mpsc::Sender::send(_, u8): Send & Sync); -async_assert_fn!(tokio::sync::mpsc::Sender>::send(_, Cell): Send & !Sync); -async_assert_fn!(tokio::sync::mpsc::Sender>::send(_, Rc): !Send & !Sync); +assert_value!(tokio::runtime::Builder: Send & Sync & Unpin); +assert_value!(tokio::runtime::EnterGuard<'_>: Send & Sync & Unpin); +assert_value!(tokio::runtime::Handle: Send & Sync & Unpin); +assert_value!(tokio::runtime::Runtime: Send & Sync & Unpin); -async_assert_fn!(tokio::sync::mpsc::UnboundedReceiver::recv(_): Send & Sync); -async_assert_fn!(tokio::sync::mpsc::UnboundedReceiver>::recv(_): Send & Sync); -async_assert_fn!(tokio::sync::mpsc::UnboundedReceiver>::recv(_): !Send & !Sync); +assert_value!(tokio::time::Interval: Send & Sync & Unpin); +assert_value!(tokio::time::Instant: Send & Sync & Unpin); +assert_value!(tokio::time::Sleep: Send & Sync & !Unpin); +assert_value!(tokio::time::Timeout>: Send & Sync & !Unpin); +assert_value!(tokio::time::Timeout>: Send & !Sync & !Unpin); +assert_value!(tokio::time::Timeout>: !Send & !Sync & !Unpin); +assert_value!(tokio::time::error::Elapsed: Send & Sync & Unpin); +assert_value!(tokio::time::error::Error: Send & Sync & Unpin); +async_assert_fn!(tokio::time::advance(Duration): Send & Sync & !Unpin); +async_assert_fn!(tokio::time::sleep(Duration): Send & Sync & !Unpin); +async_assert_fn!(tokio::time::sleep_until(Instant): Send & Sync & !Unpin); +async_assert_fn!(tokio::time::timeout(Duration, BoxFutureSync<()>): Send & Sync & !Unpin); +async_assert_fn!(tokio::time::timeout(Duration, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::time::timeout(Duration, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::time::timeout_at(Instant, BoxFutureSync<()>): Send & Sync & !Unpin); +async_assert_fn!(tokio::time::timeout_at(Instant, BoxFutureSend<()>): Send & !Sync & !Unpin); +async_assert_fn!(tokio::time::timeout_at(Instant, BoxFuture<()>): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::time::Interval::tick(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::sync::watch::Receiver::changed(_): Send & Sync); -async_assert_fn!(tokio::sync::watch::Sender::closed(_): Send & Sync); -async_assert_fn!(tokio::sync::watch::Sender>::closed(_): !Send & !Sync); -async_assert_fn!(tokio::sync::watch::Sender>::closed(_): !Send & !Sync); +assert_value!(tokio::io::BufReader: Send & Sync & Unpin); +assert_value!(tokio::io::BufStream: Send & Sync & Unpin); +assert_value!(tokio::io::BufWriter: Send & Sync & Unpin); +assert_value!(tokio::io::DuplexStream: Send & Sync & Unpin); +assert_value!(tokio::io::Empty: Send & Sync & Unpin); +assert_value!(tokio::io::Interest: Send & Sync & Unpin); +assert_value!(tokio::io::Lines: Send & Sync & Unpin); +assert_value!(tokio::io::ReadBuf<'_>: Send & Sync & Unpin); +assert_value!(tokio::io::ReadHalf: Send & Sync & Unpin); +assert_value!(tokio::io::Ready: Send & Sync & Unpin); +assert_value!(tokio::io::Repeat: Send & Sync & Unpin); +assert_value!(tokio::io::Sink: Send & Sync & Unpin); +assert_value!(tokio::io::Split: Send & Sync & Unpin); +assert_value!(tokio::io::Stderr: Send & Sync & Unpin); +assert_value!(tokio::io::Stdin: Send & Sync & Unpin); +assert_value!(tokio::io::Stdout: Send & Sync & Unpin); +assert_value!(tokio::io::Take: Send & Sync & Unpin); +assert_value!(tokio::io::WriteHalf: Send & Sync & Unpin); +async_assert_fn!(tokio::io::copy(&mut TcpStream, &mut TcpStream): Send & Sync & !Unpin); +async_assert_fn!( + tokio::io::copy_bidirectional(&mut TcpStream, &mut TcpStream): Send & Sync & !Unpin +); +async_assert_fn!(tokio::io::copy_buf(&mut tokio::io::BufReader, &mut TcpStream): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::empty(): Send & Sync & Unpin); +async_assert_fn!(tokio::io::repeat(u8): Send & Sync & Unpin); +async_assert_fn!(tokio::io::sink(): Send & Sync & Unpin); +async_assert_fn!(tokio::io::split(TcpStream): Send & Sync & Unpin); +async_assert_fn!(tokio::io::stderr(): Send & Sync & Unpin); +async_assert_fn!(tokio::io::stdin(): Send & Sync & Unpin); +async_assert_fn!(tokio::io::stdout(): Send & Sync & Unpin); +async_assert_fn!(tokio::io::Split>::next_segment(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::Lines>::next_line(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncBufReadExt::read_until(&mut BoxAsyncRead, u8, &mut Vec): Send & Sync & !Unpin); +async_assert_fn!( + tokio::io::AsyncBufReadExt::read_line(&mut BoxAsyncRead, &mut String): Send & Sync & !Unpin +); +async_assert_fn!(tokio::io::AsyncReadExt::read(&mut BoxAsyncRead, &mut [u8]): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_buf(&mut BoxAsyncRead, &mut Vec): Send & Sync & !Unpin); +async_assert_fn!( + tokio::io::AsyncReadExt::read_exact(&mut BoxAsyncRead, &mut [u8]): Send & Sync & !Unpin +); +async_assert_fn!(tokio::io::AsyncReadExt::read_u8(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i8(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u16(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i16(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u32(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i32(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u64(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i64(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u128(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i128(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u16_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i16_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u32_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i32_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u64_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i64_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_u128_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_i128_le(&mut BoxAsyncRead): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncReadExt::read_to_end(&mut BoxAsyncRead, &mut Vec): Send & Sync & !Unpin); +async_assert_fn!( + tokio::io::AsyncReadExt::read_to_string(&mut BoxAsyncRead, &mut String): Send & Sync & !Unpin +); +async_assert_fn!(tokio::io::AsyncSeekExt::seek(&mut BoxAsyncSeek, SeekFrom): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncSeekExt::stream_position(&mut BoxAsyncSeek): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncWriteExt::write(&mut BoxAsyncWrite, &[u8]): Send & Sync & !Unpin); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_vectored(&mut BoxAsyncWrite, _): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_buf(&mut BoxAsyncWrite, &mut bytes::Bytes): Send + & Sync + & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_all_buf(&mut BoxAsyncWrite, &mut bytes::Bytes): Send + & Sync + & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_all(&mut BoxAsyncWrite, &[u8]): Send & Sync & !Unpin +); +async_assert_fn!(tokio::io::AsyncWriteExt::write_u8(&mut BoxAsyncWrite, u8): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncWriteExt::write_i8(&mut BoxAsyncWrite, i8): Send & Sync & !Unpin); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u16(&mut BoxAsyncWrite, u16): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i16(&mut BoxAsyncWrite, i16): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u32(&mut BoxAsyncWrite, u32): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i32(&mut BoxAsyncWrite, i32): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u64(&mut BoxAsyncWrite, u64): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i64(&mut BoxAsyncWrite, i64): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u128(&mut BoxAsyncWrite, u128): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i128(&mut BoxAsyncWrite, i128): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u16_le(&mut BoxAsyncWrite, u16): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i16_le(&mut BoxAsyncWrite, i16): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u32_le(&mut BoxAsyncWrite, u32): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i32_le(&mut BoxAsyncWrite, i32): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u64_le(&mut BoxAsyncWrite, u64): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i64_le(&mut BoxAsyncWrite, i64): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_u128_le(&mut BoxAsyncWrite, u128): Send & Sync & !Unpin +); +async_assert_fn!( + tokio::io::AsyncWriteExt::write_i128_le(&mut BoxAsyncWrite, i128): Send & Sync & !Unpin +); +async_assert_fn!(tokio::io::AsyncWriteExt::flush(&mut BoxAsyncWrite): Send & Sync & !Unpin); +async_assert_fn!(tokio::io::AsyncWriteExt::shutdown(&mut BoxAsyncWrite): Send & Sync & !Unpin); -async_assert_fn!(tokio::sync::OnceCell::get_or_init( - _, fn() -> Pin + Send + Sync>>): Send & Sync); -async_assert_fn!(tokio::sync::OnceCell::get_or_init( - _, fn() -> Pin + Send>>): Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell::get_or_init( - _, fn() -> Pin>>): !Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell>::get_or_init( - _, fn() -> Pin> + Send + Sync>>): !Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell>::get_or_init( - _, fn() -> Pin> + Send>>): !Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell>::get_or_init( - _, fn() -> Pin>>>): !Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell>::get_or_init( - _, fn() -> Pin> + Send + Sync>>): !Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell>::get_or_init( - _, fn() -> Pin> + Send>>): !Send & !Sync); -async_assert_fn!(tokio::sync::OnceCell>::get_or_init( - _, fn() -> Pin>>>): !Send & !Sync); -assert_value!(tokio::sync::OnceCell: Send & Sync); -assert_value!(tokio::sync::OnceCell>: Send & !Sync); -assert_value!(tokio::sync::OnceCell>: !Send & !Sync); +#[cfg(unix)] +mod unix_asyncfd { + use super::*; + use tokio::io::unix::*; -async_assert_fn!(tokio::task::LocalKey::scope(_, u32, BoxFutureSync<()>): Send & Sync); -async_assert_fn!(tokio::task::LocalKey::scope(_, u32, BoxFutureSend<()>): Send & !Sync); -async_assert_fn!(tokio::task::LocalKey::scope(_, u32, BoxFuture<()>): !Send & !Sync); -async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFutureSync<()>): Send & !Sync); -async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFutureSend<()>): Send & !Sync); -async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFuture<()>): !Send & !Sync); -async_assert_fn!(tokio::task::LocalKey>::scope(_, Rc, BoxFutureSync<()>): !Send & !Sync); -async_assert_fn!(tokio::task::LocalKey>::scope(_, Rc, BoxFutureSend<()>): !Send & !Sync); -async_assert_fn!(tokio::task::LocalKey>::scope(_, Rc, BoxFuture<()>): !Send & !Sync); -async_assert_fn!(tokio::task::LocalSet::run_until(_, BoxFutureSync<()>): !Send & !Sync); -assert_value!(tokio::task::LocalSet: !Send & !Sync); + struct ImplsFd { + _t: T, + } + impl std::os::unix::io::AsRawFd for ImplsFd { + fn as_raw_fd(&self) -> std::os::unix::io::RawFd { + unreachable!() + } + } -async_assert_fn!(tokio::time::advance(Duration): Send & Sync); -async_assert_fn!(tokio::time::sleep(Duration): Send & Sync); -async_assert_fn!(tokio::time::sleep_until(Instant): Send & Sync); -async_assert_fn!(tokio::time::timeout(Duration, BoxFutureSync<()>): Send & Sync); -async_assert_fn!(tokio::time::timeout(Duration, BoxFutureSend<()>): Send & !Sync); -async_assert_fn!(tokio::time::timeout(Duration, BoxFuture<()>): !Send & !Sync); -async_assert_fn!(tokio::time::timeout_at(Instant, BoxFutureSync<()>): Send & Sync); -async_assert_fn!(tokio::time::timeout_at(Instant, BoxFutureSend<()>): Send & !Sync); -async_assert_fn!(tokio::time::timeout_at(Instant, BoxFuture<()>): !Send & !Sync); -async_assert_fn!(tokio::time::Interval::tick(_): Send & Sync); - -assert_value!(tokio::time::Interval: Unpin); -async_assert_fn!(tokio::time::sleep(Duration): !Unpin); -async_assert_fn!(tokio::time::sleep_until(Instant): !Unpin); -async_assert_fn!(tokio::time::timeout(Duration, BoxFuture<()>): !Unpin); -async_assert_fn!(tokio::time::timeout_at(Instant, BoxFuture<()>): !Unpin); -async_assert_fn!(tokio::time::Interval::tick(_): !Unpin); -async_assert_fn!(tokio::io::AsyncBufReadExt::read_until(&mut BoxAsyncRead, u8, &mut Vec): !Unpin); -async_assert_fn!(tokio::io::AsyncBufReadExt::read_line(&mut BoxAsyncRead, &mut String): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read(&mut BoxAsyncRead, &mut [u8]): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_exact(&mut BoxAsyncRead, &mut [u8]): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u8(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i8(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u16(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i16(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u32(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i32(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u64(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i64(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u128(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i128(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u16_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i16_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u32_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i32_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u64_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i64_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_u128_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_i128_le(&mut BoxAsyncRead): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_to_end(&mut BoxAsyncRead, &mut Vec): !Unpin); -async_assert_fn!(tokio::io::AsyncReadExt::read_to_string(&mut BoxAsyncRead, &mut String): !Unpin); -async_assert_fn!(tokio::io::AsyncSeekExt::seek(&mut BoxAsyncSeek, SeekFrom): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write(&mut BoxAsyncWrite, &[u8]): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_all(&mut BoxAsyncWrite, &[u8]): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u8(&mut BoxAsyncWrite, u8): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i8(&mut BoxAsyncWrite, i8): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u16(&mut BoxAsyncWrite, u16): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i16(&mut BoxAsyncWrite, i16): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u32(&mut BoxAsyncWrite, u32): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i32(&mut BoxAsyncWrite, i32): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u64(&mut BoxAsyncWrite, u64): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i64(&mut BoxAsyncWrite, i64): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u128(&mut BoxAsyncWrite, u128): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i128(&mut BoxAsyncWrite, i128): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u16_le(&mut BoxAsyncWrite, u16): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i16_le(&mut BoxAsyncWrite, i16): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u32_le(&mut BoxAsyncWrite, u32): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i32_le(&mut BoxAsyncWrite, i32): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u64_le(&mut BoxAsyncWrite, u64): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i64_le(&mut BoxAsyncWrite, i64): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_u128_le(&mut BoxAsyncWrite, u128): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::write_i128_le(&mut BoxAsyncWrite, i128): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::flush(&mut BoxAsyncWrite): !Unpin); -async_assert_fn!(tokio::io::AsyncWriteExt::shutdown(&mut BoxAsyncWrite): !Unpin); + assert_value!(AsyncFd>: Send & Sync & Unpin); + assert_value!(AsyncFd>: Send & !Sync & Unpin); + assert_value!(AsyncFd>: !Send & !Sync & Unpin); + assert_value!(AsyncFdReadyGuard<'_, ImplsFd>: Send & Sync & Unpin); + assert_value!(AsyncFdReadyGuard<'_, ImplsFd>: !Send & !Sync & Unpin); + assert_value!(AsyncFdReadyGuard<'_, ImplsFd>: !Send & !Sync & Unpin); + assert_value!(AsyncFdReadyMutGuard<'_, ImplsFd>: Send & Sync & Unpin); + assert_value!(AsyncFdReadyMutGuard<'_, ImplsFd>: Send & !Sync & Unpin); + assert_value!(AsyncFdReadyMutGuard<'_, ImplsFd>: !Send & !Sync & Unpin); + assert_value!(TryIoError: Send & Sync & Unpin); + async_assert_fn!(AsyncFd>::readable(_): Send & Sync & !Unpin); + async_assert_fn!(AsyncFd>::readable_mut(_): Send & Sync & !Unpin); + async_assert_fn!(AsyncFd>::writable(_): Send & Sync & !Unpin); + async_assert_fn!(AsyncFd>::writable_mut(_): Send & Sync & !Unpin); + async_assert_fn!(AsyncFd>::readable(_): !Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::readable_mut(_): Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::writable(_): !Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::writable_mut(_): Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::readable(_): !Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::readable_mut(_): !Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::writable(_): !Send & !Sync & !Unpin); + async_assert_fn!(AsyncFd>::writable_mut(_): !Send & !Sync & !Unpin); +} From 8f10d816131868030996b5079fbc632750e69332 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Tue, 13 Jul 2021 14:23:25 +0200 Subject: [PATCH 21/35] net: documentation updates (#3944) --- tokio/src/net/tcp/stream.rs | 20 +++++++-------- tokio/src/net/udp.rs | 20 +++++++-------- tokio/src/net/unix/datagram/socket.rs | 20 +++++++-------- tokio/src/net/unix/stream.rs | 20 +++++++-------- tokio/src/net/windows/named_pipe.rs | 36 +++++++++++++-------------- 5 files changed, 58 insertions(+), 58 deletions(-) diff --git a/tokio/src/net/tcp/stream.rs b/tokio/src/net/tcp/stream.rs index daf3dccde..34ac6eeb8 100644 --- a/tokio/src/net/tcp/stream.rs +++ b/tokio/src/net/tcp/stream.rs @@ -936,14 +936,14 @@ impl TcpStream { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(bufs)) } - /// Try to perform IO operation from the socket using a user-provided IO operation. + /// Try to read or write from the socket using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The - /// closure should attempt to perform IO operation from the socket by manually calling the - /// appropriate syscall. If the operation fails because the socket is not - /// actually ready, then the closure should return a `WouldBlock` error and - /// the readiness flag is cleared. The return value of the closure is - /// then returned by `try_io`. + /// If the socket is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the socket by manually + /// calling the appropriate syscall. If the operation fails because the + /// socket is not actually ready, then the closure should return a + /// `WouldBlock` error and the readiness flag is cleared. The return value + /// of the closure is then returned by `try_io`. /// /// If the socket is not ready, then the closure is not called /// and a `WouldBlock` error is returned. @@ -954,9 +954,9 @@ impl TcpStream { /// incorrectly clear the readiness flag, which can cause the socket to /// behave incorrectly. /// - /// The closure should not perform the read operation using any of the - /// methods defined on the Tokio `TcpStream` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `TcpStream` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// diff --git a/tokio/src/net/udp.rs b/tokio/src/net/udp.rs index a47f071d7..75cc6f3de 100644 --- a/tokio/src/net/udp.rs +++ b/tokio/src/net/udp.rs @@ -1170,14 +1170,14 @@ impl UdpSocket { .try_io(Interest::READABLE, || self.io.recv_from(buf)) } - /// Try to perform IO operation from the socket using a user-provided IO operation. + /// Try to read or write from the socket using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The - /// closure should attempt to perform IO operation from the socket by manually calling the - /// appropriate syscall. If the operation fails because the socket is not - /// actually ready, then the closure should return a `WouldBlock` error and - /// the readiness flag is cleared. The return value of the closure is - /// then returned by `try_io`. + /// If the socket is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the socket by manually + /// calling the appropriate syscall. If the operation fails because the + /// socket is not actually ready, then the closure should return a + /// `WouldBlock` error and the readiness flag is cleared. The return value + /// of the closure is then returned by `try_io`. /// /// If the socket is not ready, then the closure is not called /// and a `WouldBlock` error is returned. @@ -1188,9 +1188,9 @@ impl UdpSocket { /// incorrectly clear the readiness flag, which can cause the socket to /// behave incorrectly. /// - /// The closure should not perform the read operation using any of the - /// methods defined on the Tokio `UdpSocket` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `UdpSocket` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// diff --git a/tokio/src/net/unix/datagram/socket.rs b/tokio/src/net/unix/datagram/socket.rs index c7c6568f2..7874b8a13 100644 --- a/tokio/src/net/unix/datagram/socket.rs +++ b/tokio/src/net/unix/datagram/socket.rs @@ -1143,14 +1143,14 @@ impl UnixDatagram { Ok((n, SocketAddr(addr))) } - /// Try to perform IO operation from the socket using a user-provided IO operation. + /// Try to read or write from the socket using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The - /// closure should attempt to perform IO operation from the socket by manually calling the - /// appropriate syscall. If the operation fails because the socket is not - /// actually ready, then the closure should return a `WouldBlock` error and - /// the readiness flag is cleared. The return value of the closure is - /// then returned by `try_io`. + /// If the socket is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the socket by manually + /// calling the appropriate syscall. If the operation fails because the + /// socket is not actually ready, then the closure should return a + /// `WouldBlock` error and the readiness flag is cleared. The return value + /// of the closure is then returned by `try_io`. /// /// If the socket is not ready, then the closure is not called /// and a `WouldBlock` error is returned. @@ -1161,9 +1161,9 @@ impl UnixDatagram { /// incorrectly clear the readiness flag, which can cause the socket to /// behave incorrectly. /// - /// The closure should not perform the read operation using any of the - /// methods defined on the Tokio `UnixDatagram` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `UnixDatagram` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// diff --git a/tokio/src/net/unix/stream.rs b/tokio/src/net/unix/stream.rs index 39f6ee258..5837f3673 100644 --- a/tokio/src/net/unix/stream.rs +++ b/tokio/src/net/unix/stream.rs @@ -653,14 +653,14 @@ impl UnixStream { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Try to perform IO operation from the socket using a user-provided IO operation. + /// Try to read or write from the socket using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The - /// closure should attempt to perform IO operation from the socket by manually calling the - /// appropriate syscall. If the operation fails because the socket is not - /// actually ready, then the closure should return a `WouldBlock` error and - /// the readiness flag is cleared. The return value of the closure is - /// then returned by `try_io`. + /// If the socket is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the socket by manually + /// calling the appropriate syscall. If the operation fails because the + /// socket is not actually ready, then the closure should return a + /// `WouldBlock` error and the readiness flag is cleared. The return value + /// of the closure is then returned by `try_io`. /// /// If the socket is not ready, then the closure is not called /// and a `WouldBlock` error is returned. @@ -671,9 +671,9 @@ impl UnixStream { /// incorrectly clear the readiness flag, which can cause the socket to /// behave incorrectly. /// - /// The closure should not perform the read operation using any of the - /// methods defined on the Tokio `UnixStream` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `UnixStream` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// diff --git a/tokio/src/net/windows/named_pipe.rs b/tokio/src/net/windows/named_pipe.rs index 4e7458b58..de6ab58cf 100644 --- a/tokio/src/net/windows/named_pipe.rs +++ b/tokio/src/net/windows/named_pipe.rs @@ -724,14 +724,14 @@ impl NamedPipeServer { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Try to perform IO operation from the socket using a user-provided IO operation. + /// Try to read or write from the socket using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The - /// closure should attempt to perform IO operation from the socket by manually calling the - /// appropriate syscall. If the operation fails because the socket is not - /// actually ready, then the closure should return a `WouldBlock` error and - /// the readiness flag is cleared. The return value of the closure is - /// then returned by `try_io`. + /// If the socket is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the socket by manually + /// calling the appropriate syscall. If the operation fails because the + /// socket is not actually ready, then the closure should return a + /// `WouldBlock` error and the readiness flag is cleared. The return value + /// of the closure is then returned by `try_io`. /// /// If the socket is not ready, then the closure is not called /// and a `WouldBlock` error is returned. @@ -742,7 +742,7 @@ impl NamedPipeServer { /// incorrectly clear the readiness flag, which can cause the socket to /// behave incorrectly. /// - /// The closure should not perform the read operation using any of the + /// The closure should not perform the IO operation using any of the /// methods defined on the Tokio `NamedPipeServer` type, as this will mess with /// the readiness flag and can cause the socket to behave incorrectly. /// @@ -1379,14 +1379,14 @@ impl NamedPipeClient { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Try to perform IO operation from the socket using a user-provided IO operation. + /// Try to read or write from the socket using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The - /// closure should attempt to perform IO operation from the socket by manually calling the - /// appropriate syscall. If the operation fails because the socket is not - /// actually ready, then the closure should return a `WouldBlock` error and - /// the readiness flag is cleared. The return value of the closure is - /// then returned by `try_io`. + /// If the socket is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the socket by manually + /// calling the appropriate syscall. If the operation fails because the + /// socket is not actually ready, then the closure should return a + /// `WouldBlock` error and the readiness flag is cleared. The return value + /// of the closure is then returned by `try_io`. /// /// If the socket is not ready, then the closure is not called /// and a `WouldBlock` error is returned. @@ -1397,9 +1397,9 @@ impl NamedPipeClient { /// incorrectly clear the readiness flag, which can cause the socket to /// behave incorrectly. /// - /// The closure should not perform the read operation using any of the - /// methods defined on the Tokio `NamedPipeClient` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `NamedPipeClient` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// From aef2d64b0a519ff6726f8c139ee1d3e6b1959b0b Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 16 Jul 2021 09:50:03 +0200 Subject: [PATCH 22/35] task: remove mutex in JoinError (#3959) --- tokio/src/runtime/task/error.rs | 9 +++++---- tokio/src/util/mod.rs | 3 +++ tokio/src/util/sync_wrapper.rs | 26 ++++++++++++++++++++++++++ tokio/tests/async_send_sync.rs | 1 + 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 tokio/src/util/sync_wrapper.rs diff --git a/tokio/src/runtime/task/error.rs b/tokio/src/runtime/task/error.rs index 177fe65e9..17fb09390 100644 --- a/tokio/src/runtime/task/error.rs +++ b/tokio/src/runtime/task/error.rs @@ -1,7 +1,8 @@ use std::any::Any; use std::fmt; use std::io; -use std::sync::Mutex; + +use crate::util::SyncWrapper; cfg_rt! { /// Task failed to execute to completion. @@ -12,7 +13,7 @@ cfg_rt! { enum Repr { Cancelled, - Panic(Mutex>), + Panic(SyncWrapper>), } impl JoinError { @@ -24,7 +25,7 @@ impl JoinError { pub(crate) fn panic(err: Box) -> JoinError { JoinError { - repr: Repr::Panic(Mutex::new(err)), + repr: Repr::Panic(SyncWrapper::new(err)), } } @@ -106,7 +107,7 @@ impl JoinError { /// ``` pub fn try_into_panic(self) -> Result, JoinError> { match self.repr { - Repr::Panic(p) => Ok(p.into_inner().expect("Extracting panic from mutex")), + Repr::Panic(p) => Ok(p.into_inner()), _ => Err(self), } } diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index b267125b1..bcc73ed68 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -21,6 +21,9 @@ cfg_rt! { mod wake; pub(crate) use wake::WakerRef; pub(crate) use wake::{waker_ref, Wake}; + + mod sync_wrapper; + pub(crate) use sync_wrapper::SyncWrapper; } cfg_rt_multi_thread! { diff --git a/tokio/src/util/sync_wrapper.rs b/tokio/src/util/sync_wrapper.rs new file mode 100644 index 000000000..5ffc8f96b --- /dev/null +++ b/tokio/src/util/sync_wrapper.rs @@ -0,0 +1,26 @@ +//! This module contains a type that can make `Send + !Sync` types `Sync` by +//! disallowing all immutable access to the value. +//! +//! A similar primitive is provided in the `sync_wrapper` crate. + +pub(crate) struct SyncWrapper { + value: T, +} + +// safety: The SyncWrapper being send allows you to send the inner value across +// thread boundaries. +unsafe impl Send for SyncWrapper {} + +// safety: An immutable reference to a SyncWrapper is useless, so moving such an +// immutable reference across threads is safe. +unsafe impl Sync for SyncWrapper {} + +impl SyncWrapper { + pub(crate) fn new(value: T) -> Self { + Self { value } + } + + pub(crate) fn into_inner(self) -> T { + self.value + } +} diff --git a/tokio/tests/async_send_sync.rs b/tokio/tests/async_send_sync.rs index 12d132393..97118ce66 100644 --- a/tokio/tests/async_send_sync.rs +++ b/tokio/tests/async_send_sync.rs @@ -452,6 +452,7 @@ assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin); assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); assert_value!(tokio::task::JoinHandle: !Send & !Sync & Unpin); +assert_value!(tokio::task::JoinError: Send & Sync & Unpin); assert_value!(tokio::runtime::Builder: Send & Sync & Unpin); assert_value!(tokio::runtime::EnterGuard<'_>: Send & Sync & Unpin); From 7c4183a45d3548840104df87d9c8b81c1b19c006 Mon Sep 17 00:00:00 2001 From: Diggory Blake Date: Sun, 18 Jul 2021 21:46:35 +0100 Subject: [PATCH 23/35] runtime: drop future when polling cancelled future (#3965) --- tokio/src/runtime/task/harness.rs | 2 +- tokio/tests/task_abort.rs | 95 +++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/harness.rs b/tokio/src/runtime/task/harness.rs index 7f1c4e4cb..9f0b10711 100644 --- a/tokio/src/runtime/task/harness.rs +++ b/tokio/src/runtime/task/harness.rs @@ -420,7 +420,7 @@ fn poll_future( cx: Context<'_>, ) -> PollFuture { if snapshot.is_cancelled() { - PollFuture::Complete(Err(JoinError::cancelled()), snapshot.is_join_interested()) + PollFuture::Complete(Err(cancel_task(core)), snapshot.is_join_interested()) } else { let res = panic::catch_unwind(panic::AssertUnwindSafe(|| { struct Guard<'a, T: Future> { diff --git a/tokio/tests/task_abort.rs b/tokio/tests/task_abort.rs index c524dc287..8f621683f 100644 --- a/tokio/tests/task_abort.rs +++ b/tokio/tests/task_abort.rs @@ -1,6 +1,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "full")] +use std::sync::Arc; use std::thread::sleep; use std::time::Duration; @@ -138,3 +139,97 @@ fn remote_abort_local_set_3929() { rt.block_on(local); jh2.join().unwrap(); } + +/// Checks that a suspended task can be aborted even if the `JoinHandle` is immediately dropped. +/// issue #3964: . +#[test] +fn test_abort_wakes_task_3964() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + rt.block_on(async move { + let notify_dropped = Arc::new(()); + let weak_notify_dropped = Arc::downgrade(¬ify_dropped); + + let handle = tokio::spawn(async move { + // Make sure the Arc is moved into the task + let _notify_dropped = notify_dropped; + println!("task started"); + tokio::time::sleep(std::time::Duration::new(100, 0)).await + }); + + // wait for task to sleep. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + handle.abort(); + drop(handle); + + // wait for task to abort. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // Check that the Arc has been dropped. + assert!(weak_notify_dropped.upgrade().is_none()); + }); +} + +struct PanicOnDrop; + +impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("Well what did you expect would happen..."); + } +} + +/// Checks that aborting a task whose destructor panics does not allow the +/// panic to escape the task. +#[test] +fn test_abort_task_that_panics_on_drop_contained() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + rt.block_on(async move { + let handle = tokio::spawn(async move { + // Make sure the Arc is moved into the task + let _panic_dropped = PanicOnDrop; + println!("task started"); + tokio::time::sleep(std::time::Duration::new(100, 0)).await + }); + + // wait for task to sleep. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + handle.abort(); + drop(handle); + + // wait for task to abort. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + }); +} + +/// Checks that aborting a task whose destructor panics has the expected result. +#[test] +fn test_abort_task_that_panics_on_drop_returned() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + rt.block_on(async move { + let handle = tokio::spawn(async move { + // Make sure the Arc is moved into the task + let _panic_dropped = PanicOnDrop; + println!("task started"); + tokio::time::sleep(std::time::Duration::new(100, 0)).await + }); + + // wait for task to sleep. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + handle.abort(); + assert!(handle.await.unwrap_err().is_panic()); + }); +} From 549e89e9cd2073ffa70f1bd12022c5543343be78 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Tue, 20 Jul 2021 12:12:30 +0200 Subject: [PATCH 24/35] chore: use the loom mutex wrapper everywhere (#3958) --- tokio/src/loom/std/atomic_u64.rs | 10 +- tokio/src/runtime/shell.rs | 132 ------------------------ tokio/src/runtime/tests/loom_oneshot.rs | 7 +- tokio/src/sync/barrier.rs | 5 +- tokio/src/task/local.rs | 9 +- tokio/src/time/clock.rs | 12 +-- 6 files changed, 20 insertions(+), 155 deletions(-) delete mode 100644 tokio/src/runtime/shell.rs diff --git a/tokio/src/loom/std/atomic_u64.rs b/tokio/src/loom/std/atomic_u64.rs index a86a195b1..7eb457a24 100644 --- a/tokio/src/loom/std/atomic_u64.rs +++ b/tokio/src/loom/std/atomic_u64.rs @@ -15,8 +15,8 @@ mod imp { #[cfg(any(target_arch = "arm", target_arch = "mips", target_arch = "powerpc"))] mod imp { + use crate::loom::sync::Mutex; use std::sync::atomic::Ordering; - use std::sync::Mutex; #[derive(Debug)] pub(crate) struct AtomicU64 { @@ -31,15 +31,15 @@ mod imp { } pub(crate) fn load(&self, _: Ordering) -> u64 { - *self.inner.lock().unwrap() + *self.inner.lock() } pub(crate) fn store(&self, val: u64, _: Ordering) { - *self.inner.lock().unwrap() = val; + *self.inner.lock() = val; } pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 { - let mut lock = self.inner.lock().unwrap(); + let mut lock = self.inner.lock(); let prev = *lock; *lock = prev | val; prev @@ -52,7 +52,7 @@ mod imp { _success: Ordering, _failure: Ordering, ) -> Result { - let mut lock = self.inner.lock().unwrap(); + let mut lock = self.inner.lock(); if *lock == current { *lock = new; diff --git a/tokio/src/runtime/shell.rs b/tokio/src/runtime/shell.rs deleted file mode 100644 index 486d4fa5b..000000000 --- a/tokio/src/runtime/shell.rs +++ /dev/null @@ -1,132 +0,0 @@ -#![allow(clippy::redundant_clone)] - -use crate::future::poll_fn; -use crate::park::{Park, Unpark}; -use crate::runtime::driver::Driver; -use crate::sync::Notify; -use crate::util::{waker_ref, Wake}; - -use std::sync::{Arc, Mutex}; -use std::task::Context; -use std::task::Poll::{Pending, Ready}; -use std::{future::Future, sync::PoisonError}; - -#[derive(Debug)] -pub(super) struct Shell { - driver: Mutex>, - - notify: Notify, - - /// TODO: don't store this - unpark: Arc, -} - -#[derive(Debug)] -struct Handle(::Unpark); - -impl Shell { - pub(super) fn new(driver: Driver) -> Shell { - let unpark = Arc::new(Handle(driver.unpark())); - - Shell { - driver: Mutex::new(Some(driver)), - notify: Notify::new(), - unpark, - } - } - - pub(super) fn block_on(&self, f: F) -> F::Output - where - F: Future, - { - let mut enter = crate::runtime::enter(true); - - pin!(f); - - loop { - if let Some(driver) = &mut self.take_driver() { - return driver.block_on(f); - } else { - let notified = self.notify.notified(); - pin!(notified); - - if let Some(out) = enter - .block_on(poll_fn(|cx| { - if notified.as_mut().poll(cx).is_ready() { - return Ready(None); - } - - if let Ready(out) = f.as_mut().poll(cx) { - return Ready(Some(out)); - } - - Pending - })) - .expect("Failed to `Enter::block_on`") - { - return out; - } - } - } - } - - fn take_driver(&self) -> Option> { - let mut lock = self.driver.lock().unwrap(); - let driver = lock.take()?; - - Some(DriverGuard { - inner: Some(driver), - shell: &self, - }) - } -} - -impl Wake for Handle { - /// Wake by value - fn wake(self: Arc) { - Wake::wake_by_ref(&self); - } - - /// Wake by reference - fn wake_by_ref(arc_self: &Arc) { - arc_self.0.unpark(); - } -} - -struct DriverGuard<'a> { - inner: Option, - shell: &'a Shell, -} - -impl DriverGuard<'_> { - fn block_on(&mut self, f: F) -> F::Output { - let driver = self.inner.as_mut().unwrap(); - - pin!(f); - - let waker = waker_ref(&self.shell.unpark); - let mut cx = Context::from_waker(&waker); - - loop { - if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) { - return v; - } - - driver.park().unwrap(); - } - } -} - -impl Drop for DriverGuard<'_> { - fn drop(&mut self) { - if let Some(inner) = self.inner.take() { - self.shell - .driver - .lock() - .unwrap_or_else(PoisonError::into_inner) - .replace(inner); - - self.shell.notify.notify_one(); - } - } -} diff --git a/tokio/src/runtime/tests/loom_oneshot.rs b/tokio/src/runtime/tests/loom_oneshot.rs index c126fe479..87eb63864 100644 --- a/tokio/src/runtime/tests/loom_oneshot.rs +++ b/tokio/src/runtime/tests/loom_oneshot.rs @@ -1,7 +1,6 @@ +use crate::loom::sync::{Arc, Mutex}; use loom::sync::Notify; -use std::sync::{Arc, Mutex}; - pub(crate) fn channel() -> (Sender, Receiver) { let inner = Arc::new(Inner { notify: Notify::new(), @@ -31,7 +30,7 @@ struct Inner { impl Sender { pub(crate) fn send(self, value: T) { - *self.inner.value.lock().unwrap() = Some(value); + *self.inner.value.lock() = Some(value); self.inner.notify.notify(); } } @@ -39,7 +38,7 @@ impl Sender { impl Receiver { pub(crate) fn recv(self) -> T { loop { - if let Some(v) = self.inner.value.lock().unwrap().take() { + if let Some(v) = self.inner.value.lock().take() { return v; } diff --git a/tokio/src/sync/barrier.rs b/tokio/src/sync/barrier.rs index e3c95f6a6..0e39dac8b 100644 --- a/tokio/src/sync/barrier.rs +++ b/tokio/src/sync/barrier.rs @@ -1,7 +1,6 @@ +use crate::loom::sync::Mutex; use crate::sync::watch; -use std::sync::Mutex; - /// A barrier enables multiple tasks to synchronize the beginning of some computation. /// /// ``` @@ -94,7 +93,7 @@ impl Barrier { // NOTE: the extra scope here is so that the compiler doesn't think `state` is held across // a yield point, and thus marks the returned future as !Send. let generation = { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock(); let generation = state.generation; state.arrived += 1; if state.arrived == self.n { diff --git a/tokio/src/task/local.rs b/tokio/src/task/local.rs index 01af63fc9..7404cc2c1 100644 --- a/tokio/src/task/local.rs +++ b/tokio/src/task/local.rs @@ -1,4 +1,5 @@ //! Runs `!Send` futures on the current thread. +use crate::loom::sync::{Arc, Mutex}; use crate::runtime::task::{self, JoinHandle, OwnedTasks, Task}; use crate::sync::AtomicWaker; @@ -8,7 +9,6 @@ use std::fmt; use std::future::Future; use std::marker::PhantomData; use std::pin::Pin; -use std::sync::{Arc, Mutex}; use std::task::Poll; use pin_project_lite::pin_project; @@ -537,7 +537,6 @@ impl LocalSet { .shared .queue .lock() - .unwrap() .pop_front() .or_else(|| self.context.tasks.borrow_mut().queue.pop_front()) } else { @@ -546,7 +545,7 @@ impl LocalSet { .borrow_mut() .queue .pop_front() - .or_else(|| self.context.shared.queue.lock().unwrap().pop_front()) + .or_else(|| self.context.shared.queue.lock().pop_front()) } } @@ -610,7 +609,7 @@ impl Drop for LocalSet { task.shutdown(); } - for task in self.context.shared.queue.lock().unwrap().drain(..) { + for task in self.context.shared.queue.lock().drain(..) { task.shutdown(); } @@ -660,7 +659,7 @@ impl Shared { cx.tasks.borrow_mut().queue.push_back(task); } _ => { - self.queue.lock().unwrap().push_back(task); + self.queue.lock().push_back(task); self.waker.wake(); } }); diff --git a/tokio/src/time/clock.rs b/tokio/src/time/clock.rs index b9ec5c5aa..a44d75f3c 100644 --- a/tokio/src/time/clock.rs +++ b/tokio/src/time/clock.rs @@ -29,7 +29,7 @@ cfg_not_test_util! { cfg_test_util! { use crate::time::{Duration, Instant}; - use std::sync::{Arc, Mutex}; + use crate::loom::sync::{Arc, Mutex}; cfg_rt! { fn clock() -> Option { @@ -102,7 +102,7 @@ cfg_test_util! { /// runtime. pub fn resume() { let clock = clock().expect("time cannot be frozen from outside the Tokio runtime"); - let mut inner = clock.inner.lock().unwrap(); + let mut inner = clock.inner.lock(); if inner.unfrozen.is_some() { panic!("time is not frozen"); @@ -164,7 +164,7 @@ cfg_test_util! { } pub(crate) fn pause(&self) { - let mut inner = self.inner.lock().unwrap(); + let mut inner = self.inner.lock(); if !inner.enable_pausing { drop(inner); // avoid poisoning the lock @@ -178,12 +178,12 @@ cfg_test_util! { } pub(crate) fn is_paused(&self) -> bool { - let inner = self.inner.lock().unwrap(); + let inner = self.inner.lock(); inner.unfrozen.is_none() } pub(crate) fn advance(&self, duration: Duration) { - let mut inner = self.inner.lock().unwrap(); + let mut inner = self.inner.lock(); if inner.unfrozen.is_some() { panic!("time is not frozen"); @@ -193,7 +193,7 @@ cfg_test_util! { } pub(crate) fn now(&self) -> Instant { - let inner = self.inner.lock().unwrap(); + let inner = self.inner.lock(); let mut ret = inner.base; From 252004811f39c092adb04fdb177738b57c46cbe4 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Tue, 20 Jul 2021 14:28:31 +0100 Subject: [PATCH 25/35] sync: add getter for the mutex from a guard (#3928) --- tokio/src/sync/mutex.rs | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tokio/src/sync/mutex.rs b/tokio/src/sync/mutex.rs index 8ae824770..6acd28b62 100644 --- a/tokio/src/sync/mutex.rs +++ b/tokio/src/sync/mutex.rs @@ -573,6 +573,32 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { marker: marker::PhantomData, }) } + + /// Returns a reference to the original `Mutex`. + /// + /// ``` + /// use tokio::sync::{Mutex, MutexGuard}; + /// + /// async fn unlock_and_relock<'l>(guard: MutexGuard<'l, u32>) -> MutexGuard<'l, u32> { + /// println!("1. contains: {:?}", *guard); + /// let mutex = MutexGuard::mutex(&guard); + /// drop(guard); + /// let guard = mutex.lock().await; + /// println!("2. contains: {:?}", *guard); + /// guard + /// } + /// # + /// # #[tokio::main] + /// # async fn main() { + /// # let mutex = Mutex::new(0u32); + /// # let guard = mutex.lock().await; + /// # unlock_and_relock(guard).await; + /// # } + /// ``` + #[inline] + pub fn mutex(this: &Self) -> &'a Mutex { + this.lock + } } impl Drop for MutexGuard<'_, T> { @@ -608,6 +634,35 @@ impl fmt::Display for MutexGuard<'_, T> { // === impl OwnedMutexGuard === +impl OwnedMutexGuard { + /// Returns a reference to the original `Arc`. + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::sync::{Mutex, OwnedMutexGuard}; + /// + /// async fn unlock_and_relock(guard: OwnedMutexGuard) -> OwnedMutexGuard { + /// println!("1. contains: {:?}", *guard); + /// let mutex: Arc> = OwnedMutexGuard::mutex(&guard).clone(); + /// drop(guard); + /// let guard = mutex.lock_owned().await; + /// println!("2. contains: {:?}", *guard); + /// guard + /// } + /// # + /// # #[tokio::main] + /// # async fn main() { + /// # let mutex = Arc::new(Mutex::new(0u32)); + /// # let guard = mutex.lock_owned().await; + /// # unlock_and_relock(guard).await; + /// # } + /// ``` + #[inline] + pub fn mutex(this: &Self) -> &Arc> { + &this.lock + } +} + impl Drop for OwnedMutexGuard { fn drop(&mut self) { self.lock.s.release(1) From 5ae2855eaf29fc78687c6d17b0be35f8b4934788 Mon Sep 17 00:00:00 2001 From: Blas Rodriguez Irizar Date: Tue, 20 Jul 2021 15:35:18 +0200 Subject: [PATCH 26/35] io: fix docs referring to clear_{read,write}_ready (#3957) --- tokio/src/io/poll_evented.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tokio/src/io/poll_evented.rs b/tokio/src/io/poll_evented.rs index a31e6db7b..987257480 100644 --- a/tokio/src/io/poll_evented.rs +++ b/tokio/src/io/poll_evented.rs @@ -40,9 +40,8 @@ cfg_io_driver! { /// [`poll_read_ready`] again will also indicate read readiness. /// /// When the operation is attempted and is unable to succeed due to the I/O - /// resource not being ready, the caller must call `clear_read_ready` or - /// `clear_write_ready`. This clears the readiness state until a new - /// readiness event is received. + /// resource not being ready, the caller must call `clear_readiness`. + /// This clears the readiness state until a new readiness event is received. /// /// This allows the caller to implement additional functions. For example, /// [`TcpListener`] implements poll_accept by using [`poll_read_ready`] and From 2087f3e0ebb08d633d59c5f964b3901e68b3c038 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Tue, 20 Jul 2021 16:43:34 +0200 Subject: [PATCH 27/35] runtime: rework binding of new tasks (#3955) --- tokio/src/runtime/basic_scheduler.rs | 28 ++-- tokio/src/runtime/blocking/mod.rs | 4 +- tokio/src/runtime/blocking/schedule.rs | 5 - tokio/src/runtime/handle.rs | 4 +- tokio/src/runtime/queue.rs | 9 +- tokio/src/runtime/task/core.rs | 32 +---- tokio/src/runtime/task/harness.rs | 21 +-- tokio/src/runtime/task/inject.rs | 9 +- tokio/src/runtime/task/list.rs | 124 ++++++++++++++-- tokio/src/runtime/task/mod.rs | 65 ++++----- tokio/src/runtime/task/raw.rs | 4 +- tokio/src/runtime/task/state.rs | 29 ++-- tokio/src/runtime/tests/loom_queue.rs | 36 ++--- tokio/src/runtime/tests/mod.rs | 33 +++-- tokio/src/runtime/tests/queue.rs | 15 +- tokio/src/runtime/tests/task.rs | 179 ++++++++++++++++++++++-- tokio/src/runtime/thread_pool/mod.rs | 14 +- tokio/src/runtime/thread_pool/worker.rs | 124 +++++++++++----- tokio/src/task/local.rs | 46 +++--- tokio/tests/support/mock_file.rs | 2 +- 20 files changed, 508 insertions(+), 275 deletions(-) diff --git a/tokio/src/runtime/basic_scheduler.rs b/tokio/src/runtime/basic_scheduler.rs index 9efe3844a..09a3d7dd7 100644 --- a/tokio/src/runtime/basic_scheduler.rs +++ b/tokio/src/runtime/basic_scheduler.rs @@ -311,6 +311,9 @@ impl Drop for BasicScheduler

{ }; enter(&mut inner, |scheduler, context| { + // By closing the OwnedTasks, no new tasks can be spawned on it. + context.shared.owned.close(); + // Drain the OwnedTasks collection. while let Some(task) = context.shared.owned.pop_back() { task.shutdown(); } @@ -354,14 +357,18 @@ impl fmt::Debug for BasicScheduler

{ // ===== impl Spawner ===== impl Spawner { - /// Spawns a future onto the thread pool + /// Spawns a future onto the basic scheduler pub(crate) fn spawn(&self, future: F) -> JoinHandle where F: crate::future::Future + Send + 'static, F::Output: Send + 'static, { - let (task, handle) = task::joinable(future); - self.shared.schedule(task); + let (handle, notified) = self.shared.owned.bind(future, self.shared.clone()); + + if let Some(notified) = notified { + self.shared.schedule(notified); + } + handle } @@ -392,14 +399,6 @@ impl fmt::Debug for Spawner { // ===== impl Shared ===== impl Schedule for Arc { - fn bind(task: Task) -> Arc { - CURRENT.with(|maybe_cx| { - let cx = maybe_cx.expect("scheduler context missing"); - cx.shared.owned.push_front(task); - cx.shared.clone() - }) - } - fn release(&self, task: &Task) -> Option> { // SAFETY: Inserted into the list in bind above. unsafe { self.owned.remove(task) } @@ -411,16 +410,13 @@ impl Schedule for Arc { cx.tasks.borrow_mut().queue.push_back(task); } _ => { + // If the queue is None, then the runtime has shut down. We + // don't need to do anything with the notification in that case. let mut guard = self.queue.lock(); if let Some(queue) = guard.as_mut() { queue.push_back(RemoteMsg::Schedule(task)); drop(guard); self.unpark.unpark(); - } else { - // The runtime has shut down. We drop the new task - // immediately. - drop(guard); - task.shutdown(); } } }); diff --git a/tokio/src/runtime/blocking/mod.rs b/tokio/src/runtime/blocking/mod.rs index fece3c279..670ec3a4b 100644 --- a/tokio/src/runtime/blocking/mod.rs +++ b/tokio/src/runtime/blocking/mod.rs @@ -8,7 +8,9 @@ pub(crate) use pool::{spawn_blocking, BlockingPool, Spawner}; mod schedule; mod shutdown; -pub(crate) mod task; +mod task; +pub(crate) use schedule::NoopSchedule; +pub(crate) use task::BlockingTask; use crate::runtime::Builder; diff --git a/tokio/src/runtime/blocking/schedule.rs b/tokio/src/runtime/blocking/schedule.rs index 4e044ab29..54252241d 100644 --- a/tokio/src/runtime/blocking/schedule.rs +++ b/tokio/src/runtime/blocking/schedule.rs @@ -9,11 +9,6 @@ use crate::runtime::task::{self, Task}; pub(crate) struct NoopSchedule; impl task::Schedule for NoopSchedule { - fn bind(_task: Task) -> NoopSchedule { - // Do nothing w/ the task - NoopSchedule - } - fn release(&self, _task: &Task) -> Option> { None } diff --git a/tokio/src/runtime/handle.rs b/tokio/src/runtime/handle.rs index 7dff91448..0bfed988a 100644 --- a/tokio/src/runtime/handle.rs +++ b/tokio/src/runtime/handle.rs @@ -1,4 +1,4 @@ -use crate::runtime::blocking::task::BlockingTask; +use crate::runtime::blocking::{BlockingTask, NoopSchedule}; use crate::runtime::task::{self, JoinHandle}; use crate::runtime::{blocking, context, driver, Spawner}; use crate::util::error::CONTEXT_MISSING_ERROR; @@ -213,7 +213,7 @@ impl Handle { #[cfg(not(all(tokio_unstable, feature = "tracing")))] let _ = name; - let (task, handle) = task::joinable(fut); + let (task, handle) = task::unowned(fut, NoopSchedule); let _ = self.blocking_spawner.spawn(task, &self); handle } diff --git a/tokio/src/runtime/queue.rs b/tokio/src/runtime/queue.rs index 6e91dfa23..c45cb6a5a 100644 --- a/tokio/src/runtime/queue.rs +++ b/tokio/src/runtime/queue.rs @@ -106,13 +106,8 @@ impl Local { break tail; } else if steal != real { // Concurrently stealing, this will free up capacity, so only - // push the new task onto the inject queue - // - // If the task fails to be pushed on the injection queue, there - // is nothing to be done at this point as the task cannot be a - // newly spawned task. Shutting down this task is handled by the - // worker shutdown process. - let _ = inject.push(task); + // push the task onto the inject queue + inject.push(task); return; } else { // Push the current task and half of the queue into the diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index e4624c7b7..06848e734 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -93,7 +93,7 @@ pub(super) enum Stage { impl Cell { /// Allocates a new task cell, containing the header, trailer, and core /// structures. - pub(super) fn new(future: T, state: State) -> Box> { + pub(super) fn new(future: T, scheduler: S, state: State) -> Box> { #[cfg(all(tokio_unstable, feature = "tracing"))] let id = future.id(); Box::new(Cell { @@ -107,7 +107,7 @@ impl Cell { }, core: Core { scheduler: Scheduler { - scheduler: UnsafeCell::new(None), + scheduler: UnsafeCell::new(Some(scheduler)), }, stage: CoreStage { stage: UnsafeCell::new(Stage::Running(future)), @@ -125,34 +125,6 @@ impl Scheduler { self.scheduler.with_mut(f) } - /// Bind a scheduler to the task. - /// - /// This only happens on the first poll and must be preceded by a call to - /// `is_bound` to determine if binding is appropriate or not. - /// - /// # Safety - /// - /// Binding must not be done concurrently since it will mutate the task - /// core through a shared reference. - pub(super) fn bind_scheduler(&self, task: Task) { - // This function may be called concurrently, but the __first__ time it - // is called, the caller has unique access to this field. All subsequent - // concurrent calls will be via the `Waker`, which will "happens after" - // the first poll. - // - // In other words, it is always safe to read the field and it is safe to - // write to the field when it is `None`. - debug_assert!(!self.is_bound()); - - // Bind the task to the scheduler - let scheduler = S::bind(task); - - // Safety: As `scheduler` is not set, this is the first poll - self.scheduler.with_mut(|ptr| unsafe { - *ptr = Some(scheduler); - }); - } - /// Returns true if the task is bound to a scheduler. pub(super) fn is_bound(&self) -> bool { // Safety: never called concurrently w/ a mutation. diff --git a/tokio/src/runtime/task/harness.rs b/tokio/src/runtime/task/harness.rs index 9f0b10711..7d19f3d64 100644 --- a/tokio/src/runtime/task/harness.rs +++ b/tokio/src/runtime/task/harness.rs @@ -254,16 +254,13 @@ where } fn transition_to_running(&self) -> TransitionToRunning { - // If this is the first time the task is polled, the task will be bound - // to the scheduler, in which case the task ref count must be - // incremented. - let is_not_bound = !self.scheduler.is_bound(); + debug_assert!(self.scheduler.is_bound()); // Transition the task to the running state. // // A failure to transition here indicates the task has been cancelled // while in the run queue pending execution. - let snapshot = match self.header.state.transition_to_running(is_not_bound) { + let snapshot = match self.header.state.transition_to_running() { Ok(snapshot) => snapshot, Err(_) => { // The task was shutdown while in the run queue. At this point, @@ -273,20 +270,6 @@ where } }; - if is_not_bound { - // Ensure the task is bound to a scheduler instance. Since this is - // the first time polling the task, a scheduler instance is pulled - // from the local context and assigned to the task. - // - // The scheduler maintains ownership of the task and responds to - // `wake` calls. - // - // The task reference count has been incremented. - // - // Safety: Since we have unique access to the task so that we can - // safely call `bind_scheduler`. - self.scheduler.bind_scheduler(self.to_task()); - } TransitionToRunning::Ok(snapshot) } } diff --git a/tokio/src/runtime/task/inject.rs b/tokio/src/runtime/task/inject.rs index 8ca3187a7..640da648f 100644 --- a/tokio/src/runtime/task/inject.rs +++ b/tokio/src/runtime/task/inject.rs @@ -75,15 +75,13 @@ impl Inject { /// Pushes a value into the queue. /// - /// Returns `Err(task)` if pushing fails due to the queue being shutdown. - /// The caller is expected to call `shutdown()` on the task **if and only - /// if** it is a newly spawned task. - pub(crate) fn push(&self, task: task::Notified) -> Result<(), task::Notified> { + /// This does nothing if the queue is closed. + pub(crate) fn push(&self, task: task::Notified) { // Acquire queue lock let mut p = self.pointers.lock(); if p.is_closed { - return Err(task); + return; } // safety: only mutated with the lock held @@ -102,7 +100,6 @@ impl Inject { p.tail = Some(task); self.len.store(len + 1, Release); - Ok(()) } /// Pushes several values into the queue. diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 45e22a72a..63812f9c7 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -1,33 +1,141 @@ +//! This module has containers for storing the tasks spawned on a scheduler. The +//! `OwnedTasks` container is thread-safe but can only store tasks that +//! implement Send. The `LocalOwnedTasks` container is not thread safe, but can +//! store non-Send tasks. +//! +//! The collections can be closed to prevent adding new tasks during shutdown of +//! the scheduler with the collection. + +use crate::future::Future; use crate::loom::sync::Mutex; -use crate::runtime::task::Task; +use crate::runtime::task::{JoinHandle, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; +use std::marker::PhantomData; + pub(crate) struct OwnedTasks { - list: Mutex, as Link>::Target>>, + inner: Mutex>, +} +struct OwnedTasksInner { + list: LinkedList, as Link>::Target>, + closed: bool, +} + +pub(crate) struct LocalOwnedTasks { + list: LinkedList, as Link>::Target>, + closed: bool, + _not_send: PhantomData<*const ()>, } impl OwnedTasks { pub(crate) fn new() -> Self { Self { - list: Mutex::new(LinkedList::new()), + inner: Mutex::new(OwnedTasksInner { + list: LinkedList::new(), + closed: false, + }), } } - pub(crate) fn push_front(&self, task: Task) { - self.list.lock().push_front(task); + /// Bind the provided task to this OwnedTasks instance. This fails if the + /// OwnedTasks has been closed. + pub(crate) fn bind( + &self, + task: T, + scheduler: S, + ) -> (JoinHandle, Option>) + where + S: Schedule, + T: Future + Send + 'static, + T::Output: Send + 'static, + { + let (task, notified, join) = super::new_task(task, scheduler); + + let mut lock = self.inner.lock(); + if lock.closed { + drop(lock); + drop(task); + notified.shutdown(); + (join, None) + } else { + lock.list.push_front(task); + (join, Some(notified)) + } } pub(crate) fn pop_back(&self) -> Option> { - self.list.lock().pop_back() + self.inner.lock().list.pop_back() } /// The caller must ensure that if the provided task is stored in a /// linked list, then it is in this linked list. pub(crate) unsafe fn remove(&self, task: &Task) -> Option> { - self.list.lock().remove(task.header().into()) + self.inner.lock().list.remove(task.header().into()) } pub(crate) fn is_empty(&self) -> bool { - self.list.lock().is_empty() + self.inner.lock().list.is_empty() + } + + #[cfg(feature = "rt-multi-thread")] + pub(crate) fn is_closed(&self) -> bool { + self.inner.lock().closed + } + + /// Close the OwnedTasks. This prevents adding new tasks to the collection. + pub(crate) fn close(&self) { + self.inner.lock().closed = true; + } +} + +impl LocalOwnedTasks { + pub(crate) fn new() -> Self { + Self { + list: LinkedList::new(), + closed: false, + _not_send: PhantomData, + } + } + + pub(crate) fn bind( + &mut self, + task: T, + scheduler: S, + ) -> (JoinHandle, Option>) + where + S: Schedule, + T: Future + 'static, + T::Output: 'static, + { + let (task, notified, join) = super::new_task(task, scheduler); + + if self.closed { + drop(task); + notified.shutdown(); + (join, None) + } else { + self.list.push_front(task); + (join, Some(notified)) + } + } + + pub(crate) fn pop_back(&mut self) -> Option> { + self.list.pop_back() + } + + /// The caller must ensure that if the provided task is stored in a + /// linked list, then it is in this linked list. + pub(crate) unsafe fn remove(&mut self, task: &Task) -> Option> { + self.list.remove(task.header().into()) + } + + pub(crate) fn is_empty(&self) -> bool { + self.list.is_empty() + } + + /// Close the LocalOwnedTasks. This prevents adding new tasks to the + /// collection. + pub(crate) fn close(&mut self) { + self.closed = true; } } diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 5e2477906..1ca5ba579 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -19,7 +19,7 @@ mod join; pub use self::join::JoinHandle; mod list; -pub(crate) use self::list::OwnedTasks; +pub(crate) use self::list::{LocalOwnedTasks, OwnedTasks}; mod raw; use self::raw::RawTask; @@ -57,13 +57,6 @@ unsafe impl Sync for Notified {} pub(crate) type Result = std::result::Result; pub(crate) trait Schedule: Sync + Sized + 'static { - /// Bind a task to the executor. - /// - /// Guaranteed to be called from the thread that called `poll` on the task. - /// The returned `Schedule` instance is associated with the task and is used - /// as `&self` in the other methods on this trait. - fn bind(task: Task) -> Self; - /// The task has completed work and is ready to be released. The scheduler /// should release it immediately and return it. The task module will batch /// the ref-dec with setting other options. @@ -82,42 +75,46 @@ pub(crate) trait Schedule: Sync + Sized + 'static { } cfg_rt! { - /// Create a new task with an associated join handle - pub(crate) fn joinable(task: T) -> (Notified, JoinHandle) + /// This is the constructor for a new task. Three references to the task are + /// created. The first task reference is usually put into an OwnedTasks + /// immediately. The Notified is sent to the scheduler as an ordinary + /// notification. + fn new_task( + task: T, + scheduler: S + ) -> (Task, Notified, JoinHandle) where - T: Future + Send + 'static, S: Schedule, - { - let raw = RawTask::new::<_, S>(task); - - let task = Task { - raw, - _p: PhantomData, - }; - - let join = JoinHandle::new(raw); - - (Notified(task), join) - } -} - -cfg_rt! { - /// Create a new `!Send` task with an associated join handle - pub(crate) unsafe fn joinable_local(task: T) -> (Notified, JoinHandle) - where T: Future + 'static, - S: Schedule, + T::Output: 'static, { - let raw = RawTask::new::<_, S>(task); - + let raw = RawTask::new::(task, scheduler); let task = Task { raw, _p: PhantomData, }; - + let notified = Notified(Task { + raw, + _p: PhantomData, + }); let join = JoinHandle::new(raw); - (Notified(task), join) + (task, notified, join) + } + + /// Create a new task with an associated join handle. This method is used + /// only when the task is not going to be stored in an `OwnedTasks` list. + /// + /// Currently only blocking tasks and tests use this method. + pub(crate) fn unowned(task: T, scheduler: S) -> (Notified, JoinHandle) + where + S: Schedule, + T: Send + Future + 'static, + T::Output: Send + 'static, + { + let (task, notified, join) = new_task(task, scheduler); + drop(task); + (notified, join) } } diff --git a/tokio/src/runtime/task/raw.rs b/tokio/src/runtime/task/raw.rs index 56d65d5a6..8c2c3f732 100644 --- a/tokio/src/runtime/task/raw.rs +++ b/tokio/src/runtime/task/raw.rs @@ -42,12 +42,12 @@ pub(super) fn vtable() -> &'static Vtable { } impl RawTask { - pub(super) fn new(task: T) -> RawTask + pub(super) fn new(task: T, scheduler: S) -> RawTask where T: Future, S: Schedule, { - let ptr = Box::into_raw(Cell::<_, S>::new(task, State::new())); + let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new())); let ptr = unsafe { NonNull::new_unchecked(ptr as *mut Header) }; RawTask { ptr } diff --git a/tokio/src/runtime/task/state.rs b/tokio/src/runtime/task/state.rs index 603772162..2580e9c83 100644 --- a/tokio/src/runtime/task/state.rs +++ b/tokio/src/runtime/task/state.rs @@ -54,22 +54,23 @@ const REF_ONE: usize = 1 << REF_COUNT_SHIFT; /// State a task is initialized with /// -/// A task is initialized with two references: one for the scheduler and one for -/// the `JoinHandle`. As the task starts with a `JoinHandle`, `JOIN_INTEREST` is -/// set. A new task is immediately pushed into the run queue for execution and -/// starts with the `NOTIFIED` flag set. -const INITIAL_STATE: usize = (REF_ONE * 2) | JOIN_INTEREST | NOTIFIED; +/// A task is initialized with three references: +/// +/// * A reference that will be stored in an OwnedTasks or LocalOwnedTasks. +/// * A reference that will be sent to the scheduler as an ordinary notification. +/// * A reference for the JoinHandle. +/// +/// As the task starts with a `JoinHandle`, `JOIN_INTEREST` is set. +/// As the task starts with a `Notified`, `NOTIFIED` is set. +const INITIAL_STATE: usize = (REF_ONE * 3) | JOIN_INTEREST | NOTIFIED; /// All transitions are performed via RMW operations. This establishes an /// unambiguous modification order. impl State { /// Return a task's initial state pub(super) fn new() -> State { - // A task is initialized with three references: one for the scheduler, - // one for the `JoinHandle`, one for the task handle made available in - // release. As the task starts with a `JoinHandle`, `JOIN_INTEREST` is - // set. A new task is immediately pushed into the run queue for - // execution and starts with the `NOTIFIED` flag set. + // The raw task returned by this method has a ref-count of three. See + // the comment on INITIAL_STATE for more. State { val: AtomicUsize::new(INITIAL_STATE), } @@ -82,10 +83,8 @@ impl State { /// Attempt to transition the lifecycle to `Running`. /// - /// If `ref_inc` is set, the reference count is also incremented. - /// /// The `NOTIFIED` bit is always unset. - pub(super) fn transition_to_running(&self, ref_inc: bool) -> UpdateResult { + pub(super) fn transition_to_running(&self) -> UpdateResult { self.fetch_update(|curr| { assert!(curr.is_notified()); @@ -95,10 +94,6 @@ impl State { return None; } - if ref_inc { - next.ref_inc(); - } - next.set_running(); next.unset_notified(); Some(next) diff --git a/tokio/src/runtime/tests/loom_queue.rs b/tokio/src/runtime/tests/loom_queue.rs index 977c9159b..a1ed1717b 100644 --- a/tokio/src/runtime/tests/loom_queue.rs +++ b/tokio/src/runtime/tests/loom_queue.rs @@ -1,5 +1,6 @@ +use crate::runtime::blocking::NoopSchedule; use crate::runtime::queue; -use crate::runtime::task::{self, Inject, Schedule, Task}; +use crate::runtime::task::Inject; use loom::thread; @@ -30,7 +31,7 @@ fn basic() { for _ in 0..2 { for _ in 0..2 { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); } @@ -39,7 +40,7 @@ fn basic() { } // Push another task - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); while local.pop().is_some() { @@ -81,7 +82,7 @@ fn steal_overflow() { let mut n = 0; // push a task, pop a task - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); if local.pop().is_some() { @@ -89,7 +90,7 @@ fn steal_overflow() { } for _ in 0..6 { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); } @@ -111,7 +112,7 @@ fn steal_overflow() { fn multi_stealer() { const NUM_TASKS: usize = 5; - fn steal_tasks(steal: queue::Steal) -> usize { + fn steal_tasks(steal: queue::Steal) -> usize { let (_, mut local) = queue::local(); if steal.steal_into(&mut local).is_none() { @@ -133,7 +134,7 @@ fn multi_stealer() { // Push work for _ in 0..NUM_TASKS { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); } @@ -170,10 +171,10 @@ fn chained_steal() { // Load up some tasks for _ in 0..4 { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); l1.push_back(task, &inject); - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); l2.push_back(task, &inject); } @@ -197,20 +198,3 @@ fn chained_steal() { while inject.pop().is_some() {} }); } - -struct Runtime; - -impl Schedule for Runtime { - fn bind(task: Task) -> Runtime { - std::mem::forget(task); - Runtime - } - - fn release(&self, _task: &Task) -> Option> { - None - } - - fn schedule(&self, _task: task::Notified) { - unreachable!(); - } -} diff --git a/tokio/src/runtime/tests/mod.rs b/tokio/src/runtime/tests/mod.rs index 596e47dfd..f3c6a9bd7 100644 --- a/tokio/src/runtime/tests/mod.rs +++ b/tokio/src/runtime/tests/mod.rs @@ -1,21 +1,30 @@ -#[cfg(not(all(tokio_unstable, feature = "tracing")))] -use crate::runtime::task::joinable; +use self::unowned_wrapper::unowned; -#[cfg(all(tokio_unstable, feature = "tracing"))] -use self::joinable_wrapper::joinable; +mod unowned_wrapper { + use crate::runtime::blocking::NoopSchedule; + use crate::runtime::task::{JoinHandle, Notified}; -#[cfg(all(tokio_unstable, feature = "tracing"))] -mod joinable_wrapper { - use crate::runtime::task::{JoinHandle, Notified, Schedule}; - use tracing::Instrument; - - pub(crate) fn joinable(task: T) -> (Notified, JoinHandle) + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(crate) fn unowned(task: T) -> (Notified, JoinHandle) where T: std::future::Future + Send + 'static, - S: Schedule, + T::Output: Send + 'static, { + use tracing::Instrument; let span = tracing::trace_span!("test_span"); - crate::runtime::task::joinable(task.instrument(span)) + let task = task.instrument(span); + let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule); + (task, handle) + } + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + pub(crate) fn unowned(task: T) -> (Notified, JoinHandle) + where + T: std::future::Future + Send + 'static, + T::Output: Send + 'static, + { + let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule); + (task, handle) } } diff --git a/tokio/src/runtime/tests/queue.rs b/tokio/src/runtime/tests/queue.rs index e08dc6d99..428b00207 100644 --- a/tokio/src/runtime/tests/queue.rs +++ b/tokio/src/runtime/tests/queue.rs @@ -10,7 +10,7 @@ fn fits_256() { let inject = Inject::new(); for _ in 0..256 { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); } @@ -25,7 +25,7 @@ fn overflow() { let inject = Inject::new(); for _ in 0..257 { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); } @@ -49,7 +49,7 @@ fn steal_batch() { let inject = Inject::new(); for _ in 0..4 { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local1.push_back(task, &inject); } @@ -103,7 +103,7 @@ fn stress1() { for _ in 0..NUM_LOCAL { for _ in 0..NUM_PUSH { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); } @@ -158,7 +158,7 @@ fn stress2() { let mut num_pop = 0; for i in 0..NUM_TASKS { - let (task, _) = super::joinable::<_, Runtime>(async {}); + let (task, _) = super::unowned(async {}); local.push_back(task, &inject); if i % 128 == 0 && local.pop().is_some() { @@ -187,11 +187,6 @@ fn stress2() { struct Runtime; impl Schedule for Runtime { - fn bind(task: Task) -> Runtime { - std::mem::forget(task); - Runtime - } - fn release(&self, _task: &Task) -> Option> { None } diff --git a/tokio/src/runtime/tests/task.rs b/tokio/src/runtime/tests/task.rs index 1f3e89d76..ae5c04c84 100644 --- a/tokio/src/runtime/tests/task.rs +++ b/tokio/src/runtime/tests/task.rs @@ -1,43 +1,185 @@ -use crate::runtime::task::{self, OwnedTasks, Schedule, Task}; +use crate::runtime::blocking::NoopSchedule; +use crate::runtime::task::{self, unowned, JoinHandle, OwnedTasks, Schedule, Task}; use crate::util::TryLock; use std::collections::VecDeque; +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +struct AssertDropHandle { + is_dropped: Arc, +} +impl AssertDropHandle { + #[track_caller] + fn assert_dropped(&self) { + assert!(self.is_dropped.load(Ordering::SeqCst)); + } + + #[track_caller] + fn assert_not_dropped(&self) { + assert!(!self.is_dropped.load(Ordering::SeqCst)); + } +} + +struct AssertDrop { + is_dropped: Arc, +} +impl AssertDrop { + fn new() -> (Self, AssertDropHandle) { + let shared = Arc::new(AtomicBool::new(false)); + ( + AssertDrop { + is_dropped: shared.clone(), + }, + AssertDropHandle { + is_dropped: shared.clone(), + }, + ) + } +} +impl Drop for AssertDrop { + fn drop(&mut self) { + self.is_dropped.store(true, Ordering::SeqCst); + } +} + +// A Notified does not shut down on drop, but it is dropped once the ref-count +// hits zero. #[test] -fn create_drop() { - let _ = super::joinable::<_, Runtime>(async { unreachable!() }); +fn create_drop1() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + ); + drop(notified); + handle.assert_not_dropped(); + drop(join); + handle.assert_dropped(); +} + +#[test] +fn create_drop2() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + ); + drop(join); + handle.assert_not_dropped(); + drop(notified); + handle.assert_dropped(); +} + +// Shutting down through Notified works +#[test] +fn create_shutdown1() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + ); + drop(join); + handle.assert_not_dropped(); + notified.shutdown(); + handle.assert_dropped(); +} + +#[test] +fn create_shutdown2() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + ); + handle.assert_not_dropped(); + notified.shutdown(); + handle.assert_dropped(); + drop(join); } #[test] fn schedule() { with(|rt| { - let (task, _) = super::joinable(async { + rt.spawn(async { crate::task::yield_now().await; }); - rt.schedule(task); - assert_eq!(2, rt.tick()); + rt.shutdown(); }) } #[test] fn shutdown() { with(|rt| { - let (task, _) = super::joinable(async { + rt.spawn(async { loop { crate::task::yield_now().await; } }); - rt.schedule(task); rt.tick_max(1); rt.shutdown(); }) } +#[test] +fn shutdown_immediately() { + with(|rt| { + rt.spawn(async { + loop { + crate::task::yield_now().await; + } + }); + + rt.shutdown(); + }) +} + +#[test] +fn spawn_during_shutdown() { + static DID_SPAWN: AtomicBool = AtomicBool::new(false); + + struct SpawnOnDrop(Runtime); + impl Drop for SpawnOnDrop { + fn drop(&mut self) { + DID_SPAWN.store(true, Ordering::SeqCst); + self.0.spawn(async {}); + } + } + + with(|rt| { + let rt2 = rt.clone(); + rt.spawn(async move { + let _spawn_on_drop = SpawnOnDrop(rt2); + + loop { + crate::task::yield_now().await; + } + }); + + rt.tick_max(1); + rt.shutdown(); + }); + + assert!(DID_SPAWN.load(Ordering::SeqCst)); +} + fn with(f: impl FnOnce(Runtime)) { struct Reset; @@ -75,6 +217,20 @@ struct Core { static CURRENT: TryLock> = TryLock::new(None); impl Runtime { + fn spawn(&self, future: T) -> JoinHandle + where + T: 'static + Send + Future, + T::Output: 'static + Send, + { + let (handle, notified) = self.0.owned.bind(future, self.clone()); + + if let Some(notified) = notified { + self.schedule(notified); + } + + handle + } + fn tick(&self) -> usize { self.tick_max(usize::MAX) } @@ -102,6 +258,7 @@ impl Runtime { fn shutdown(&self) { let mut core = self.0.core.try_lock().unwrap(); + self.0.owned.close(); while let Some(task) = self.0.owned.pop_back() { task.shutdown(); } @@ -117,12 +274,6 @@ impl Runtime { } impl Schedule for Runtime { - fn bind(task: Task) -> Runtime { - let rt = CURRENT.try_lock().unwrap().as_ref().unwrap().clone(); - rt.0.owned.push_front(task); - rt - } - fn release(&self, task: &Task) -> Option> { // safety: copying worker.rs unsafe { self.0.owned.remove(task) } diff --git a/tokio/src/runtime/thread_pool/mod.rs b/tokio/src/runtime/thread_pool/mod.rs index 96312d346..3808aa264 100644 --- a/tokio/src/runtime/thread_pool/mod.rs +++ b/tokio/src/runtime/thread_pool/mod.rs @@ -12,7 +12,7 @@ pub(crate) use worker::Launch; pub(crate) use worker::block_in_place; use crate::loom::sync::Arc; -use crate::runtime::task::{self, JoinHandle}; +use crate::runtime::task::JoinHandle; use crate::runtime::Parker; use std::fmt; @@ -30,7 +30,7 @@ pub(crate) struct ThreadPool { /// /// The `Spawner` handle is *only* used for spawning new futures. It does not /// impact the lifecycle of the thread pool in any way. The thread pool may -/// shutdown while there are outstanding `Spawner` instances. +/// shut down while there are outstanding `Spawner` instances. /// /// `Spawner` instances are obtained by calling [`ThreadPool::spawner`]. /// @@ -93,15 +93,7 @@ impl Spawner { F: crate::future::Future + Send + 'static, F::Output: Send + 'static, { - let (task, handle) = task::joinable(future); - - if let Err(task) = self.shared.schedule(task, false) { - // The newly spawned task could not be scheduled because the runtime - // is shutting down. The task must be explicitly shutdown at this point. - task.shutdown(); - } - - handle + worker::Shared::bind_new_task(&self.shared, future) } pub(crate) fn shutdown(&mut self) { diff --git a/tokio/src/runtime/thread_pool/worker.rs b/tokio/src/runtime/thread_pool/worker.rs index e91e2ea4b..c0535238c 100644 --- a/tokio/src/runtime/thread_pool/worker.rs +++ b/tokio/src/runtime/thread_pool/worker.rs @@ -3,15 +3,68 @@ //! run queue and other state. When `block_in_place` is called, the worker's //! "core" is handed off to a new thread allowing the scheduler to continue to //! make progress while the originating thread blocks. +//! +//! # Shutdown +//! +//! Shutting down the runtime involves the following steps: +//! +//! 1. The Shared::close method is called. This closes the inject queue and +//! OwnedTasks instance and wakes up all worker threads. +//! +//! 2. Each worker thread observes the close signal next time it runs +//! Core::maintenance by checking whether the inject queue is closed. +//! The Core::is_shutdown flag is set to true. +//! +//! 3. The worker thread calls `pre_shutdown` in parallel. Here, the worker +//! will keep removing tasks from OwnedTasks until it is empty. No new +//! tasks can be pushed to the OwnedTasks during or after this step as it +//! was closed in step 1. +//! +//! 5. The workers call Shared::shutdown to enter the single-threaded phase of +//! shutdown. These calls will push their core to Shared::shutdown_cores, +//! and the last thread to push its core will finish the shutdown procedure. +//! +//! 6. The local run queue of each core is emptied, then the inject queue is +//! emptied. +//! +//! At this point, shutdown has completed. It is not possible for any of the +//! collections to contain any tasks at this point, as each collection was +//! closed first, then emptied afterwards. +//! +//! ## Spawns during shutdown +//! +//! When spawning tasks during shutdown, there are two cases: +//! +//! * The spawner observes the OwnedTasks being open, and the inject queue is +//! closed. +//! * The spawner observes the OwnedTasks being closed and doesn't check the +//! inject queue. +//! +//! The first case can only happen if the OwnedTasks::bind call happens before +//! or during step 1 of shutdown. In this case, the runtime will clean up the +//! task in step 3 of shutdown. +//! +//! In the latter case, the task was not spawned and the task is immediately +//! cancelled by the spawner. +//! +//! The correctness of shutdown requires both the inject queue and OwnedTasks +//! collection to have a closed bit. With a close bit on only the inject queue, +//! spawning could run in to a situation where a task is successfully bound long +//! after the runtime has shut down. With a close bit on only the OwnedTasks, +//! the first spawning situation could result in the notification being pushed +//! to the inject queue after step 6 of shutdown, which would leave a task in +//! the inject queue indefinitely. This would be a ref-count cycle and a memory +//! leak. use crate::coop; +use crate::future::Future; use crate::loom::rand::seed; use crate::loom::sync::{Arc, Mutex}; use crate::park::{Park, Unpark}; use crate::runtime; use crate::runtime::enter::EnterContext; use crate::runtime::park::{Parker, Unparker}; -use crate::runtime::task::{Inject, OwnedTasks}; +use crate::runtime::task::{Inject, JoinHandle, OwnedTasks}; use crate::runtime::thread_pool::{AtomicCell, Idle}; use crate::runtime::{queue, task}; use crate::util::FastRand; @@ -44,7 +97,7 @@ struct Core { lifo_slot: Option, /// The worker-local run queue. - run_queue: queue::Local>, + run_queue: queue::Local>, /// True if the worker is currently searching for more work. Searching /// involves attempting to steal from other workers. @@ -70,13 +123,13 @@ pub(super) struct Shared { remotes: Box<[Remote]>, /// Submit work to the scheduler while **not** currently on a worker thread. - inject: Inject>, + inject: Inject>, /// Coordinates idle workers idle: Idle, /// Collection of all active tasks spawned onto this executor. - owned: OwnedTasks>, + owned: OwnedTasks>, /// Cores that have observed the shutdown signal /// @@ -89,7 +142,7 @@ pub(super) struct Shared { /// Used to communicate with a worker from other threads. struct Remote { /// Steal tasks from this worker. - steal: queue::Steal>, + steal: queue::Steal>, /// Unparks the associated worker thread unpark: Unparker, @@ -113,10 +166,10 @@ pub(crate) struct Launch(Vec>); type RunResult = Result, ()>; /// A task handle -type Task = task::Task>; +type Task = task::Task>; /// A notified task handle -type Notified = task::Notified>; +type Notified = task::Notified>; // Tracks thread-local state scoped_thread_local!(static CURRENT: Context); @@ -543,13 +596,16 @@ impl Core { /// Signals all tasks to shut down, and waits for them to complete. Must run /// before we enter the single-threaded phase of shutdown processing. fn pre_shutdown(&mut self, worker: &Worker) { + // The OwnedTasks was closed in Shared::close. + debug_assert!(worker.shared.owned.is_closed()); + // Signal to all tasks to shut down. while let Some(header) = worker.shared.owned.pop_back() { header.shutdown(); } } - // Shutdown the core + /// Shutdown the core fn shutdown(&mut self) { // Take the core let mut park = self.park.take().expect("park missing"); @@ -563,46 +619,42 @@ impl Core { impl Worker { /// Returns a reference to the scheduler's injection queue - fn inject(&self) -> &Inject> { + fn inject(&self) -> &Inject> { &self.shared.inject } } -impl task::Schedule for Arc { - fn bind(task: Task) -> Arc { - CURRENT.with(|maybe_cx| { - let cx = maybe_cx.expect("scheduler context missing"); - - // Track the task - cx.worker.shared.owned.push_front(task); - - // Return a clone of the worker - cx.worker.clone() - }) - } - +impl task::Schedule for Arc { fn release(&self, task: &Task) -> Option { // SAFETY: Inserted into owned in bind. - unsafe { self.shared.owned.remove(task) } + unsafe { self.owned.remove(task) } } fn schedule(&self, task: Notified) { - // Because this is not a newly spawned task, if scheduling fails due to - // the runtime shutting down, there is no special work that must happen - // here. - let _ = self.shared.schedule(task, false); + (**self).schedule(task, false); } fn yield_now(&self, task: Notified) { - // Because this is not a newly spawned task, if scheduling fails due to - // the runtime shutting down, there is no special work that must happen - // here. - let _ = self.shared.schedule(task, true); + (**self).schedule(task, true); } } impl Shared { - pub(super) fn schedule(&self, task: Notified, is_yield: bool) -> Result<(), Notified> { + pub(super) fn bind_new_task(me: &Arc, future: T) -> JoinHandle + where + T: Future + Send + 'static, + T::Output: Send + 'static, + { + let (handle, notified) = me.owned.bind(future, me.clone()); + + if let Some(notified) = notified { + me.schedule(notified, false); + } + + handle + } + + pub(super) fn schedule(&self, task: Notified, is_yield: bool) { CURRENT.with(|maybe_cx| { if let Some(cx) = maybe_cx { // Make sure the task is part of the **current** scheduler. @@ -610,15 +662,14 @@ impl Shared { // And the current thread still holds a core if let Some(core) = cx.core.borrow_mut().as_mut() { self.schedule_local(core, task, is_yield); - return Ok(()); + return; } } } - // Otherwise, use the inject queue - self.inject.push(task)?; + // Otherwise, use the inject queue. + self.inject.push(task); self.notify_parked(); - Ok(()) }) } @@ -654,6 +705,7 @@ impl Shared { pub(super) fn close(&self) { if self.inject.close() { + self.owned.close(); self.notify_all(); } } diff --git a/tokio/src/task/local.rs b/tokio/src/task/local.rs index 7404cc2c1..c74c8b438 100644 --- a/tokio/src/task/local.rs +++ b/tokio/src/task/local.rs @@ -1,6 +1,6 @@ //! Runs `!Send` futures on the current thread. use crate::loom::sync::{Arc, Mutex}; -use crate::runtime::task::{self, JoinHandle, OwnedTasks, Task}; +use crate::runtime::task::{self, JoinHandle, LocalOwnedTasks, Task}; use crate::sync::AtomicWaker; use std::cell::{Cell, RefCell}; @@ -232,7 +232,7 @@ struct Context { struct Tasks { /// Collection of all active tasks spawned onto this executor. - owned: OwnedTasks>, + owned: LocalOwnedTasks>, /// Local run queue sender and receiver. queue: VecDeque>>, @@ -308,10 +308,12 @@ cfg_rt! { let cx = maybe_cx .expect("`spawn_local` called from outside of a `task::LocalSet`"); - // Safety: Tasks are only polled and dropped from the thread that - // spawns them. - let (task, handle) = unsafe { task::joinable_local(future) }; - cx.tasks.borrow_mut().queue.push_back(task); + let (handle, notified) = cx.tasks.borrow_mut().owned.bind(future, cx.shared.clone()); + + if let Some(notified) = notified { + cx.shared.schedule(notified); + } + handle }) } @@ -333,7 +335,7 @@ impl LocalSet { tick: Cell::new(0), context: Context { tasks: RefCell::new(Tasks { - owned: OwnedTasks::new(), + owned: LocalOwnedTasks::new(), queue: VecDeque::with_capacity(INITIAL_CAPACITY), }), shared: Arc::new(Shared { @@ -388,8 +390,18 @@ impl LocalSet { F::Output: 'static, { let future = crate::util::trace::task(future, "local", None); - let (task, handle) = unsafe { task::joinable_local(future) }; - self.context.tasks.borrow_mut().queue.push_back(task); + + let (handle, notified) = self + .context + .tasks + .borrow_mut() + .owned + .bind(future, self.context.shared.clone()); + + if let Some(notified) = notified { + self.context.shared.schedule(notified); + } + self.context.shared.waker.wake(); handle } @@ -593,6 +605,12 @@ impl Default for LocalSet { impl Drop for LocalSet { fn drop(&mut self) { self.with(|| { + // Close the LocalOwnedTasks. This ensures that any calls to + // spawn_local in the destructor of a future on this LocalSet will + // immediately cancel the task, and prevents the task from being + // added to `owned`. + self.context.tasks.borrow_mut().owned.close(); + // Loop required here to ensure borrow is dropped between iterations #[allow(clippy::while_let_loop)] loop { @@ -671,14 +689,6 @@ impl Shared { } impl task::Schedule for Arc { - fn bind(task: Task) -> Arc { - CURRENT.with(|maybe_cx| { - let cx = maybe_cx.expect("scheduler context missing"); - cx.tasks.borrow_mut().owned.push_front(task); - cx.shared.clone() - }) - } - fn release(&self, task: &Task) -> Option> { CURRENT.with(|maybe_cx| { let cx = maybe_cx.expect("scheduler context missing"); @@ -686,7 +696,7 @@ impl task::Schedule for Arc { assert!(cx.shared.ptr_eq(self)); // safety: task must be contained by list. It is inserted into the - // list in `bind`. + // list when spawning. unsafe { cx.tasks.borrow_mut().owned.remove(&task) } }) } diff --git a/tokio/tests/support/mock_file.rs b/tokio/tests/support/mock_file.rs index 1ce326b62..60f6bedbb 100644 --- a/tokio/tests/support/mock_file.rs +++ b/tokio/tests/support/mock_file.rs @@ -211,7 +211,7 @@ impl Read for &'_ File { assert!(dst.len() >= data.len()); assert!(dst.len() <= 16 * 1024, "actual = {}", dst.len()); // max buffer - &mut dst[..data.len()].copy_from_slice(&data); + dst[..data.len()].copy_from_slice(&data); Ok(data.len()) } Some(Read(Err(e))) => Err(e), From 0cefa85bfa8a0d03b7a50eceff798ffa64f0bef9 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 21 Jul 2021 20:05:21 +0200 Subject: [PATCH 28/35] runtime: make scheduler non-optional (#3980) --- tokio/src/runtime/task/core.rs | 70 ++----------------------------- tokio/src/runtime/task/harness.rs | 21 ++++------ 2 files changed, 12 insertions(+), 79 deletions(-) diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index 06848e734..e8e4fef7c 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -13,7 +13,7 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; use crate::runtime::task::raw::{self, Vtable}; use crate::runtime::task::state::State; -use crate::runtime::task::{Notified, Schedule, Task}; +use crate::runtime::task::Schedule; use crate::util::linked_list; use std::pin::Pin; @@ -36,10 +36,6 @@ pub(super) struct Cell { pub(super) trailer: Trailer, } -pub(super) struct Scheduler { - scheduler: UnsafeCell>, -} - pub(super) struct CoreStage { stage: UnsafeCell>, } @@ -49,7 +45,7 @@ pub(super) struct CoreStage { /// Holds the future or output, depending on the stage of execution. pub(super) struct Core { /// Scheduler used to drive this future - pub(super) scheduler: Scheduler, + pub(super) scheduler: S, /// Either the future or the output pub(super) stage: CoreStage, @@ -106,9 +102,7 @@ impl Cell { id, }, core: Core { - scheduler: Scheduler { - scheduler: UnsafeCell::new(Some(scheduler)), - }, + scheduler, stage: CoreStage { stage: UnsafeCell::new(Stage::Running(future)), }, @@ -120,64 +114,6 @@ impl Cell { } } -impl Scheduler { - pub(super) fn with_mut(&self, f: impl FnOnce(*mut Option) -> R) -> R { - self.scheduler.with_mut(f) - } - - /// Returns true if the task is bound to a scheduler. - pub(super) fn is_bound(&self) -> bool { - // Safety: never called concurrently w/ a mutation. - self.scheduler.with(|ptr| unsafe { (*ptr).is_some() }) - } - - /// Schedule the future for execution - pub(super) fn schedule(&self, task: Notified) { - self.scheduler.with(|ptr| { - // Safety: Can only be called after initial `poll`, which is the - // only time the field is mutated. - match unsafe { &*ptr } { - Some(scheduler) => scheduler.schedule(task), - None => panic!("no scheduler set"), - } - }); - } - - /// Schedule the future for execution in the near future, yielding the - /// thread to other tasks. - pub(super) fn yield_now(&self, task: Notified) { - self.scheduler.with(|ptr| { - // Safety: Can only be called after initial `poll`, which is the - // only time the field is mutated. - match unsafe { &*ptr } { - Some(scheduler) => scheduler.yield_now(task), - None => panic!("no scheduler set"), - } - }); - } - - /// Release the task - /// - /// If the `Scheduler` implementation is able to, it returns the `Task` - /// handle immediately. The caller of this function will batch a ref-dec - /// with a state change. - pub(super) fn release(&self, task: Task) -> Option> { - use std::mem::ManuallyDrop; - - let task = ManuallyDrop::new(task); - - self.scheduler.with(|ptr| { - // Safety: Can only be called after initial `poll`, which is the - // only time the field is mutated. - match unsafe { &*ptr } { - Some(scheduler) => scheduler.release(&*task), - // Task was never polled - None => None, - } - }) - } -} - impl CoreStage { pub(super) fn with_mut(&self, f: impl FnOnce(*mut Stage) -> R) -> R { self.stage.with_mut(f) diff --git a/tokio/src/runtime/task/harness.rs b/tokio/src/runtime/task/harness.rs index 7d19f3d64..7984b6127 100644 --- a/tokio/src/runtime/task/harness.rs +++ b/tokio/src/runtime/task/harness.rs @@ -1,5 +1,5 @@ use crate::future::Future; -use crate::runtime::task::core::{Cell, Core, CoreStage, Header, Scheduler, Trailer}; +use crate::runtime::task::core::{Cell, Core, CoreStage, Header, Trailer}; use crate::runtime::task::state::Snapshot; use crate::runtime::task::waker::waker_ref; use crate::runtime::task::{JoinError, Notified, Schedule, Task}; @@ -95,7 +95,6 @@ where // Check causality self.core().stage.with_mut(drop); - self.core().scheduler.with_mut(drop); unsafe { drop(Box::from_raw(self.cell.as_ptr())); @@ -219,7 +218,7 @@ enum TransitionToRunning { struct SchedulerView<'a, S> { header: &'a Header, - scheduler: &'a Scheduler, + scheduler: &'a S, } impl<'a, S> SchedulerView<'a, S> @@ -233,17 +232,17 @@ where /// Returns true if the task should be deallocated. fn transition_to_terminal(&self, is_join_interested: bool) -> bool { - let ref_dec = if self.scheduler.is_bound() { - if let Some(task) = self.scheduler.release(self.to_task()) { - mem::forget(task); - true - } else { - false - } + let me = self.to_task(); + + let ref_dec = if let Some(task) = self.scheduler.release(&me) { + mem::forget(task); + true } else { false }; + mem::forget(me); + // This might deallocate let snapshot = self .header @@ -254,8 +253,6 @@ where } fn transition_to_running(&self) -> TransitionToRunning { - debug_assert!(self.scheduler.is_bound()); - // Transition the task to the running state. // // A failure to transition here indicates the task has been cancelled From ced7992f6555d884cac452d177b094eb44a58a29 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 22 Jul 2021 09:06:38 +0200 Subject: [PATCH 29/35] runtime: add large test and fix leak it found (#3967) --- tokio/src/runtime/task/harness.rs | 41 +- tokio/src/runtime/tests/mod.rs | 3 + tokio/src/runtime/tests/task_combinations.rs | 380 +++++++++++++++++++ tokio/tests/task_abort.rs | 43 +-- 4 files changed, 429 insertions(+), 38 deletions(-) create mode 100644 tokio/src/runtime/tests/task_combinations.rs diff --git a/tokio/src/runtime/task/harness.rs b/tokio/src/runtime/task/harness.rs index 7984b6127..c3603b50f 100644 --- a/tokio/src/runtime/task/harness.rs +++ b/tokio/src/runtime/task/harness.rs @@ -111,6 +111,8 @@ where } pub(super) fn drop_join_handle_slow(self) { + let mut maybe_panic = None; + // Try to unset `JOIN_INTEREST`. This must be done as a first step in // case the task concurrently completed. if self.header().state.unset_join_interested().is_err() { @@ -119,11 +121,20 @@ where // the scheduler or `JoinHandle`. i.e. if the output remains in the // task structure until the task is deallocated, it may be dropped // by a Waker on any arbitrary thread. - self.core().stage.drop_future_or_output(); + let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + self.core().stage.drop_future_or_output(); + })); + if let Err(panic) = panic { + maybe_panic = Some(panic); + } } // Drop the `JoinHandle` reference, possibly deallocating the task self.drop_reference(); + + if let Some(panic) = maybe_panic { + panic::resume_unwind(panic); + } } // ===== waker behavior ===== @@ -182,17 +193,25 @@ where // ====== internal ====== fn complete(self, output: super::Result, is_join_interested: bool) { - if is_join_interested { - // Store the output. The future has already been dropped - // - // Safety: Mutual exclusion is obtained by having transitioned the task - // state -> Running - let stage = &self.core().stage; - stage.store_output(output); + // We catch panics here because dropping the output may panic. + // + // Dropping the output can also happen in the first branch inside + // transition_to_complete. + let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| { + if is_join_interested { + // Store the output. The future has already been dropped + // + // Safety: Mutual exclusion is obtained by having transitioned the task + // state -> Running + let stage = &self.core().stage; + stage.store_output(output); - // Transition to `Complete`, notifying the `JoinHandle` if necessary. - transition_to_complete(self.header(), stage, &self.trailer()); - } + // Transition to `Complete`, notifying the `JoinHandle` if necessary. + transition_to_complete(self.header(), stage, &self.trailer()); + } else { + drop(output); + } + })); // The task has completed execution and will no longer be scheduled. // diff --git a/tokio/src/runtime/tests/mod.rs b/tokio/src/runtime/tests/mod.rs index f3c6a9bd7..e0379a68c 100644 --- a/tokio/src/runtime/tests/mod.rs +++ b/tokio/src/runtime/tests/mod.rs @@ -40,6 +40,9 @@ cfg_loom! { cfg_not_loom! { mod queue; + #[cfg(not(miri))] + mod task_combinations; + #[cfg(miri)] mod task; } diff --git a/tokio/src/runtime/tests/task_combinations.rs b/tokio/src/runtime/tests/task_combinations.rs new file mode 100644 index 000000000..76ce2330c --- /dev/null +++ b/tokio/src/runtime/tests/task_combinations.rs @@ -0,0 +1,380 @@ +use std::future::Future; +use std::panic; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use crate::runtime::Builder; +use crate::sync::oneshot; +use crate::task::JoinHandle; + +use futures::future::FutureExt; + +// Enums for each option in the combinations being tested + +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiRuntime { + CurrentThread, + Multi1, + Multi2, +} +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiLocalSet { + Yes, + No, +} +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiTask { + PanicOnRun, + PanicOnDrop, + PanicOnRunAndDrop, + NoPanic, +} +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiOutput { + PanicOnDrop, + NoPanic, +} +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiJoinInterest { + Polled, + NotPolled, +} +#[allow(clippy::enum_variant_names)] // we aren't using glob imports +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiJoinHandle { + DropImmediately = 1, + DropFirstPoll = 2, + DropAfterNoConsume = 3, + DropAfterConsume = 4, +} +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiAbort { + NotAborted = 0, + AbortedImmediately = 1, + AbortedFirstPoll = 2, + AbortedAfterFinish = 3, + AbortedAfterConsumeOutput = 4, +} + +#[test] +fn test_combinations() { + let mut rt = &[ + CombiRuntime::CurrentThread, + CombiRuntime::Multi1, + CombiRuntime::Multi2, + ][..]; + + if cfg!(miri) { + rt = &[CombiRuntime::CurrentThread]; + } + + let ls = [CombiLocalSet::Yes, CombiLocalSet::No]; + let task = [ + CombiTask::NoPanic, + CombiTask::PanicOnRun, + CombiTask::PanicOnDrop, + CombiTask::PanicOnRunAndDrop, + ]; + let output = [CombiOutput::NoPanic, CombiOutput::PanicOnDrop]; + let ji = [CombiJoinInterest::Polled, CombiJoinInterest::NotPolled]; + let jh = [ + CombiJoinHandle::DropImmediately, + CombiJoinHandle::DropFirstPoll, + CombiJoinHandle::DropAfterNoConsume, + CombiJoinHandle::DropAfterConsume, + ]; + let abort = [ + CombiAbort::NotAborted, + CombiAbort::AbortedImmediately, + CombiAbort::AbortedFirstPoll, + CombiAbort::AbortedAfterFinish, + CombiAbort::AbortedAfterConsumeOutput, + ]; + + for rt in rt.iter().copied() { + for ls in ls.iter().copied() { + for task in task.iter().copied() { + for output in output.iter().copied() { + for ji in ji.iter().copied() { + for jh in jh.iter().copied() { + for abort in abort.iter().copied() { + test_combination(rt, ls, task, output, ji, jh, abort); + } + } + } + } + } + } + } +} + +fn test_combination( + rt: CombiRuntime, + ls: CombiLocalSet, + task: CombiTask, + output: CombiOutput, + ji: CombiJoinInterest, + jh: CombiJoinHandle, + abort: CombiAbort, +) { + if (jh as usize) < (abort as usize) { + // drop before abort not possible + return; + } + if (task == CombiTask::PanicOnDrop) && (output == CombiOutput::PanicOnDrop) { + // this causes double panic + return; + } + if (task == CombiTask::PanicOnRunAndDrop) && (abort != CombiAbort::AbortedImmediately) { + // this causes double panic + return; + } + + println!("Runtime {:?}, LocalSet {:?}, Task {:?}, Output {:?}, JoinInterest {:?}, JoinHandle {:?}, Abort {:?}", rt, ls, task, output, ji, jh, abort); + + // A runtime optionally with a LocalSet + struct Rt { + rt: crate::runtime::Runtime, + ls: Option, + } + impl Rt { + fn new(rt: CombiRuntime, ls: CombiLocalSet) -> Self { + let rt = match rt { + CombiRuntime::CurrentThread => Builder::new_current_thread().build().unwrap(), + CombiRuntime::Multi1 => Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(), + CombiRuntime::Multi2 => Builder::new_multi_thread() + .worker_threads(2) + .build() + .unwrap(), + }; + + let ls = match ls { + CombiLocalSet::Yes => Some(crate::task::LocalSet::new()), + CombiLocalSet::No => None, + }; + + Self { rt, ls } + } + fn block_on(&self, task: T) -> T::Output + where + T: Future, + { + match &self.ls { + Some(ls) => ls.block_on(&self.rt, task), + None => self.rt.block_on(task), + } + } + fn spawn(&self, task: T) -> JoinHandle + where + T: Future + Send + 'static, + T::Output: Send + 'static, + { + match &self.ls { + Some(ls) => ls.spawn_local(task), + None => self.rt.spawn(task), + } + } + } + + // The type used for the output of the future + struct Output { + panic_on_drop: bool, + on_drop: Option>, + } + impl Output { + fn disarm(&mut self) { + self.panic_on_drop = false; + } + } + impl Drop for Output { + fn drop(&mut self) { + let _ = self.on_drop.take().unwrap().send(()); + if self.panic_on_drop { + panic!("Panicking in Output"); + } + } + } + + // A wrapper around the future that is spawned + struct FutWrapper { + inner: F, + on_drop: Option>, + panic_on_drop: bool, + } + impl Future for FutWrapper { + type Output = F::Output; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + unsafe { + let me = Pin::into_inner_unchecked(self); + let inner = Pin::new_unchecked(&mut me.inner); + inner.poll(cx) + } + } + } + impl Drop for FutWrapper { + fn drop(&mut self) { + let _: Result<(), ()> = self.on_drop.take().unwrap().send(()); + if self.panic_on_drop { + panic!("Panicking in FutWrapper"); + } + } + } + + // The channels passed to the task + struct Signals { + on_first_poll: Option>, + wait_complete: Option>, + on_output_drop: Option>, + } + + // The task we will spawn + async fn my_task(mut signal: Signals, task: CombiTask, out: CombiOutput) -> Output { + // Signal that we have been polled once + let _ = signal.on_first_poll.take().unwrap().send(()); + + // Wait for a signal, then complete the future + let _ = signal.wait_complete.take().unwrap().await; + + // If the task gets past wait_complete without yielding, then aborts + // may not be caught without this yield_now. + crate::task::yield_now().await; + + if task == CombiTask::PanicOnRun || task == CombiTask::PanicOnRunAndDrop { + panic!("Panicking in my_task on {:?}", std::thread::current().id()); + } + + Output { + panic_on_drop: out == CombiOutput::PanicOnDrop, + on_drop: signal.on_output_drop.take(), + } + } + + let rt = Rt::new(rt, ls); + + let (on_first_poll, wait_first_poll) = oneshot::channel(); + let (on_complete, wait_complete) = oneshot::channel(); + let (on_future_drop, wait_future_drop) = oneshot::channel(); + let (on_output_drop, wait_output_drop) = oneshot::channel(); + let signal = Signals { + on_first_poll: Some(on_first_poll), + wait_complete: Some(wait_complete), + on_output_drop: Some(on_output_drop), + }; + + // === Spawn task === + let mut handle = Some(rt.spawn(FutWrapper { + inner: my_task(signal, task, output), + on_drop: Some(on_future_drop), + panic_on_drop: task == CombiTask::PanicOnDrop || task == CombiTask::PanicOnRunAndDrop, + })); + + // Keep track of whether the task has been killed with an abort + let mut aborted = false; + + // If we want to poll the JoinHandle, do it now + if ji == CombiJoinInterest::Polled { + assert!( + handle.as_mut().unwrap().now_or_never().is_none(), + "Polling handle succeeded" + ); + } + + if abort == CombiAbort::AbortedImmediately { + handle.as_mut().unwrap().abort(); + aborted = true; + } + if jh == CombiJoinHandle::DropImmediately { + drop(handle.take().unwrap()); + } + + // === Wait for first poll === + let got_polled = rt.block_on(wait_first_poll).is_ok(); + if !got_polled { + // it's possible that we are aborted but still got polled + assert!( + aborted, + "Task completed without ever being polled but was not aborted." + ); + } + + if abort == CombiAbort::AbortedFirstPoll { + handle.as_mut().unwrap().abort(); + aborted = true; + } + if jh == CombiJoinHandle::DropFirstPoll { + drop(handle.take().unwrap()); + } + + // Signal the future that it can return now + let _ = on_complete.send(()); + // === Wait for future to be dropped === + assert!( + rt.block_on(wait_future_drop).is_ok(), + "The future should always be dropped." + ); + + if abort == CombiAbort::AbortedAfterFinish { + // Don't set aborted to true here as the task already finished + handle.as_mut().unwrap().abort(); + } + if jh == CombiJoinHandle::DropAfterNoConsume { + // The runtime will usually have dropped every ref-count at this point, + // in which case dropping the JoinHandle drops the output. + // + // (But it might race and still hold a ref-count) + let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + drop(handle.take().unwrap()); + })); + if panic.is_err() { + assert!( + (output == CombiOutput::PanicOnDrop) + && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) + && !aborted, + "Dropping JoinHandle shouldn't panic here" + ); + } + } + + // Check whether we drop after consuming the output + if jh == CombiJoinHandle::DropAfterConsume { + // Using as_mut() to not immediately drop the handle + let result = rt.block_on(handle.as_mut().unwrap()); + + match result { + Ok(mut output) => { + // Don't panic here. + output.disarm(); + assert!(!aborted, "Task was aborted but returned output"); + } + Err(err) if err.is_cancelled() => assert!(aborted, "Cancelled output but not aborted"), + Err(err) if err.is_panic() => { + assert!( + (task == CombiTask::PanicOnRun) + || (task == CombiTask::PanicOnDrop) + || (task == CombiTask::PanicOnRunAndDrop) + || (output == CombiOutput::PanicOnDrop), + "Panic but nothing should panic" + ); + } + _ => unreachable!(), + } + + let handle = handle.take().unwrap(); + if abort == CombiAbort::AbortedAfterConsumeOutput { + handle.abort(); + } + drop(handle); + } + + // The output should have been dropped now. Check whether the output + // object was created at all. + let output_created = rt.block_on(wait_output_drop).is_ok(); + assert_eq!( + output_created, + (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) && !aborted, + "Creation of output object" + ); +} diff --git a/tokio/tests/task_abort.rs b/tokio/tests/task_abort.rs index 8f621683f..cdaa405b8 100644 --- a/tokio/tests/task_abort.rs +++ b/tokio/tests/task_abort.rs @@ -5,11 +5,21 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; +use tokio::runtime::Builder; + +struct PanicOnDrop; + +impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("Well what did you expect would happen..."); + } +} + /// Checks that a suspended task can be aborted without panicking as reported in /// issue #3157: . #[test] fn test_abort_without_panic_3157() { - let rt = tokio::runtime::Builder::new_multi_thread() + let rt = Builder::new_multi_thread() .enable_time() .worker_threads(1) .build() @@ -45,9 +55,7 @@ fn test_abort_without_panic_3662() { } } - let rt = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); + let rt = Builder::new_current_thread().build().unwrap(); rt.block_on(async move { let drop_flag = Arc::new(AtomicBool::new(false)); @@ -120,9 +128,7 @@ fn remote_abort_local_set_3929() { } } - let rt = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); + let rt = Builder::new_current_thread().build().unwrap(); let local = tokio::task::LocalSet::new(); let check = DropCheck::new(); @@ -144,10 +150,7 @@ fn remote_abort_local_set_3929() { /// issue #3964: . #[test] fn test_abort_wakes_task_3964() { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); + let rt = Builder::new_current_thread().enable_time().build().unwrap(); rt.block_on(async move { let notify_dropped = Arc::new(()); @@ -174,22 +177,11 @@ fn test_abort_wakes_task_3964() { }); } -struct PanicOnDrop; - -impl Drop for PanicOnDrop { - fn drop(&mut self) { - panic!("Well what did you expect would happen..."); - } -} - /// Checks that aborting a task whose destructor panics does not allow the /// panic to escape the task. #[test] fn test_abort_task_that_panics_on_drop_contained() { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); + let rt = Builder::new_current_thread().enable_time().build().unwrap(); rt.block_on(async move { let handle = tokio::spawn(async move { @@ -213,10 +205,7 @@ fn test_abort_task_that_panics_on_drop_contained() { /// Checks that aborting a task whose destructor panics has the expected result. #[test] fn test_abort_task_that_panics_on_drop_returned() { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); + let rt = Builder::new_current_thread().enable_time().build().unwrap(); rt.block_on(async move { let handle = tokio::spawn(async move { From 998dc5a2eb48bd0937b10bf0638b2f8346b5f9f5 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 22 Jul 2021 12:05:02 +0200 Subject: [PATCH 30/35] task: fix leak in LocalSet (#3978) --- tokio/src/runtime/tests/loom_local.rs | 47 +++++++++++++++++++++++++++ tokio/src/runtime/tests/mod.rs | 1 + tokio/src/task/local.rs | 33 +++++++++++++++---- 3 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 tokio/src/runtime/tests/loom_local.rs diff --git a/tokio/src/runtime/tests/loom_local.rs b/tokio/src/runtime/tests/loom_local.rs new file mode 100644 index 000000000..d9a07a45f --- /dev/null +++ b/tokio/src/runtime/tests/loom_local.rs @@ -0,0 +1,47 @@ +use crate::runtime::tests::loom_oneshot as oneshot; +use crate::runtime::Builder; +use crate::task::LocalSet; + +use std::task::Poll; + +/// Waking a runtime will attempt to push a task into a queue of notifications +/// in the runtime, however the tasks in such a queue usually have a reference +/// to the runtime itself. This means that if they are not properly removed at +/// runtime shutdown, this will cause a memory leak. +/// +/// This test verifies that waking something during shutdown of a LocalSet does +/// not result in tasks lingering in the queue once shutdown is complete. This +/// is verified using loom's leak finder. +#[test] +fn wake_during_shutdown() { + loom::model(|| { + let rt = Builder::new_current_thread().build().unwrap(); + let ls = LocalSet::new(); + + let (send, recv) = oneshot::channel(); + + ls.spawn_local(async move { + let mut send = Some(send); + + let () = futures::future::poll_fn(|cx| { + if let Some(send) = send.take() { + send.send(cx.waker().clone()); + } + + Poll::Pending + }) + .await; + }); + + let handle = loom::thread::spawn(move || { + let waker = recv.recv(); + waker.wake(); + }); + + ls.block_on(&rt, crate::task::yield_now()); + + drop(ls); + handle.join().unwrap(); + drop(rt); + }); +} diff --git a/tokio/src/runtime/tests/mod.rs b/tokio/src/runtime/tests/mod.rs index e0379a68c..48919fa71 100644 --- a/tokio/src/runtime/tests/mod.rs +++ b/tokio/src/runtime/tests/mod.rs @@ -30,6 +30,7 @@ mod unowned_wrapper { cfg_loom! { mod loom_basic_scheduler; + mod loom_local; mod loom_blocking; mod loom_oneshot; mod loom_pool; diff --git a/tokio/src/task/local.rs b/tokio/src/task/local.rs index c74c8b438..f05e0ebaf 100644 --- a/tokio/src/task/local.rs +++ b/tokio/src/task/local.rs @@ -241,7 +241,7 @@ struct Tasks { /// LocalSet state shared between threads. struct Shared { /// Remote run queue sender - queue: Mutex>>>, + queue: Mutex>>>>, /// Wake the `LocalSet` task waker: AtomicWaker, @@ -339,7 +339,7 @@ impl LocalSet { queue: VecDeque::with_capacity(INITIAL_CAPACITY), }), shared: Arc::new(Shared { - queue: Mutex::new(VecDeque::with_capacity(INITIAL_CAPACITY)), + queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), waker: AtomicWaker::new(), }), }, @@ -549,7 +549,8 @@ impl LocalSet { .shared .queue .lock() - .pop_front() + .as_mut() + .and_then(|queue| queue.pop_front()) .or_else(|| self.context.tasks.borrow_mut().queue.pop_front()) } else { self.context @@ -557,7 +558,14 @@ impl LocalSet { .borrow_mut() .queue .pop_front() - .or_else(|| self.context.shared.queue.lock().pop_front()) + .or_else(|| { + self.context + .shared + .queue + .lock() + .as_mut() + .and_then(|queue| queue.pop_front()) + }) } } @@ -627,7 +635,10 @@ impl Drop for LocalSet { task.shutdown(); } - for task in self.context.shared.queue.lock().drain(..) { + // Take the queue from the Shared object to prevent pushing + // notifications to it in the future. + let queue = self.context.shared.queue.lock().take().unwrap(); + for task in queue { task.shutdown(); } @@ -677,8 +688,16 @@ impl Shared { cx.tasks.borrow_mut().queue.push_back(task); } _ => { - self.queue.lock().push_back(task); - self.waker.wake(); + // First check whether the queue is still there (if not, the + // LocalSet is dropped). Then push to it if so, and if not, + // do nothing. + let mut lock = self.queue.lock(); + + if let Some(queue) = lock.as_mut() { + queue.push_back(task); + drop(lock); + self.waker.wake(); + } } }); } From b280c6dcd72b2607cde68d2b2192480932f48ecb Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 22 Jul 2021 12:05:39 +0200 Subject: [PATCH 31/35] chore: prepare Tokio v1.9.0 (#3961) --- README.md | 2 +- tokio/CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ tokio/Cargo.toml | 4 ++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ddad6d312..75c64058f 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml: ```toml [dependencies] -tokio = { version = "1.8.0", features = ["full"] } +tokio = { version = "1.9.0", features = ["full"] } ``` Then, on your main.rs: diff --git a/tokio/CHANGELOG.md b/tokio/CHANGELOG.md index a96d27500..5de80a3f9 100644 --- a/tokio/CHANGELOG.md +++ b/tokio/CHANGELOG.md @@ -1,3 +1,37 @@ +# 1.9.0 (July 22, 2021) + +### Added + + - net: allow customized I/O operations for `TcpStream` ([#3888]) + - sync: add getter for the mutex from a guard ([#3928]) + - task: expose nameable future for `TaskLocal::scope` ([#3273]) + +### Fixed + + - Fix leak if output of future panics on drop ([#3967]) + - Fix leak in `LocalSet` ([#3978]) + +### Changes + + - runtime: reorganize parts of the runtime ([#3909], [#3939], [#3950], [#3955], [#3980]) + - sync: clean up `OnceCell` ([#3945]) + - task: remove mutex in `JoinError` ([#3959]) + +[#3273]: https://github.com/tokio-rs/tokio/pull/3273 +[#3888]: https://github.com/tokio-rs/tokio/pull/3888 +[#3909]: https://github.com/tokio-rs/tokio/pull/3909 +[#3928]: https://github.com/tokio-rs/tokio/pull/3928 +[#3934]: https://github.com/tokio-rs/tokio/pull/3934 +[#3939]: https://github.com/tokio-rs/tokio/pull/3939 +[#3945]: https://github.com/tokio-rs/tokio/pull/3945 +[#3950]: https://github.com/tokio-rs/tokio/pull/3950 +[#3955]: https://github.com/tokio-rs/tokio/pull/3955 +[#3959]: https://github.com/tokio-rs/tokio/pull/3959 +[#3967]: https://github.com/tokio-rs/tokio/pull/3967 +[#3967]: https://github.com/tokio-rs/tokio/pull/3967 +[#3978]: https://github.com/tokio-rs/tokio/pull/3978 +[#3980]: https://github.com/tokio-rs/tokio/pull/3980 + # 1.8.2 (July 19, 2021) Fixes a missed edge case from 1.8.1. diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index a30ba23a1..8ead829d6 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -7,12 +7,12 @@ name = "tokio" # - README.md # - Update CHANGELOG.md. # - Create "v1.0.x" git tag. -version = "1.8.2" +version = "1.9.0" edition = "2018" authors = ["Tokio Contributors "] license = "MIT" readme = "README.md" -documentation = "https://docs.rs/tokio/1.8.2/tokio/" +documentation = "https://docs.rs/tokio/1.9.0/tokio/" repository = "https://github.com/tokio-rs/tokio" homepage = "https://tokio.rs" description = """ From df10b68d47631c3053342c7cf0b1fab8786565b2 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 22 Jul 2021 13:53:43 +0200 Subject: [PATCH 32/35] readme: add release schedule and bugfix policy (#3948) --- README.md | 35 ++++++++++++++++++++++++++++++----- tokio/README.md | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 75c64058f..e382f69d1 100644 --- a/README.md +++ b/README.md @@ -140,8 +140,7 @@ several other libraries, including: * [`tower`]: A library of modular and reusable components for building robust networking clients and servers. -* [`tracing`] (formerly `tokio-trace`): A framework for application-level - tracing and async-aware diagnostics. +* [`tracing`] (formerly `tokio-trace`): A framework for application-level tracing and async-aware diagnostics. * [`rdbc`]: A Rust database connectivity library for MySQL, Postgres and SQLite. @@ -164,9 +163,35 @@ several other libraries, including: ## Supported Rust Versions -Tokio is built against the latest stable release. The minimum supported version is 1.45. -The current Tokio version is not guaranteed to build on Rust versions earlier than the -minimum supported version. +Tokio is built against the latest stable release. The minimum supported version +is 1.45. The current Tokio version is not guaranteed to build on Rust versions +earlier than the minimum supported version. + +## Release schedule + +Tokio doesn't follow a fixed release schedule, but we typically make one to two +new minor releases each month. We make patch releases for bugfixes as necessary. + +## Bug patching policy + +For the purposes of making patch releases with bugfixes, we have designated +certain minor releases as LTS (long term support) releases. Whenever a bug +warrants a patch release with a fix for the bug, it will be backported and +released as a new patch release for each LTS minor version. Our current LTS +releases are: + + * `1.8.x` - LTS release until February 2022. + +Each LTS release will continue to receive backported fixes for at least half a +year. If you wish to use a fixed minor release in your project, we recommend +that you use an LTS release. + +To use a fixed minor version, you can specify the version with a tilde. For +example, to specify that you wish to use the newest `1.8.x` patch release, you +can use the following dependency specification: +```text +tokio = { version = "~1.8", features = [...] } +``` ## License diff --git a/tokio/README.md b/tokio/README.md index 8ee7bbda1..094f5f000 100644 --- a/tokio/README.md +++ b/tokio/README.md @@ -50,7 +50,15 @@ an asynchronous application. ## Example -A basic TCP echo server with Tokio: +A basic TCP echo server with Tokio. + +Make sure you activated the full features of the tokio crate on Cargo.toml: + +```toml +[dependencies] +tokio = { version = "1.8.0", features = ["full"] } +``` +Then, on your main.rs: ```rust,no_run use tokio::net::TcpListener; @@ -58,7 +66,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[tokio::main] async fn main() -> Result<(), Box> { - let mut listener = TcpListener::bind("127.0.0.1:8080").await?; + let listener = TcpListener::bind("127.0.0.1:8080").await?; loop { let (mut socket, _) = listener.accept().await?; @@ -132,7 +140,7 @@ several other libraries, including: * [`tower`]: A library of modular and reusable components for building robust networking clients and servers. -* [`tracing`]: A framework for application-level tracing and async-aware diagnostics. +* [`tracing`] (formerly `tokio-trace`): A framework for application-level tracing and async-aware diagnostics. * [`rdbc`]: A Rust database connectivity library for MySQL, Postgres and SQLite. @@ -155,9 +163,35 @@ several other libraries, including: ## Supported Rust Versions -Tokio is built against the latest stable release. The minimum supported version is 1.45. -The current Tokio version is not guaranteed to build on Rust versions earlier than the -minimum supported version. +Tokio is built against the latest stable release. The minimum supported version +is 1.45. The current Tokio version is not guaranteed to build on Rust versions +earlier than the minimum supported version. + +## Release schedule + +Tokio doesn't follow a fixed release schedule, but we typically make one to two +new minor releases each month. We make patch releases for bugfixes as necessary. + +## Bug patching policy + +For the purposes of making patch releases with bugfixes, we have designated +certain minor releases as LTS (long term support) releases. Whenever a bug +warrants a patch release with a fix for the bug, it will be backported and +released as a new patch release for each LTS minor version. Our current LTS +releases are: + + * `1.8.x` - LTS release until February 2022. + +Each LTS release will continue to receive backported fixes for at least half a +year. If you wish to use a fixed minor release in your project, we recommend +that you use an LTS release. + +To use a fixed minor version, you can specify the version with a tilde. For +example, to specify that you wish to use the newest `1.8.x` patch release, you +can use the following dependency specification: +```text +tokio = { version = "~1.8", features = [...] } +``` ## License From c85a0e524e171531770cdb04521a61033747b3c3 Mon Sep 17 00:00:00 2001 From: LinkTed Date: Mon, 26 Jul 2021 12:43:55 +0300 Subject: [PATCH 33/35] process: add arg0 method to Command (#3984) --- tokio/src/process/mod.rs | 18 ++++++++++++++++++ tokio/tests/process_arg0.rs | 13 +++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tokio/tests/process_arg0.rs diff --git a/tokio/src/process/mod.rs b/tokio/src/process/mod.rs index 96ceb6d8d..7ae503f51 100644 --- a/tokio/src/process/mod.rs +++ b/tokio/src/process/mod.rs @@ -551,6 +551,7 @@ impl Command { /// /// [1]: https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863(v=vs.85).aspx #[cfg(windows)] + #[cfg_attr(docsrs, doc(cfg(windows)))] pub fn creation_flags(&mut self, flags: u32) -> &mut Command { self.std.creation_flags(flags); self @@ -560,6 +561,7 @@ impl Command { /// `setuid` call in the child process. Failure in the `setuid` /// call will cause the spawn to fail. #[cfg(unix)] + #[cfg_attr(docsrs, doc(cfg(unix)))] pub fn uid(&mut self, id: u32) -> &mut Command { self.std.uid(id); self @@ -568,11 +570,26 @@ impl Command { /// Similar to `uid` but sets the group ID of the child process. This has /// the same semantics as the `uid` field. #[cfg(unix)] + #[cfg_attr(docsrs, doc(cfg(unix)))] pub fn gid(&mut self, id: u32) -> &mut Command { self.std.gid(id); self } + /// Set executable argument + /// + /// Set the first process argument, `argv[0]`, to something other than the + /// default executable path. + #[cfg(unix)] + #[cfg_attr(docsrs, doc(cfg(unix)))] + pub fn arg0(&mut self, arg: S) -> &mut Command + where + S: AsRef, + { + self.std.arg0(arg); + self + } + /// Schedules a closure to be run just before the `exec` function is /// invoked. /// @@ -603,6 +620,7 @@ impl Command { /// working directory have successfully been changed, so output to these /// locations may not appear where intended. #[cfg(unix)] + #[cfg_attr(docsrs, doc(cfg(unix)))] pub unsafe fn pre_exec(&mut self, f: F) -> &mut Command where F: FnMut() -> io::Result<()> + Send + Sync + 'static, diff --git a/tokio/tests/process_arg0.rs b/tokio/tests/process_arg0.rs new file mode 100644 index 000000000..4fabea0fe --- /dev/null +++ b/tokio/tests/process_arg0.rs @@ -0,0 +1,13 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", unix))] + +use tokio::process::Command; + +#[tokio::test] +async fn arg0() { + let mut cmd = Command::new("sh"); + cmd.arg0("test_string").arg("-c").arg("echo $0"); + + let output = cmd.output().await.unwrap(); + assert_eq!(output.stdout, b"test_string\n"); +} From 8f27c04a9e923f815f1cd14feae3d8cad6018b22 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 26 Jul 2021 12:32:24 +0200 Subject: [PATCH 34/35] codec: remove unnecessary doc(cfg(...)) (#3989) --- tokio-util/src/udp/frame.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tokio-util/src/udp/frame.rs b/tokio-util/src/udp/frame.rs index cbf0f6355..65a5af2df 100644 --- a/tokio-util/src/udp/frame.rs +++ b/tokio-util/src/udp/frame.rs @@ -35,7 +35,6 @@ use std::{io, mem::MaybeUninit}; /// [`Sink`]: futures_sink::Sink /// [`split`]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html#method.split #[must_use = "sinks do nothing unless polled"] -#[cfg_attr(docsrs, doc(cfg(all(feature = "codec", feature = "udp"))))] #[derive(Debug)] pub struct UdpFramed { socket: T, From 2b731c0e01756d695af4d2a77f372a7e6874761e Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 26 Jul 2021 15:37:17 +0200 Subject: [PATCH 35/35] task: elaborate on queue behavior of spawn_blocking (#3981) --- tokio/src/task/blocking.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tokio/src/task/blocking.rs b/tokio/src/task/blocking.rs index e4fe254a0..806dbbdd0 100644 --- a/tokio/src/task/blocking.rs +++ b/tokio/src/task/blocking.rs @@ -89,13 +89,14 @@ cfg_rt! { /// /// Tokio will spawn more blocking threads when they are requested through this /// function until the upper limit configured on the [`Builder`] is reached. - /// This limit is very large by default, because `spawn_blocking` is often used - /// for various kinds of IO operations that cannot be performed asynchronously. - /// When you run CPU-bound code using `spawn_blocking`, you should keep this - /// large upper limit in mind. When running many CPU-bound computations, a - /// semaphore or some other synchronization primitive should be used to limit - /// the number of computation executed in parallel. Specialized CPU-bound - /// executors, such as [rayon], may also be a good fit. + /// After reaching the upper limit, the tasks are put in a queue. + /// The thread limit is very large by default, because `spawn_blocking` is often + /// used for various kinds of IO operations that cannot be performed + /// asynchronously. When you run CPU-bound code using `spawn_blocking`, you + /// should keep this large upper limit in mind. When running many CPU-bound + /// computations, a semaphore or some other synchronization primitive should be + /// used to limit the number of computation executed in parallel. Specialized + /// CPU-bound executors, such as [rayon], may also be a good fit. /// /// This function is intended for non-async operations that eventually finish on /// their own. If you want to spawn an ordinary thread, you should use