From a2941e48beb7d34ae8ed6dc4618fd3b27ea23547 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 8 Jun 2023 10:36:25 +0200 Subject: [PATCH 01/13] ci: temporarily disable semver check (#5774) --- .github/workflows/ci.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46b0faf67..3958755d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,6 @@ jobs: - test-unstable - miri - asan - - semver - cross-check - cross-test - no-atomic-u64 @@ -300,17 +299,19 @@ jobs: # Ignore `trybuild` errors as they are irrelevant and flaky on nightly TRYBUILD: overwrite - semver: - name: semver - needs: basics - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Check semver - uses: obi1kenobi/cargo-semver-checks-action@v2 - with: - rust-toolchain: ${{ env.rust_stable }} - release-type: minor + # Re-enable this after the next release. + # + #semver: + # name: semver + # needs: basics + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Check semver + # uses: obi1kenobi/cargo-semver-checks-action@v2 + # with: + # rust-toolchain: ${{ env.rust_stable }} + # release-type: minor cross-check: name: cross-check From e63d0f10bf614dbd7b85d04a6e01bf8378b5194a Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Sat, 10 Jun 2023 18:38:52 +0800 Subject: [PATCH 02/13] task: use pin-project for `TaskLocalFuture` (#5758) Signed-off-by: Bugen Zhao --- tokio-stream/Cargo.toml | 2 +- tokio-util/Cargo.toml | 2 +- tokio/Cargo.toml | 2 +- tokio/src/task/task_local.rs | 119 ++++++++++++++++++----------------- 4 files changed, 64 insertions(+), 61 deletions(-) diff --git a/tokio-stream/Cargo.toml b/tokio-stream/Cargo.toml index 9a90cd32c..e937ef901 100644 --- a/tokio-stream/Cargo.toml +++ b/tokio-stream/Cargo.toml @@ -37,7 +37,7 @@ signal = ["tokio/signal"] [dependencies] futures-core = { version = "0.3.0" } -pin-project-lite = "0.2.0" +pin-project-lite = "0.2.7" tokio = { version = "1.15.0", path = "../tokio", features = ["sync"] } tokio-util = { version = "0.7.0", path = "../tokio-util", optional = true } diff --git a/tokio-util/Cargo.toml b/tokio-util/Cargo.toml index b6ae0166a..4b406d818 100644 --- a/tokio-util/Cargo.toml +++ b/tokio-util/Cargo.toml @@ -40,7 +40,7 @@ futures-core = "0.3.0" futures-sink = "0.3.0" futures-io = { version = "0.3.0", optional = true } futures-util = { version = "0.3.0", optional = true } -pin-project-lite = "0.2.0" +pin-project-lite = "0.2.7" slab = { version = "0.4.4", optional = true } # Backs `DelayQueue` tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true } diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index 9656f0a4f..da7988ce1 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -99,7 +99,7 @@ autocfg = "1.1" [dependencies] tokio-macros = { version = "~2.1.0", path = "../tokio-macros", optional = true } -pin-project-lite = "0.2.0" +pin-project-lite = "0.2.7" # Everything else is optional... bytes = { version = "1.0.0", optional = true } diff --git a/tokio/src/task/task_local.rs b/tokio/src/task/task_local.rs index d3b108fe6..eeadfbd3e 100644 --- a/tokio/src/task/task_local.rs +++ b/tokio/src/task/task_local.rs @@ -1,3 +1,4 @@ +use pin_project_lite::pin_project; use std::cell::RefCell; use std::error::Error; use std::future::Future; @@ -299,36 +300,53 @@ impl fmt::Debug for LocalKey { } } -/// A future that sets a value `T` of a task local for the future `F` during -/// its execution. -/// -/// The value of the task-local must be `'static` and will be dropped on the -/// completion of the future. -/// -/// Created by the function [`LocalKey::scope`](self::LocalKey::scope). -/// -/// ### Examples -/// -/// ``` -/// # async fn dox() { -/// tokio::task_local! { -/// static NUMBER: u32; -/// } -/// -/// NUMBER.scope(1, async move { -/// println!("task local value: {}", NUMBER.get()); -/// }).await; -/// # } -/// ``` -// Doesn't use pin_project due to custom Drop. -pub struct TaskLocalFuture -where - T: 'static, -{ - local: &'static LocalKey, - slot: Option, - future: Option, - _pinned: PhantomPinned, +pin_project! { + /// A future that sets a value `T` of a task local for the future `F` during + /// its execution. + /// + /// The value of the task-local must be `'static` and will be dropped on the + /// completion of the future. + /// + /// Created by the function [`LocalKey::scope`](self::LocalKey::scope). + /// + /// ### Examples + /// + /// ``` + /// # async fn dox() { + /// tokio::task_local! { + /// static NUMBER: u32; + /// } + /// + /// NUMBER.scope(1, async move { + /// println!("task local value: {}", NUMBER.get()); + /// }).await; + /// # } + /// ``` + pub struct TaskLocalFuture + where + T: 'static, + { + local: &'static LocalKey, + slot: Option, + #[pin] + future: Option, + #[pin] + _pinned: PhantomPinned, + } + + impl PinnedDrop for TaskLocalFuture { + fn drop(this: Pin<&mut Self>) { + let this = this.project(); + if mem::needs_drop::() && this.future.is_some() { + // Drop the future while the task-local is set, if possible. Otherwise + // the future is dropped normally when the `Option` field drops. + let mut future = this.future; + let _ = this.local.scope_inner(this.slot, || { + future.set(None); + }); + } + } + } } impl Future for TaskLocalFuture { @@ -336,23 +354,21 @@ impl Future for TaskLocalFuture { #[track_caller] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // safety: The TaskLocalFuture struct is `!Unpin` so there is no way to - // move `self.future` from now on. - let this = unsafe { Pin::into_inner_unchecked(self) }; - let mut future_opt = unsafe { Pin::new_unchecked(&mut this.future) }; + let this = self.project(); + let mut future_opt = this.future; - let res = - this.local - .scope_inner(&mut this.slot, || match future_opt.as_mut().as_pin_mut() { - Some(fut) => { - let res = fut.poll(cx); - if res.is_ready() { - future_opt.set(None); - } - Some(res) + let res = this + .local + .scope_inner(this.slot, || match future_opt.as_mut().as_pin_mut() { + Some(fut) => { + let res = fut.poll(cx); + if res.is_ready() { + future_opt.set(None); } - None => None, - }); + Some(res) + } + None => None, + }); match res { Ok(Some(res)) => res, @@ -362,19 +378,6 @@ impl Future for TaskLocalFuture { } } -impl Drop for TaskLocalFuture { - fn drop(&mut self) { - if mem::needs_drop::() && self.future.is_some() { - // Drop the future while the task-local is set, if possible. Otherwise - // the future is dropped normally when the `Option` field drops. - let future = &mut self.future; - let _ = self.local.scope_inner(&mut self.slot, || { - *future = None; - }); - } - } -} - impl fmt::Debug for TaskLocalFuture where T: fmt::Debug, From 7ccd3e0c6d0d1341bee4dd136eef38092e1aad11 Mon Sep 17 00:00:00 2001 From: nvartolomei Date: Sat, 10 Jun 2023 12:19:07 +0100 Subject: [PATCH 03/13] task: add `JoinSet::poll_join_next` (#5721) --- tokio/src/task/join_set.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tokio/src/task/join_set.rs b/tokio/src/task/join_set.rs index 041b06cb6..4eb15a24d 100644 --- a/tokio/src/task/join_set.rs +++ b/tokio/src/task/join_set.rs @@ -362,7 +362,7 @@ impl JoinSet { /// This can happen if the [coop budget] is reached. /// /// [coop budget]: crate::task#cooperative-scheduling - fn poll_join_next(&mut self, cx: &mut Context<'_>) -> Poll>> { + pub fn poll_join_next(&mut self, cx: &mut Context<'_>) -> Poll>> { // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to // the `notified` list if the waker is notified in the `poll` call below. let mut entry = match self.inner.pop_notified(cx.waker()) { @@ -419,7 +419,8 @@ impl JoinSet { /// [coop budget]: crate::task#cooperative-scheduling /// [task ID]: crate::task::Id #[cfg(tokio_unstable)] - fn poll_join_next_with_id( + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn poll_join_next_with_id( &mut self, cx: &mut Context<'_>, ) -> Poll>> { From cb18b0a231828a3e65d3f64c6d2744751c6b30ed Mon Sep 17 00:00:00 2001 From: Jack Wrenn Date: Sat, 10 Jun 2023 07:30:08 -0400 Subject: [PATCH 04/13] tokio: improve task dump documentation (#5778) Adds depth to the taskdump example, and documentation to Handle::dump. --- examples/dump.rs | 76 ++++++++++++++++++++++++++----------- tokio/src/runtime/dump.rs | 10 +++++ tokio/src/runtime/handle.rs | 72 ++++++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 23 deletions(-) diff --git a/examples/dump.rs b/examples/dump.rs index c73584899..4d8ff19c0 100644 --- a/examples/dump.rs +++ b/examples/dump.rs @@ -1,4 +1,6 @@ -//! This example demonstrates tokio's experimental taskdumping functionality. +//! This example demonstrates tokio's experimental task dumping functionality. +//! This application deadlocks. Input CTRL+C to display traces of each task, or +//! input CTRL+C twice within 1 second to quit. #[cfg(all( tokio_unstable, @@ -7,44 +9,74 @@ any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") ))] #[tokio::main] -async fn main() { - use std::hint::black_box; +async fn main() -> Result<(), Box> { + use std::sync::Arc; + use tokio::sync::Barrier; #[inline(never)] - async fn a() { - black_box(b()).await + async fn a(barrier: Arc) { + b(barrier).await } #[inline(never)] - async fn b() { - black_box(c()).await + async fn b(barrier: Arc) { + c(barrier).await } #[inline(never)] - async fn c() { - loop { - tokio::task::yield_now().await; - } + async fn c(barrier: Arc) { + barrier.wait().await; } - async fn dump() { + // Prints a task dump upon receipt of CTRL+C, or returns if CTRL+C is + // inputted twice within a second. + async fn dump_or_quit() { + use tokio::time::{timeout, Duration, Instant}; let handle = tokio::runtime::Handle::current(); - let dump = handle.dump().await; + let mut last_signal: Option = None; + // wait for CTRL+C + while let Ok(_) = tokio::signal::ctrl_c().await { + // exit if a CTRL+C is inputted twice within 1 second + if let Some(time_since_last_signal) = last_signal.map(|i| i.elapsed()) { + if time_since_last_signal < Duration::from_secs(1) { + return; + } + } + last_signal = Some(Instant::now()); - for (i, task) in dump.tasks().iter().enumerate() { - let trace = task.trace(); - println!("task {i} trace:"); - println!("{trace}\n"); + // capture a dump, and print each trace + println!("{:-<80}", ""); + if let Ok(dump) = timeout(Duration::from_secs(2), handle.dump()).await { + for (i, task) in dump.tasks().iter().enumerate() { + let trace = task.trace(); + println!("TASK {i}:"); + println!("{trace}\n"); + } + } else { + println!("Task dumping timed out. Use a native debugger (like gdb) to debug the deadlock."); + } + println!("{:-<80}", ""); + println!("Input CTRL+C twice within 1 second to exit."); } } + println!("This program has a deadlock."); + println!("Input CTRL+C to print a task dump."); + println!("Input CTRL+C twice within 1 second to exit."); + + // oops! this barrier waits for one more task than will ever come. + let barrier = Arc::new(Barrier::new(3)); + + let task_1 = tokio::spawn(a(barrier.clone())); + let task_2 = tokio::spawn(a(barrier)); + tokio::select!( - biased; - _ = tokio::spawn(a()) => {}, - _ = tokio::spawn(b()) => {}, - _ = tokio::spawn(c()) => {}, - _ = dump() => {}, + _ = dump_or_quit() => {}, + _ = task_1 => {}, + _ = task_2 => {}, ); + + Ok(()) } #[cfg(not(all( diff --git a/tokio/src/runtime/dump.rs b/tokio/src/runtime/dump.rs index 846839e4d..994b7f9c0 100644 --- a/tokio/src/runtime/dump.rs +++ b/tokio/src/runtime/dump.rs @@ -1,26 +1,36 @@ //! Snapshots of runtime state. +//! +//! See [Handle::dump][crate::runtime::Handle::dump]. use std::fmt; /// A snapshot of a runtime's state. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. #[derive(Debug)] pub struct Dump { tasks: Tasks, } /// Snapshots of tasks. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. #[derive(Debug)] pub struct Tasks { tasks: Vec, } /// A snapshot of a task. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. #[derive(Debug)] pub struct Task { trace: Trace, } /// An execution trace of a task's last poll. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. #[derive(Debug)] pub struct Trace { inner: super::task::trace::Trace, diff --git a/tokio/src/runtime/handle.rs b/tokio/src/runtime/handle.rs index 0951d8a37..02ba279ff 100644 --- a/tokio/src/runtime/handle.rs +++ b/tokio/src/runtime/handle.rs @@ -373,7 +373,77 @@ cfg_metrics! { cfg_taskdump! { impl Handle { - /// Capture a snapshot of this runtime's state. + /// Captures a snapshot of the runtime's state. + /// + /// This functionality is experimental, and comes with a number of + /// requirements and limitations. + /// + /// # Examples + /// + /// This can be used to get call traces of each task in the runtime. + /// Calls to `Handle::dump` should usually be enclosed in a + /// [timeout][crate::time::timeout], so that dumping does not escalate a + /// single blocked runtime thread into an entirely blocked runtime. + /// + /// ``` + /// # use tokio::runtime::Runtime; + /// # fn dox() { + /// # let rt = Runtime::new().unwrap(); + /// # rt.spawn(async { + /// use tokio::runtime::Handle; + /// use tokio::time::{timeout, Duration}; + /// + /// // Inside an async block or function. + /// let handle = Handle::current(); + /// if let Ok(dump) = timeout(Duration::from_secs(2), handle.dump()).await { + /// for (i, task) in dump.tasks().iter().enumerate() { + /// let trace = task.trace(); + /// println!("TASK {i}:"); + /// println!("{trace}\n"); + /// } + /// } + /// # }); + /// # } + /// ``` + /// + /// # Requirements + /// + /// ## Debug Info Must Be Available + /// To produce task traces, the application must **not** be compiled + /// with split debuginfo. On Linux, including debuginfo within the + /// application binary is the (correct) default. You can further ensure + /// this behavior with the following directive in your `Cargo.toml`: + /// + /// ```toml + /// [profile.*] + /// split-debuginfo = "off" + /// ``` + /// + /// ## Platform Requirements + /// + /// Task dumps are supported on Linux atop x86 and x86_64. + /// + /// ## Current Thread Runtime Requirements + /// + /// On the `current_thread` runtime, task dumps may only be requested + /// from *within* the context of the runtime being dumped. Do not, for + /// example, await `Handle::dump()` on a different runtime. + /// + /// # Limitations + /// + /// ## Local Executors + /// + /// Tasks managed by local executors (e.g., `FuturesUnordered` and + /// [`LocalSet`][crate::task::LocalSet]) may not appear in task dumps. + /// + /// ## Non-Termination When Workers Are Blocked + /// + /// The future produced by `Handle::dump` may never produce `Ready` if + /// another runtime worker is blocked for more than 250ms. This may + /// occur if a dump is requested during shutdown, or if another runtime + /// worker is infinite looping or synchronously deadlocked. For these + /// reasons, task dumping should usually be paired with an explicit + /// [timeout][crate::time::timeout]. pub async fn dump(&self) -> crate::runtime::Dump { match &self.inner { scheduler::Handle::CurrentThread(handle) => handle.dump(), From 2a54ad01d0945c851a093849c83019de69e98dee Mon Sep 17 00:00:00 2001 From: Erk Date: Sat, 10 Jun 2023 14:24:19 +0200 Subject: [PATCH 05/13] time: do not overflow to signal value (#5710) --- tokio/src/runtime/time/entry.rs | 6 +++++- tokio/src/runtime/time/mod.rs | 2 +- tokio/src/runtime/time/source.rs | 3 ++- tokio/tests/time_sleep.rs | 14 ++++++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/time/entry.rs b/tokio/src/runtime/time/entry.rs index 6aea2b15c..798d3c11e 100644 --- a/tokio/src/runtime/time/entry.rs +++ b/tokio/src/runtime/time/entry.rs @@ -72,6 +72,10 @@ type TimerResult = Result<(), crate::time::error::Error>; const STATE_DEREGISTERED: u64 = u64::MAX; const STATE_PENDING_FIRE: u64 = STATE_DEREGISTERED - 1; const STATE_MIN_VALUE: u64 = STATE_PENDING_FIRE; +/// The largest safe integer to use for ticks. +/// +/// This value should be updated if any other signal values are added above. +pub(super) const MAX_SAFE_MILLIS_DURATION: u64 = u64::MAX - 2; /// This structure holds the current shared state of the timer - its scheduled /// time (if registered), or otherwise the result of the timer completing, as @@ -126,7 +130,7 @@ impl StateCell { fn when(&self) -> Option { let cur_state = self.state.load(Ordering::Relaxed); - if cur_state == u64::MAX { + if cur_state == STATE_DEREGISTERED { None } else { Some(cur_state) diff --git a/tokio/src/runtime/time/mod.rs b/tokio/src/runtime/time/mod.rs index 215714dd5..423ad79ab 100644 --- a/tokio/src/runtime/time/mod.rs +++ b/tokio/src/runtime/time/mod.rs @@ -8,7 +8,7 @@ mod entry; pub(crate) use entry::TimerEntry; -use entry::{EntryList, TimerHandle, TimerShared}; +use entry::{EntryList, TimerHandle, TimerShared, MAX_SAFE_MILLIS_DURATION}; mod handle; pub(crate) use self::handle::Handle; diff --git a/tokio/src/runtime/time/source.rs b/tokio/src/runtime/time/source.rs index 412812da1..4647bc412 100644 --- a/tokio/src/runtime/time/source.rs +++ b/tokio/src/runtime/time/source.rs @@ -1,3 +1,4 @@ +use super::MAX_SAFE_MILLIS_DURATION; use crate::time::{Clock, Duration, Instant}; /// A structure which handles conversion from Instants to u64 timestamps. @@ -25,7 +26,7 @@ impl TimeSource { .unwrap_or_else(|| Duration::from_secs(0)); let ms = dur.as_millis(); - ms.try_into().unwrap_or(u64::MAX) + ms.try_into().unwrap_or(MAX_SAFE_MILLIS_DURATION) } pub(crate) fn tick_to_duration(&self, t: u64) -> Duration { diff --git a/tokio/tests/time_sleep.rs b/tokio/tests/time_sleep.rs index 4174a73b1..94022e3c0 100644 --- a/tokio/tests/time_sleep.rs +++ b/tokio/tests/time_sleep.rs @@ -267,6 +267,20 @@ async fn exactly_max() { time::sleep(ms(MAX_DURATION)).await; } +#[tokio::test] +async fn issue_5183() { + time::pause(); + + let big = std::time::Duration::from_secs(u64::MAX / 10); + // This is a workaround since awaiting sleep(big) will never finish. + #[rustfmt::skip] + tokio::select! { + biased; + _ = tokio::time::sleep(big) => {} + _ = tokio::time::sleep(std::time::Duration::from_nanos(1)) => {} + } +} + #[tokio::test] async fn no_out_of_bounds_close_to_max() { time::pause(); From c5d52c17ae800f1d39d7132fd9c452f2fae0e168 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sun, 11 Jun 2023 17:34:22 +0900 Subject: [PATCH 06/13] chore: enable cargo v2 resolver to prevent dev-deps from enabling log feature of mio (#5787) --- .github/workflows/ci.yml | 12 ++++++------ Cargo.toml | 2 +- tokio/Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3958755d5..d0d1366e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,7 +207,7 @@ jobs: # in order to run doctests for unstable features, we must also pass # the unstable cfg to RustDoc RUSTDOCFLAGS: --cfg tokio_unstable - + test-unstable-taskdump: name: test tokio full --unstable --taskdump needs: basics @@ -399,10 +399,10 @@ jobs: RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64 # https://github.com/tokio-rs/tokio/pull/5356 # https://github.com/tokio-rs/tokio/issues/5373 - - run: cargo hack build -p tokio --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going + - run: cargo hack build -p tokio --feature-powerset --depth 2 --keep-going env: RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64 --cfg tokio_no_const_mutex_new - - run: cargo hack build -p tokio --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going + - run: cargo hack build -p tokio --feature-powerset --depth 2 --keep-going env: RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings --cfg tokio_no_atomic_u64 @@ -421,15 +421,15 @@ jobs: - name: Install cargo-hack uses: taiki-e/install-action@cargo-hack - name: check --feature-powerset - run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going + run: cargo hack check --all --feature-powerset --depth 2 --keep-going # Try with unstable feature flags - name: check --feature-powerset --unstable - run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going + run: cargo hack check --all --feature-powerset --depth 2 --keep-going env: RUSTFLAGS: --cfg tokio_unstable -Dwarnings # Try with unstable and taskdump feature flags - name: check --feature-powerset --unstable --taskdump - run: cargo hack check --all --feature-powerset --depth 2 -Z avoid-dev-deps --keep-going + run: cargo hack check --all --feature-powerset --depth 2 --keep-going env: RUSTFLAGS: --cfg tokio_unstable --cfg tokio_taskdump -Dwarnings diff --git a/Cargo.toml b/Cargo.toml index bc01f1862..f3e19312e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] - +resolver = "2" members = [ "tokio", "tokio-macros", diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index da7988ce1..92897a485 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -132,7 +132,7 @@ nix = { version = "0.26", default-features = false, features = ["fs", "socket"] version = "0.48" optional = true -[target.'cfg(docsrs)'.dependencies.windows-sys] +[target.'cfg(windows)'.dev-dependencies.windows-sys] version = "0.48" features = [ "Win32_Foundation", From 6257712d6837fea36e1a201b7df75885b57a3148 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sun, 11 Jun 2023 19:02:12 +0900 Subject: [PATCH 07/13] ci: update cargo-check-external-types to 0.1.7 (#5786) --- .github/workflows/ci.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0d1366e2..51635a93e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -677,7 +677,7 @@ jobs: working-directory: tests-integration check-external-types: - name: check-external-types + name: check-external-types (${{ matrix.os }}) needs: basics runs-on: ${{ matrix.os }} strategy: @@ -685,20 +685,23 @@ jobs: os: - windows-latest - ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install Rust nightly-2022-11-16 - uses: dtolnay/rust-toolchain@master - with: + rust: # `check-external-types` requires a specific Rust nightly version. See # the README for details: https://github.com/awslabs/cargo-check-external-types - toolchain: nightly-2022-11-16 + - nightly-2023-05-31 + steps: + - uses: actions/checkout@v3 + - name: Install Rust ${{ matrix.rust }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} - uses: Swatinem/rust-cache@v2 + - name: Install cargo-check-external-types + uses: taiki-e/cache-cargo-install-action@v1 + with: + tool: cargo-check-external-types@0.1.7 - name: check-external-types - run: | - set -x - cargo install cargo-check-external-types --locked --version 0.1.6 - cargo check-external-types --all-features --config external-types.toml + run: cargo check-external-types --all-features --config external-types.toml working-directory: tokio check-fuzzing: From af6c87a045f413f2ce41d89db55663653a2dca67 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Mon, 12 Jun 2023 02:21:50 +0900 Subject: [PATCH 08/13] chore: upgrade remaining 2018 edition crates to 2021 edition (#5788) --- .github/workflows/ci.yml | 4 ++-- CONTRIBUTING.md | 8 ++++---- benches/Cargo.toml | 2 +- examples/Cargo.toml | 2 +- examples/tinyhttp.rs | 2 +- stress-test/Cargo.toml | 2 +- tests-build/Cargo.toml | 2 +- tests-integration/Cargo.toml | 2 +- tests-integration/tests/process_stdio.rs | 1 - tokio-macros/Cargo.toml | 2 +- tokio-stream/fuzz/Cargo.toml | 2 +- tokio/fuzz/Cargo.toml | 2 +- 12 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51635a93e..582e379f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -495,8 +495,8 @@ jobs: - name: "rustfmt --check" # Workaround for rust-lang/cargo#7732 run: | - if ! rustfmt --check --edition 2018 $(git ls-files '*.rs'); then - printf "Please run \`rustfmt --edition 2018 \$(git ls-files '*.rs')\` to fix rustfmt errors.\nSee CONTRIBUTING.md for more details.\n" >&2 + if ! rustfmt --check --edition 2021 $(git ls-files '*.rs'); then + printf "Please run \`rustfmt --edition 2021 \$(git ls-files '*.rs')\` to fix rustfmt errors.\nSee CONTRIBUTING.md for more details.\n" >&2 exit 1 fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index affa70f69..57a3bb366 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,10 +173,10 @@ command below instead: ``` # Mac or Linux -rustfmt --check --edition 2018 $(git ls-files '*.rs') +rustfmt --check --edition 2021 $(git ls-files '*.rs') # Powershell -Get-ChildItem . -Filter "*.rs" -Recurse | foreach { rustfmt --check --edition 2018 $_.FullName } +Get-ChildItem . -Filter "*.rs" -Recurse | foreach { rustfmt --check --edition 2021 $_.FullName } ``` The `--check` argument prints the things that need to be fixed. If you remove it, `rustfmt` will update your files locally instead. @@ -230,7 +230,7 @@ integration tests in the crate and follow the style. Some of our crates include a set of fuzz tests, this will be marked by a directory `fuzz`. It is a good idea to run fuzz tests after each change. -To get started with fuzz testing you'll need to install +To get started with fuzz testing you'll need to install [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz). `cargo install cargo-fuzz` @@ -678,4 +678,4 @@ When releasing a new version of a crate, follow these steps: [unit-tests]: https://doc.rust-lang.org/rust-by-example/testing/unit_testing.html [integration-tests]: https://doc.rust-lang.org/rust-by-example/testing/integration_testing.html [documentation-tests]: https://doc.rust-lang.org/rust-by-example/testing/doc_testing.html -[conditional-compilation]: https://doc.rust-lang.org/reference/conditional-compilation.html \ No newline at end of file +[conditional-compilation]: https://doc.rust-lang.org/reference/conditional-compilation.html diff --git a/benches/Cargo.toml b/benches/Cargo.toml index fdbf50c3c..47a830416 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -2,7 +2,7 @@ name = "benches" version = "0.0.0" publish = false -edition = "2018" +edition = "2021" [features] test-util = ["tokio/test-util"] diff --git a/examples/Cargo.toml b/examples/Cargo.toml index f8592c587..a244fccac 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -2,7 +2,7 @@ name = "examples" version = "0.0.0" publish = false -edition = "2018" +edition = "2021" # If you copy one of the examples into a new project, you should be using # [dependencies] instead, and delete the **path**. diff --git a/examples/tinyhttp.rs b/examples/tinyhttp.rs index 44534c6c1..8c6184f94 100644 --- a/examples/tinyhttp.rs +++ b/examples/tinyhttp.rs @@ -18,7 +18,7 @@ use futures::SinkExt; use http::{header::HeaderValue, Request, Response, StatusCode}; #[macro_use] extern crate serde_derive; -use std::{convert::TryFrom, env, error::Error, fmt, io}; +use std::{env, error::Error, fmt, io}; use tokio::net::{TcpListener, TcpStream}; use tokio_stream::StreamExt; use tokio_util::codec::{Decoder, Encoder, Framed}; diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml index e14256192..ee7431f09 100644 --- a/stress-test/Cargo.toml +++ b/stress-test/Cargo.toml @@ -2,7 +2,7 @@ name = "stress-test" version = "0.1.0" authors = ["Tokio Contributors "] -edition = "2018" +edition = "2021" publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/tests-build/Cargo.toml b/tests-build/Cargo.toml index 299af0cf4..251399c9f 100644 --- a/tests-build/Cargo.toml +++ b/tests-build/Cargo.toml @@ -2,7 +2,7 @@ name = "tests-build" version = "0.1.0" authors = ["Tokio Contributors "] -edition = "2018" +edition = "2021" publish = false [features] diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 5daeed08a..76b9956b8 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -2,7 +2,7 @@ name = "tests-integration" version = "0.1.0" authors = ["Tokio Contributors "] -edition = "2018" +edition = "2021" publish = false [[bin]] diff --git a/tests-integration/tests/process_stdio.rs b/tests-integration/tests/process_stdio.rs index 3ccb69002..526fd9ca6 100644 --- a/tests-integration/tests/process_stdio.rs +++ b/tests-integration/tests/process_stdio.rs @@ -7,7 +7,6 @@ use tokio::process::{Child, Command}; use tokio_test::assert_ok; use futures::future::{self, FutureExt}; -use std::convert::TryInto; use std::env; use std::io; use std::process::{ExitStatus, Stdio}; diff --git a/tokio-macros/Cargo.toml b/tokio-macros/Cargo.toml index 4ca789d55..a0289313a 100644 --- a/tokio-macros/Cargo.toml +++ b/tokio-macros/Cargo.toml @@ -5,7 +5,7 @@ name = "tokio-macros" # - Update CHANGELOG.md. # - Create "tokio-macros-1.x.y" git tag. version = "2.1.0" -edition = "2018" +edition = "2021" rust-version = "1.56" authors = ["Tokio Contributors "] license = "MIT" diff --git a/tokio-stream/fuzz/Cargo.toml b/tokio-stream/fuzz/Cargo.toml index e1003ee57..4a713b190 100644 --- a/tokio-stream/fuzz/Cargo.toml +++ b/tokio-stream/fuzz/Cargo.toml @@ -2,7 +2,7 @@ name = "tokio-stream-fuzz" version = "0.0.0" publish = false -edition = "2018" +edition = "2021" [package.metadata] cargo-fuzz = true diff --git a/tokio/fuzz/Cargo.toml b/tokio/fuzz/Cargo.toml index 4b47d7bdf..be05dc74b 100644 --- a/tokio/fuzz/Cargo.toml +++ b/tokio/fuzz/Cargo.toml @@ -2,7 +2,7 @@ name = "tokio-fuzz" version = "0.0.0" publish = false -edition = "2018" +edition = "2021" [package.metadata] cargo-fuzz = true From b7290910f7e471f0119ecd717af3a20a2f37fb09 Mon Sep 17 00:00:00 2001 From: icedrocket <114203630+icedrocket@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:43:12 +0900 Subject: [PATCH 09/13] sync: fix typo in batch semaphore (#5789) --- tokio/src/sync/batch_semaphore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/sync/batch_semaphore.rs b/tokio/src/sync/batch_semaphore.rs index a7885bdf1..a762f799d 100644 --- a/tokio/src/sync/batch_semaphore.rs +++ b/tokio/src/sync/batch_semaphore.rs @@ -264,7 +264,7 @@ impl Semaphore { match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => { - // TODO: Instrument once issue has been solved} + // TODO: Instrument once issue has been solved return Ok(()); } Err(actual) => curr = actual, From 00af6eff77f9fdcf6b4883671cb186b580cddce8 Mon Sep 17 00:00:00 2001 From: Andrew Mackenzie Date: Tue, 13 Jun 2023 12:42:40 +0200 Subject: [PATCH 10/13] net: add support for Redox OS (#5790) --- tokio/Cargo.toml | 4 ++-- tokio/src/net/unix/ucred.rs | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index 92897a485..d5b55eff6 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -121,11 +121,11 @@ tracing = { version = "0.1.25", default-features = false, features = ["std"], op backtrace = { version = "0.3.58" } [target.'cfg(unix)'.dependencies] -libc = { version = "0.2.42", optional = true } +libc = { version = "0.2.145", optional = true } signal-hook-registry = { version = "1.1.1", optional = true } [target.'cfg(unix)'.dev-dependencies] -libc = { version = "0.2.42" } +libc = { version = "0.2.145" } nix = { version = "0.26", default-features = false, features = ["fs", "socket"] } [target.'cfg(windows)'.dependencies.windows-sys] diff --git a/tokio/src/net/unix/ucred.rs b/tokio/src/net/unix/ucred.rs index 0a78fc5d6..edfab08ab 100644 --- a/tokio/src/net/unix/ucred.rs +++ b/tokio/src/net/unix/ucred.rs @@ -31,7 +31,12 @@ impl UCred { } } -#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] +#[cfg(any( + target_os = "linux", + target_os = "redox", + target_os = "android", + target_os = "openbsd" +))] pub(crate) use self::impl_linux::get_peer_cred; #[cfg(any(target_os = "netbsd"))] @@ -49,7 +54,12 @@ pub(crate) use self::impl_solaris::get_peer_cred; #[cfg(target_os = "aix")] pub(crate) use self::impl_aix::get_peer_cred; -#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] +#[cfg(any( + target_os = "linux", + target_os = "redox", + target_os = "android", + target_os = "openbsd" +))] pub(crate) mod impl_linux { use crate::net::unix::{self, UnixStream}; @@ -58,7 +68,7 @@ pub(crate) mod impl_linux { #[cfg(target_os = "openbsd")] use libc::sockpeercred as ucred; - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(any(target_os = "linux", target_os = "redox", target_os = "android"))] use libc::ucred; pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result { From 848482d2bb5761cd8fab3ef0dd92b8241e75e3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9B=8F=E4=B8=80?= Date: Wed, 14 Jun 2023 18:42:31 +0800 Subject: [PATCH 11/13] rt(threaded): adjust `transition_from_parked` behavior after introducing `disable_lifo_slot` feature (#5753) --- .../runtime/scheduler/multi_thread/queue.rs | 6 ++--- .../runtime/scheduler/multi_thread/worker.rs | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tokio/src/runtime/scheduler/multi_thread/queue.rs b/tokio/src/runtime/scheduler/multi_thread/queue.rs index 6444df88b..dd66fa2dd 100644 --- a/tokio/src/runtime/scheduler/multi_thread/queue.rs +++ b/tokio/src/runtime/scheduler/multi_thread/queue.rs @@ -105,9 +105,9 @@ pub(crate) fn local() -> (Steal, Local) { } impl Local { - /// Returns true if the queue has entries that can be stolen. - pub(crate) fn is_stealable(&self) -> bool { - !self.inner.is_empty() + /// Returns the number of entries in the queue + pub(crate) fn len(&self) -> usize { + self.inner.len() as usize } /// How many tasks can be pushed into the queue diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 47ff86a5c..7fc335f51 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -718,9 +718,7 @@ impl Context { // Place `park` back in `core` core.park = Some(park); - // If there are tasks available to steal, but this worker is not - // looking for tasks to steal, notify another worker. - if !core.is_searching && core.run_queue.is_stealable() { + if core.should_notify_others() { self.worker.handle.notify_parked_local(); } @@ -846,12 +844,25 @@ impl Core { worker.handle.transition_worker_from_searching(); } + fn has_tasks(&self) -> bool { + self.lifo_slot.is_some() || self.run_queue.has_tasks() + } + + fn should_notify_others(&self) -> bool { + // If there are tasks available to steal, but this worker is not + // looking for tasks to steal, notify another worker. + if self.is_searching { + return false; + } + self.lifo_slot.is_some() as usize + self.run_queue.len() > 1 + } + /// Prepares the worker state for parking. /// /// Returns true if the transition happened, false if there is work to do first. fn transition_to_parked(&mut self, worker: &Worker) -> bool { // Workers should not park if they have work to do - if self.lifo_slot.is_some() || self.run_queue.has_tasks() || self.is_traced { + if self.has_tasks() || self.is_traced { return false; } @@ -877,9 +888,9 @@ impl Core { /// Returns `true` if the transition happened. fn transition_from_parked(&mut self, worker: &Worker) -> bool { - // If a task is in the lifo slot, then we must unpark regardless of + // If a task is in the lifo slot/run queue, then we must unpark regardless of // being notified - if self.lifo_slot.is_some() { + if self.has_tasks() { // When a worker wakes, it should only transition to the "searching" // state when the wake originates from another worker *or* a new task // is pushed. We do *not* want the worker to transition to "searching" From fb0d305a7a78a97f032830c9e07bc932f093e165 Mon Sep 17 00:00:00 2001 From: Andrew Mackenzie Date: Mon, 19 Jun 2023 19:33:17 +0200 Subject: [PATCH 12/13] ci: build tokio for redox-os (#5800) --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 582e379f1..95050892d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - check-readme - test-hyper - x86_64-fortanix-unknown-sgx + - check-redox - wasm32-unknown-unknown - wasm32-wasi - check-external-types @@ -611,6 +612,21 @@ jobs: run: cargo build --target x86_64-fortanix-unknown-sgx --features rt,sync working-directory: tokio + check-redox: + name: build tokio for redox-os + needs: basics + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Rust ${{ env.rust_nightly }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.rust_nightly }} + target: x86_64-unknown-redox + - name: check tokio on redox + run: cargo check --target x86_64-unknown-redox --all-features + working-directory: tokio + wasm32-unknown-unknown: name: test tokio for wasm32-unknown-unknown needs: basics From 56c43655845b109f59a8cdd5d31d36992fc3ecef Mon Sep 17 00:00:00 2001 From: Jack Wrenn Date: Mon, 19 Jun 2023 13:34:48 -0400 Subject: [PATCH 13/13] tokio: improve taskdump documentation (#5805) - Add example trace output. - Add note on enabling unstable features. - Add note on performance overhead. --- tokio/src/runtime/handle.rs | 48 ++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/handle.rs b/tokio/src/runtime/handle.rs index 02ba279ff..be4743d47 100644 --- a/tokio/src/runtime/handle.rs +++ b/tokio/src/runtime/handle.rs @@ -406,9 +406,26 @@ cfg_taskdump! { /// # } /// ``` /// + /// This produces highly detailed traces of tasks; e.g.: + /// + /// ```plain + /// TASK 0: + /// ╼ dump::main::{{closure}}::a::{{closure}} at /tokio/examples/dump.rs:18:20 + /// └╼ dump::main::{{closure}}::b::{{closure}} at /tokio/examples/dump.rs:23:20 + /// └╼ dump::main::{{closure}}::c::{{closure}} at /tokio/examples/dump.rs:28:24 + /// └╼ tokio::sync::barrier::Barrier::wait::{{closure}} at /tokio/tokio/src/sync/barrier.rs:129:10 + /// └╼ as core::future::future::Future>::poll at /tokio/tokio/src/util/trace.rs:77:46 + /// └╼ tokio::sync::barrier::Barrier::wait_internal::{{closure}} at /tokio/tokio/src/sync/barrier.rs:183:36 + /// └╼ tokio::sync::watch::Receiver::changed::{{closure}} at /tokio/tokio/src/sync/watch.rs:604:55 + /// └╼ tokio::sync::watch::changed_impl::{{closure}} at /tokio/tokio/src/sync/watch.rs:755:18 + /// └╼ ::poll at /tokio/tokio/src/sync/notify.rs:1103:9 + /// └╼ tokio::sync::notify::Notified::poll_notified at /tokio/tokio/src/sync/notify.rs:996:32 + /// ``` + /// /// # Requirements /// /// ## Debug Info Must Be Available + /// /// To produce task traces, the application must **not** be compiled /// with split debuginfo. On Linux, including debuginfo within the /// application binary is the (correct) default. You can further ensure @@ -419,9 +436,30 @@ cfg_taskdump! { /// split-debuginfo = "off" /// ``` /// + /// ## Unstable Features + /// + /// This functionality is **unstable**, and requires both the + /// `tokio_unstable` and `tokio_taskdump` cfg flags to be set. + /// + /// You can do this by setting the `RUSTFLAGS` environment variable + /// before invoking `cargo`; e.g.: + /// ```bash + /// RUSTFLAGS="--cfg tokio_unstable --cfg tokio_taskdump" cargo run --example dump + /// ``` + /// + /// Or by [configuring][cargo-config] `rustflags` in + /// `.cargo/config.toml`: + /// ```text + /// [build] + /// rustflags = ["--cfg tokio_unstable", "--cfg tokio_taskdump"] + /// ``` + /// + /// [cargo-config]: + /// https://doc.rust-lang.org/cargo/reference/config.html + /// /// ## Platform Requirements /// - /// Task dumps are supported on Linux atop x86 and x86_64. + /// Task dumps are supported on Linux atop aarch64, x86 and x86_64. /// /// ## Current Thread Runtime Requirements /// @@ -431,6 +469,14 @@ cfg_taskdump! { /// /// # Limitations /// + /// ## Performance + /// + /// Although enabling the `tokio_taskdump` feature imposes virtually no + /// additional runtime overhead, actually calling `Handle::dump` is + /// expensive. The runtime must synchronize and pause its workers, then + /// re-poll every task in a special tracing mode. Avoid requesting dumps + /// often. + /// /// ## Local Executors /// /// Tasks managed by local executors (e.g., `FuturesUnordered` and